From 3fedb1a8285bfd5859765d73000f2c1056c81508 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 09:32:18 -0700 Subject: [PATCH 01/43] M0: hidden `quarto dev-call axe` skeleton command Registers `axe ` under the already-hidden `dev-call` parent, with the six v1 flags (--pages, --max-pages, --viewports, --themes, --timeout, --settle) parsed into a typed AxeScanConfig and echoed. The scan, aggregate and report stages follow. --- src/command/dev-call/axe/cmd.ts | 166 ++++++++++++++++++++++++++++++++ src/command/dev-call/cmd.ts | 4 +- 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 src/command/dev-call/axe/cmd.ts diff --git a/src/command/dev-call/axe/cmd.ts b/src/command/dev-call/axe/cmd.ts new file mode 100644 index 00000000000..55e29784545 --- /dev/null +++ b/src/command/dev-call/axe/cmd.ts @@ -0,0 +1,166 @@ +/* + * cmd.ts + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { Command } from "cliffy/command/mod.ts"; +import { info } from "../../../deno_ral/log.ts"; +import { ErrorEx } from "../../../core/lib/error.ts"; + +// Fixed in v1: this prototype deliberately has no knobs for the output dir, +// the baseline path or whether the report is written. +export const kAxeOutputDir = "_axe-checks"; +export const kAxeBaselineFile = "_axe-baseline.json"; + +export const kDefaultViewports = "1440x900,390x844"; +export const kDefaultThemes = "light,dark"; +export const kDefaultTimeout = 30000; +export const kDefaultSettle = 500; + +export interface AxeViewport { + width: number; + height: number; + // canonical "WxH" label, used in cell ids and findings.json + label: string; +} + +export type AxeTheme = "light" | "dark"; + +export interface AxeScanConfig { + siteDir: string; + // undefined means "every *.html under siteDir" + pages?: string[]; + // undefined means "no cap" + maxPages?: number; + viewports: AxeViewport[]; + themes: AxeTheme[]; + timeout: number; + settle: number; +} + +function optionError(message: string): ErrorEx { + return new ErrorEx("AxeOptionError", message, false, false); +} + +function splitList(value: string): string[] { + return value.split(",").map((entry) => entry.trim()).filter((entry) => + entry.length > 0 + ); +} + +function parseViewports(value: string): AxeViewport[] { + const viewports = splitList(value).map((entry) => { + const match = entry.match(/^(\d+)x(\d+)$/i); + if (!match) { + throw optionError( + `Invalid viewport '${entry}': expected WxH, e.g. 1440x900.`, + ); + } + return { + width: parseInt(match[1], 10), + height: parseInt(match[2], 10), + label: `${match[1]}x${match[2]}`, + }; + }); + if (viewports.length === 0) { + throw optionError("No viewports specified."); + } + return viewports; +} + +function parseThemes(value: string): AxeTheme[] { + const themes = splitList(value).map((entry) => { + const theme = entry.toLowerCase(); + if (theme !== "light" && theme !== "dark") { + throw optionError( + `Invalid theme '${entry}': expected 'light' or 'dark'.`, + ); + } + return theme; + }); + if (themes.length === 0) { + throw optionError("No themes specified."); + } + return themes; +} + +function parsePositiveInt(value: unknown, flag: string): number { + const parsed = typeof value === "number" ? value : parseInt(`${value}`, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw optionError( + `Invalid ${flag} '${value}': expected a positive integer.`, + ); + } + return parsed; +} + +// deno-lint-ignore no-explicit-any +export function axeScanConfig(options: any, siteDir: string): AxeScanConfig { + return { + siteDir, + pages: options.pages ? splitList(options.pages) : undefined, + maxPages: options.maxPages === undefined + ? undefined + : parsePositiveInt(options.maxPages, "--max-pages"), + viewports: parseViewports(options.viewports ?? kDefaultViewports), + themes: parseThemes(options.themes ?? kDefaultThemes), + timeout: parsePositiveInt(options.timeout ?? kDefaultTimeout, "--timeout"), + settle: parsePositiveInt(options.settle ?? kDefaultSettle, "--settle"), + }; +} + +export const axeCommand = new Command() + .name("axe") + .hidden() + .arguments("") + .description( + "Scan a rendered site for accessibility violations with axe-core.\n\n" + + "Prototype: scans every page in across the viewport x theme " + + "matrix, groups violations by root-cause signature, reconciles " + + `${kAxeBaselineFile} in the working directory, and writes findings.json ` + + `plus report.html to ${kAxeOutputDir}/.`, + ) + .option( + "--pages ", + "Comma-separated site-relative globs to scan (default: all *.html).", + ) + .option( + "--max-pages ", + "Cap the number of pages scanned (sorted, first n).", + ) + .option( + "--viewports ", + "Comma-separated WxH viewports to emulate.", + { default: kDefaultViewports }, + ) + .option( + "--themes ", + "Comma-separated color schemes to emulate (light, dark).", + { default: kDefaultThemes }, + ) + .option( + "--timeout ", + "Per-cell budget in milliseconds.", + { default: kDefaultTimeout }, + ) + .option( + "--settle ", + "Delay in milliseconds after load before axe runs.", + { default: kDefaultSettle }, + ) + .example( + "Scan a rendered site", + "quarto dev-call axe _site", + ) + .example( + "Scan two pages, desktop light only", + "quarto dev-call axe _site --pages index.html,about.html " + + "--viewports 1440x900 --themes light", + ) + // deno-lint-ignore no-explicit-any + .action((options: any, siteDir: string) => { + const config = axeScanConfig(options, siteDir); + // M0: parse and echo. The scan/aggregate/report stages come next. + info(JSON.stringify(config, null, 2)); + }); diff --git a/src/command/dev-call/cmd.ts b/src/command/dev-call/cmd.ts index 4a2b892915c..2369cbea188 100644 --- a/src/command/dev-call/cmd.ts +++ b/src/command/dev-call/cmd.ts @@ -7,6 +7,7 @@ import { showAstTraceCommand } from "./show-ast-trace/cmd.ts"; import { makeAstDiagramCommand } from "./make-ast-diagram/cmd.ts"; import { pullGitSubtreeCommand } from "./pull-git-subtree/cmd.ts"; import { typstGatherCommand } from "./typst-gather/cmd.ts"; +import { axeCommand } from "./axe/cmd.ts"; type CommandOptionInfo = { name: string; @@ -79,4 +80,5 @@ export const devCallCommand = new Command() .command("show-ast-trace", showAstTraceCommand) .command("make-ast-diagram", makeAstDiagramCommand) .command("pull-git-subtree", pullGitSubtreeCommand) - .command("typst-gather", typstGatherCommand); + .command("typst-gather", typstGatherCommand) + .command("axe", axeCommand); From f36c85dfad5ac3217ed68d87ec3620b91f682520 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 09:58:38 -0700 Subject: [PATCH 02/43] M1: scan an already-rendered site end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the quarto-web harness's raw-CDP driver to typed TypeScript and replaces its render-time axe hook with scan-time injection: after load and settle, the vendored axe.min.js (4.10.3) is evaluated in the page and axe.run() awaited under the per-cell timeout. Any rendered site now scans as-is, offline, on one known axe version — so the CDN warm-up hack, the console-payload sniffing and the special render all come out. The orchestrator serves the site dir with httpFileRequestHandler, launches headless Chrome via getBrowserExecutablePath() on its own throwaway profile, walks the page x viewport x theme matrix, and writes raw per-cell JSON to _axe-checks/cells/. Cells fail closed: timeout, evaluation error or a missing payload is reported and exits 2, never a pass. Chrome rewrites its profile as it shuts down, so the temp profile dir is removed only after the process has really exited. On Ctrl-C an onCleanup handler kills Chrome and leaves the dir behind, since cleanup handlers can't await. Verified against a rendered website: all cells ok; identical results with Chrome's DNS blackholed (offline); prefers-color-scheme emulation catches a dark-only color-contrast failure and the mobile viewport a mobile-only one; a cell that blocks the main thread times out and the next cell still scans; SIGINT leaves no orphan Chrome; bogus, empty and non-directory site-dirs exit 2. Adds an optional onListen to handleHttpRequests so the scan's own progress output isn't preceded by Deno's "Listening on ..." line. --- src/command/dev-call/axe/cmd.ts | 252 ++++++++----- src/command/dev-call/axe/config.ts | 111 ++++++ src/command/dev-call/axe/scan.ts | 572 +++++++++++++++++++++++++++++ src/core/http-server.ts | 3 + 4 files changed, 847 insertions(+), 91 deletions(-) create mode 100644 src/command/dev-call/axe/config.ts create mode 100644 src/command/dev-call/axe/scan.ts diff --git a/src/command/dev-call/axe/cmd.ts b/src/command/dev-call/axe/cmd.ts index 55e29784545..ca17251fd63 100644 --- a/src/command/dev-call/axe/cmd.ts +++ b/src/command/dev-call/axe/cmd.ts @@ -1,113 +1,182 @@ /* * cmd.ts * + * `quarto dev-call axe` — hidden prototype accessibility scanner. Serves an + * already-rendered site, drives headless Chrome over the page x viewport x + * theme matrix with quarto-cli's vendored axe-core, and writes the raw per-cell + * payloads. Aggregation and the HTML report come next. + * * Copyright (C) 2026 Posit Software, PBC */ import { Command } from "cliffy/command/mod.ts"; -import { info } from "../../../deno_ral/log.ts"; -import { ErrorEx } from "../../../core/lib/error.ts"; +import { error, info } from "../../../deno_ral/log.ts"; +import { ensureDirSync, existsSync, walkSync } from "../../../deno_ral/fs.ts"; +import { globToRegExp, join, relative } from "../../../deno_ral/path.ts"; +import { pathWithForwardSlashes } from "../../../core/path.ts"; +import { findOpenPort } from "../../../core/port.ts"; +import { httpFileRequestHandler } from "../../../core/http.ts"; +import { handleHttpRequests } from "../../../core/http-server.ts"; +import { + AxeScanConfig, + axeScanConfig, + kAxeBaselineFile, + kAxeOutputDir, + kDefaultSettle, + kDefaultThemes, + kDefaultTimeout, + kDefaultViewports, +} from "./config.ts"; +import { AxeCell, launchScanBrowser, runAxeScan } from "./scan.ts"; -// Fixed in v1: this prototype deliberately has no knobs for the output dir, -// the baseline path or whether the report is written. -export const kAxeOutputDir = "_axe-checks"; -export const kAxeBaselineFile = "_axe-baseline.json"; +/** Scan complete: every cell produced an axe payload. */ +const kExitComplete = 0; +/** + * Scan incomplete: a not-ok cell, no browser, or nothing to scan. There is + * deliberately no exit 1 in v1 — findings never fail the command. + */ +const kExitIncomplete = 2; -export const kDefaultViewports = "1440x900,390x844"; -export const kDefaultThemes = "light,dark"; -export const kDefaultTimeout = 30000; -export const kDefaultSettle = 500; +/** + * Every `*.html` under `siteDir`, as sorted site-relative forward-slash paths, + * narrowed by `--pages` and capped by `--max-pages`. Sorting before the cap is + * what makes `--max-pages` deterministic. + */ +export function discoverPages(config: AxeScanConfig): string[] { + const pages: string[] = []; + for (const entry of walkSync(config.siteDir, { includeDirs: false })) { + if (entry.path.endsWith(".html")) { + pages.push( + pathWithForwardSlashes(relative(config.siteDir, entry.path)), + ); + } + } + pages.sort(); -export interface AxeViewport { - width: number; - height: number; - // canonical "WxH" label, used in cell ids and findings.json - label: string; + let selected = pages; + if (config.pages) { + const patterns = config.pages.map((glob) => + globToRegExp(glob, { extended: true, globstar: true }) + ); + selected = pages.filter((page) => + patterns.some((pattern) => pattern.test(page)) + ); + } + if (config.maxPages !== undefined) { + selected = selected.slice(0, config.maxPages); + } + return selected; } -export type AxeTheme = "light" | "dark"; - -export interface AxeScanConfig { - siteDir: string; - // undefined means "every *.html under siteDir" - pages?: string[]; - // undefined means "no cap" - maxPages?: number; - viewports: AxeViewport[]; - themes: AxeTheme[]; - timeout: number; - settle: number; +function cellLine(cell: AxeCell): string { + const name = `${cell.page} ${cell.viewport} ${cell.theme}`; + if (cell.status !== "ok") { + return ` ${name.padEnd(56)} ${cell.status.toUpperCase()}`; + } + const violations = cell.result!.violations; + const ids = violations.map((violation) => violation.id).join(",") || "(none)"; + return ` ${name.padEnd(56)} ${ + String(violations.length).padStart(3) + } ${ids}`; } -function optionError(message: string): ErrorEx { - return new ErrorEx("AxeOptionError", message, false, false); -} +/** + * Run the scan stage against `config.siteDir` and return the process exit code. + */ +export async function axeScan(config: AxeScanConfig): Promise { + if (!existsSync(config.siteDir)) { + error(`Site directory not found: ${config.siteDir}`); + return kExitIncomplete; + } + if (!Deno.statSync(config.siteDir).isDirectory) { + error(`Not a directory: ${config.siteDir}`); + return kExitIncomplete; + } -function splitList(value: string): string[] { - return value.split(",").map((entry) => entry.trim()).filter((entry) => - entry.length > 0 - ); -} + const pages = discoverPages(config); + if (pages.length === 0) { + error( + config.pages + ? `No pages in ${config.siteDir} matched --pages.` + : `No *.html pages found in ${config.siteDir}.`, + ); + return kExitIncomplete; + } -function parseViewports(value: string): AxeViewport[] { - const viewports = splitList(value).map((entry) => { - const match = entry.match(/^(\d+)x(\d+)$/i); - if (!match) { - throw optionError( - `Invalid viewport '${entry}': expected WxH, e.g. 1440x900.`, - ); - } - return { - width: parseInt(match[1], 10), - height: parseInt(match[2], 10), - label: `${match[1]}x${match[2]}`, - }; + const cellsDir = join(kAxeOutputDir, "cells"); + ensureDirSync(cellsDir); + + // Serve the site dir first, so the bound port can't be handed to Chrome next. + const sitePort = findOpenPort(); + const server = handleHttpRequests({ + port: sitePort, + hostname: "127.0.0.1", + handler: httpFileRequestHandler({ + baseDir: config.siteDir, + defaultFile: "index.html", + }), + // the scan's own progress output is the interesting part + onListen: () => {}, }); - if (viewports.length === 0) { - throw optionError("No viewports specified."); + const baseUrl = `http://127.0.0.1:${sitePort}`; + + const cellCount = pages.length * config.viewports.length * + config.themes.length; + info( + `axe: ${pages.length} pages × ${config.viewports.length} viewports × ` + + `${config.themes.length} themes — ${cellCount} cells`, + ); + + let browser; + try { + browser = await launchScanBrowser(findOpenPort(9222)); + } catch (e) { + error( + `Could not start headless Chrome: ${ + e instanceof Error ? e.message : String(e) + }`, + ); + server.stop(); + return kExitIncomplete; } - return viewports; -} -function parseThemes(value: string): AxeTheme[] { - const themes = splitList(value).map((entry) => { - const theme = entry.toLowerCase(); - if (theme !== "light" && theme !== "dark") { - throw optionError( - `Invalid theme '${entry}': expected 'light' or 'dark'.`, + try { + const scan = await runAxeScan( + browser.client, + config, + baseUrl, + pages, + cellsDir, + (cell) => info(cellLine(cell)), + ); + + const notOk = scan.cells.filter((cell) => cell.status !== "ok"); + info(""); + info( + `${scan.cells.length} cells: ${scan.cells.length - notOk.length} ok` + + (notOk.length ? `, ${notOk.length} not ok` : ""), + ); + if (scan.axeVersion) { + info( + `axe-core ${scan.axeVersion} (quarto-cli's vendored build, injected at scan time)`, ); } - return theme; - }); - if (themes.length === 0) { - throw optionError("No themes specified."); - } - return themes; -} + info(`cells: ${cellsDir}`); -function parsePositiveInt(value: unknown, flag: string): number { - const parsed = typeof value === "number" ? value : parseInt(`${value}`, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw optionError( - `Invalid ${flag} '${value}': expected a positive integer.`, - ); + if (notOk.length) { + for (const cell of notOk) { + error( + ` ${cell.page} ${cell.viewport} ${cell.theme}: ${cell.status}` + + (cell.message ? ` — ${cell.message}` : ""), + ); + } + return kExitIncomplete; + } + return kExitComplete; + } finally { + await browser.close(); + server.stop(); } - return parsed; -} - -// deno-lint-ignore no-explicit-any -export function axeScanConfig(options: any, siteDir: string): AxeScanConfig { - return { - siteDir, - pages: options.pages ? splitList(options.pages) : undefined, - maxPages: options.maxPages === undefined - ? undefined - : parsePositiveInt(options.maxPages, "--max-pages"), - viewports: parseViewports(options.viewports ?? kDefaultViewports), - themes: parseThemes(options.themes ?? kDefaultThemes), - timeout: parsePositiveInt(options.timeout ?? kDefaultTimeout, "--timeout"), - settle: parsePositiveInt(options.settle ?? kDefaultSettle, "--settle"), - }; } export const axeCommand = new Command() @@ -159,8 +228,9 @@ export const axeCommand = new Command() "--viewports 1440x900 --themes light", ) // deno-lint-ignore no-explicit-any - .action((options: any, siteDir: string) => { - const config = axeScanConfig(options, siteDir); - // M0: parse and echo. The scan/aggregate/report stages come next. - info(JSON.stringify(config, null, 2)); + .action(async (options: any, siteDir: string) => { + const code = await axeScan(axeScanConfig(options, siteDir)); + if (code !== kExitComplete) { + Deno.exit(code); + } }); diff --git a/src/command/dev-call/axe/config.ts b/src/command/dev-call/axe/config.ts new file mode 100644 index 00000000000..907863e7b33 --- /dev/null +++ b/src/command/dev-call/axe/config.ts @@ -0,0 +1,111 @@ +/* + * config.ts + * + * Command-surface types and flag parsing for `quarto dev-call axe`. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { ErrorEx } from "../../../core/lib/error.ts"; + +// Fixed in v1: this prototype deliberately has no knobs for the output dir, +// the baseline path or whether the report is written. +export const kAxeOutputDir = "_axe-checks"; +export const kAxeBaselineFile = "_axe-baseline.json"; + +export const kDefaultViewports = "1440x900,390x844"; +export const kDefaultThemes = "light,dark"; +export const kDefaultTimeout = 30000; +export const kDefaultSettle = 500; + +export interface AxeViewport { + width: number; + height: number; + // canonical "WxH" label, used in cell ids and findings.json + label: string; +} + +export type AxeTheme = "light" | "dark"; + +export interface AxeScanConfig { + siteDir: string; + // undefined means "every *.html under siteDir" + pages?: string[]; + // undefined means "no cap" + maxPages?: number; + viewports: AxeViewport[]; + themes: AxeTheme[]; + timeout: number; + settle: number; +} + +function optionError(message: string): ErrorEx { + return new ErrorEx("AxeOptionError", message, false, false); +} + +function splitList(value: string): string[] { + return value.split(",").map((entry) => entry.trim()).filter((entry) => + entry.length > 0 + ); +} + +function parseViewports(value: string): AxeViewport[] { + const viewports = splitList(value).map((entry) => { + const match = entry.match(/^(\d+)x(\d+)$/i); + if (!match) { + throw optionError( + `Invalid viewport '${entry}': expected WxH, e.g. 1440x900.`, + ); + } + return { + width: parseInt(match[1], 10), + height: parseInt(match[2], 10), + label: `${match[1]}x${match[2]}`, + }; + }); + if (viewports.length === 0) { + throw optionError("No viewports specified."); + } + return viewports; +} + +function parseThemes(value: string): AxeTheme[] { + const themes = splitList(value).map((entry) => { + const theme = entry.toLowerCase(); + if (theme !== "light" && theme !== "dark") { + throw optionError( + `Invalid theme '${entry}': expected 'light' or 'dark'.`, + ); + } + return theme; + }); + if (themes.length === 0) { + throw optionError("No themes specified."); + } + return themes; +} + +function parsePositiveInt(value: unknown, flag: string): number { + const parsed = typeof value === "number" ? value : parseInt(`${value}`, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw optionError( + `Invalid ${flag} '${value}': expected a positive integer.`, + ); + } + return parsed; +} + +// deno-lint-ignore no-explicit-any +export function axeScanConfig(options: any, siteDir: string): AxeScanConfig { + return { + siteDir, + pages: options.pages ? splitList(options.pages) : undefined, + maxPages: options.maxPages === undefined + ? undefined + : parsePositiveInt(options.maxPages, "--max-pages"), + viewports: parseViewports(options.viewports ?? kDefaultViewports), + themes: parseThemes(options.themes ?? kDefaultThemes), + timeout: parsePositiveInt(options.timeout ?? kDefaultTimeout, "--timeout"), + settle: parsePositiveInt(options.settle ?? kDefaultSettle, "--settle"), + }; +} diff --git a/src/command/dev-call/axe/scan.ts b/src/command/dev-call/axe/scan.ts new file mode 100644 index 00000000000..cbaa7ef3591 --- /dev/null +++ b/src/command/dev-call/axe/scan.ts @@ -0,0 +1,572 @@ +/* + * scan.ts + * + * The scan stage of `quarto dev-call axe`: drives headless Chrome over raw CDP + * and runs quarto-cli's vendored axe-core against every page x viewport x theme + * cell of an already-rendered site. + * + * Ported from the quarto-web harness (`_tools/axe/scan.mjs`), with one + * architectural change: axe is injected at scan time rather than by a + * render-time hook, so any rendered site scans as-is and offline. + * + * Cells fail CLOSED. A timeout, an evaluation error, or a result that doesn't + * look like an axe payload is an infrastructure failure in the output and in + * the exit code — never a pass. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { dirname, join } from "../../../deno_ral/path.ts"; +import { debug } from "../../../deno_ral/log.ts"; +import { sleep } from "../../../core/async.ts"; +import { formatResourcePath } from "../../../core/resources.ts"; +import { getBrowserExecutablePath } from "../../../core/puppeteer.ts"; +import { onCleanup } from "../../../core/cleanup.ts"; +import { getenv } from "../../../core/env.ts"; +import { safeRemoveDirSync } from "../../../deno_ral/fs.ts"; +import { AxeScanConfig, AxeTheme, AxeViewport } from "./config.ts"; + +/** Status of a single scanned cell. Anything but "ok" fails closed. */ +export type AxeCellStatus = "ok" | "timeout" | "error" | "no-payload"; + +/** A single axe violation node, as axe-core reports it. */ +export interface AxeViolationNode { + html: string; + target: string[]; + failureSummary?: string; + impact?: string | null; +} + +/** A single axe violation, as axe-core reports it. */ +export interface AxeViolation { + id: string; + impact?: string | null; + tags: string[]; + description: string; + help: string; + helpUrl: string; + nodes: AxeViolationNode[]; +} + +/** The slice of axe-core's `axe.run()` payload the scanner keeps. */ +export interface AxeRunResult { + violations: AxeViolation[]; + testEngine?: { name: string; version: string }; + url?: string; + timestamp?: string; +} + +/** One page x viewport x theme cell: the unit of scanning and of fail-closed. */ +export interface AxeCell { + page: string; + viewport: string; + theme: AxeTheme; + url: string; + status: AxeCellStatus; + /** Present when status is not "ok". */ + message?: string; + /** Present when status is "ok". */ + result?: AxeRunResult; + /** Wall-clock milliseconds spent on this cell. */ + elapsed: number; +} + +// --------------------------------------------------------------------------- +// CDP client +// --------------------------------------------------------------------------- + +interface CdpMessage { + id?: number; + method?: string; + params?: Record; + result?: unknown; + error?: { code: number; message: string }; +} + +/** + * Minimal Chrome DevTools Protocol client: send a command, await its result, + * and wait for a named event. Everything the scanner needs is six methods, so + * there is deliberately no wrapper library here (see the design note). + */ +export class CdpClient { + private nextId = 0; + private pending = new Map< + number, + { resolve: (result: unknown) => void; reject: (err: Error) => void } + >(); + private listeners = new Map< + string, + Set<(params: Record) => void> + >(); + private closed = false; + + private constructor(private readonly ws: WebSocket) { + ws.addEventListener("message", (ev: MessageEvent) => { + this.onMessage(ev.data as string); + }); + ws.addEventListener("close", () => { + this.closed = true; + this.rejectPending(new Error("CDP connection closed")); + }); + } + + static connect(wsUrl: string): Promise { + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + ws.addEventListener("open", () => resolve(new CdpClient(ws)), { + once: true, + }); + ws.addEventListener( + "error", + () => reject(new Error(`Failed to connect to CDP at ${wsUrl}`)), + { once: true }, + ); + }); + } + + send( + method: string, + params: Record = {}, + ): Promise { + if (this.closed) { + return Promise.reject(new Error("CDP connection closed")); + } + const id = ++this.nextId; + return new Promise((resolve, reject) => { + this.pending.set(id, { + resolve: resolve as (result: unknown) => void, + reject, + }); + this.ws.send(JSON.stringify({ id, method, params })); + }); + } + + /** + * Resolve on the next occurrence of `method`. The returned `cancel` must be + * called when the caller stops caring (e.g. the cell timed out), so a late + * event can't resolve a waiter that belongs to a previous cell. + */ + once( + method: string, + ): { event: Promise>; cancel: () => void } { + let handler: (params: Record) => void = () => {}; + const event = new Promise>((resolve) => { + handler = (params) => { + this.off(method, handler); + resolve(params); + }; + this.on(method, handler); + }); + return { event, cancel: () => this.off(method, handler) }; + } + + close() { + if (!this.closed) { + this.closed = true; + try { + this.ws.close(); + } catch (_e) { + // the socket is going away regardless + } + this.rejectPending(new Error("CDP connection closed")); + } + } + + private on( + method: string, + handler: (params: Record) => void, + ) { + let handlers = this.listeners.get(method); + if (!handlers) { + handlers = new Set(); + this.listeners.set(method, handlers); + } + handlers.add(handler); + } + + private off( + method: string, + handler: (params: Record) => void, + ) { + this.listeners.get(method)?.delete(handler); + } + + private onMessage(data: string) { + let msg: CdpMessage; + try { + msg = JSON.parse(data); + } catch (_e) { + debug(`[axe] unparseable CDP message: ${data.slice(0, 200)}`); + return; + } + if (msg.id !== undefined) { + const entry = this.pending.get(msg.id); + if (entry) { + this.pending.delete(msg.id); + if (msg.error) { + entry.reject( + new Error(`CDP error ${msg.error.code}: ${msg.error.message}`), + ); + } else { + entry.resolve(msg.result); + } + } + return; + } + if (msg.method) { + for (const handler of this.listeners.get(msg.method) ?? []) { + handler(msg.params ?? {}); + } + } + } + + private rejectPending(err: Error) { + for (const entry of this.pending.values()) { + entry.reject(err); + } + this.pending.clear(); + } +} + +// --------------------------------------------------------------------------- +// Browser +// --------------------------------------------------------------------------- + +export interface ScanBrowser { + client: CdpClient; + close: () => Promise; +} + +interface CdpTarget { + type: string; + webSocketDebuggerUrl?: string; +} + +async function waitForCdp( + port: number, + timeout: number, +): Promise { + const interval = 50; + let waited = 0; + let lastError = "no CDP page target"; + while (waited < timeout) { + try { + const response = await fetch(`http://127.0.0.1:${port}/json/list`); + if (response.ok) { + const targets = (await response.json()) as CdpTarget[]; + const page = targets.find((target) => + target.type === "page" && target.webSocketDebuggerUrl + ); + if (page?.webSocketDebuggerUrl) { + return page.webSocketDebuggerUrl; + } + } else { + // drain the body so the connection can be reused + await response.body?.cancel(); + lastError = `CDP endpoint returned ${response.status}`; + } + } catch (e) { + lastError = e instanceof Error ? e.message : String(e); + } + await sleep(interval); + waited += interval; + } + throw new Error( + `Timed out waiting for headless Chrome on port ${port} (${lastError}).`, + ); +} + +/** + * Launch headless Chrome on its own CDP port and connect to its page target. + * The browser gets a throwaway user-data-dir so it can't attach to (or be + * short-circuited by) a Chrome the user already has running. + */ +export async function launchScanBrowser(port: number): Promise { + const executable = await getBrowserExecutablePath(); + const userDataDir = Deno.makeTempDirSync({ prefix: "quarto-axe-chrome" }); + + // Same headless-mode escape hatch as src/core/cri/cri.ts. + const headlessMode = getenv("QUARTO_CHROMIUM_HEADLESS_MODE", "none"); + const command = new Deno.Command(executable, { + args: [ + `--headless${headlessMode === "none" ? "" : "=" + headlessMode}`, + "--no-sandbox", + "--disable-gpu", + "--hide-scrollbars", + `--user-data-dir=${userDataDir}`, + `--remote-debugging-port=${port}`, + "about:blank", + ], + stdout: "null", + stderr: "piped", + }); + const process = command.spawn(); + + // Chrome is chatty on stderr, and an unread pipe eventually blocks it. Drain + // it to the debug log, keeping the tail around to explain a failed launch. + let stderrTail = ""; + const draining = (async () => { + const stream = process.stderr.pipeThrough(new TextDecoderStream()); + for await (const chunk of stream) { + debug(`[axe chrome] ${chunk.trimEnd()}`); + stderrTail = (stderrTail + chunk).slice(-2000); + } + })(); + + let killed = false; + const kill = () => { + if (killed) { + return; + } + killed = true; + try { + process.kill(); + } catch (_e) { + // already gone + } + }; + // Chrome will not terminate on its own, and Ctrl-C must not orphan it. The + // profile dir is left behind on that path: removing it means waiting for the + // process to exit, and cleanup handlers are synchronous. + onCleanup(kill); + + // Chrome rewrites its profile as it shuts down, so the dir can only be + // removed once the process is really gone. + const shutdown = async () => { + kill(); + await process.status; + await draining.catch(() => {}); + try { + safeRemoveDirSync(userDataDir, dirname(userDataDir)); + } catch (_e) { + // a leftover temp dir is not worth failing the scan over + } + }; + + let client: CdpClient; + try { + const wsUrl = await waitForCdp(port, 15000); + client = await CdpClient.connect(wsUrl); + } catch (e) { + await shutdown(); + const detail = stderrTail.trim(); + throw new Error( + (e instanceof Error ? e.message : String(e)) + + (detail ? `\nChrome said: ${detail}` : ""), + ); + } + + return { + client, + close: async () => { + client.close(); + await shutdown(); + }, + }; +} + +// --------------------------------------------------------------------------- +// Scanning +// --------------------------------------------------------------------------- + +/** Path to the axe-core build quarto-cli already ships for HTML output. */ +export function vendoredAxePath(): string { + return formatResourcePath("html", join("axe", "axe.min.js")); +} + +// axe context: keep our own dev chrome and the tabster shim out of results. +// (https://github.com/microsoft/tabster/issues/288 — won't fix upstream.) +const kAxeRunExpression = ` +(function () { + return axe.run( + { exclude: ["[data-tabster-dummy]", ".quarto-axe-report"] }, + // v1 reports violations only, so axe can skip collecting full pass detail. + { resultTypes: ["violations"] } + ).then(function (result) { + return { + violations: result.violations, + testEngine: result.testEngine, + url: result.url, + timestamp: result.timestamp, + }; + }); +})() +`; + +interface RuntimeEvaluateResult { + result?: { value?: unknown }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +function evaluateError(response: RuntimeEvaluateResult): string | undefined { + const details = response.exceptionDetails; + if (!details) { + return undefined; + } + return details.exception?.description ?? details.text ?? "evaluation failed"; +} + +async function scanCell( + client: CdpClient, + axeSource: string, + config: AxeScanConfig, + page: string, + viewport: AxeViewport, + theme: AxeTheme, + url: string, +): Promise { + const started = Date.now(); + const cell = ( + status: AxeCellStatus, + extra: Partial = {}, + ): AxeCell => ({ + page, + viewport: viewport.label, + theme, + url, + status, + elapsed: Date.now() - started, + ...extra, + }); + + const load = client.once("Page.loadEventFired"); + const run = (async (): Promise => { + await client.send("Emulation.setDeviceMetricsOverride", { + width: viewport.width, + height: viewport.height, + deviceScaleFactor: 1, + mobile: false, + }); + await client.send("Emulation.setEmulatedMedia", { + features: [{ name: "prefers-color-scheme", value: theme }], + }); + + const navigation = await client.send<{ errorText?: string }>( + "Page.navigate", + { url }, + ); + if (navigation.errorText) { + return cell("error", { + message: `navigation failed: ${navigation.errorText}`, + }); + } + await load.event; + + // Let webfonts, deferred scripts and any client-side layout settle before + // axe reads computed style. + await sleep(config.settle); + + const injected = await client.send( + "Runtime.evaluate", + { expression: axeSource, returnByValue: false }, + ); + const injectError = evaluateError(injected); + if (injectError) { + return cell("error", { message: `axe injection failed: ${injectError}` }); + } + + const evaluated = await client.send( + "Runtime.evaluate", + { + expression: kAxeRunExpression, + awaitPromise: true, + returnByValue: true, + }, + ); + const runError = evaluateError(evaluated); + if (runError) { + return cell("error", { message: `axe.run() failed: ${runError}` }); + } + + const value = evaluated.result?.value as AxeRunResult | undefined; + if (!value || !Array.isArray(value.violations)) { + return cell("no-payload", { + message: "axe.run() returned no violations array", + }); + } + return cell("ok", { result: value }); + })(); + + const timedOut = Symbol("timeout"); + const outcome = await Promise.race([ + run, + sleep(config.timeout).then(() => timedOut), + ]); + if (outcome !== timedOut) { + return outcome as AxeCell; + } + + // Stop caring about this cell's load event, and take the page down so a hung + // axe.run() can't bleed into the next cell. + load.cancel(); + try { + await client.send("Page.navigate", { url: "about:blank" }); + } catch (_e) { + // if even that fails the next cell will report the real problem + } + return cell("timeout", { + message: `cell exceeded --timeout of ${config.timeout}ms`, + }); +} + +/** Filesystem-safe cell name, e.g. `docs_index__1440x900__dark`. */ +export function cellName( + page: string, + viewport: string, + theme: string, +): string { + const slug = page.replace(/\.html$/, "").replace(/[\/\\]/g, "_") || "index"; + return `${slug}__${viewport}__${theme}`; +} + +export interface AxeScanResult { + cells: AxeCell[]; + /** axe-core version reported by the engine, once any cell has run. */ + axeVersion?: string; +} + +/** + * Run the page x viewport x theme matrix, writing each cell's raw payload to + * `cellsDir` as it completes. `onCell` is called after each cell so the caller + * can report progress. + */ +export async function runAxeScan( + client: CdpClient, + config: AxeScanConfig, + baseUrl: string, + pages: string[], + cellsDir: string, + onCell?: (cell: AxeCell) => void, +): Promise { + const axeSource = Deno.readTextFileSync(vendoredAxePath()); + + await client.send("Runtime.enable"); + await client.send("Page.enable"); + + const cells: AxeCell[] = []; + let axeVersion: string | undefined; + for (const page of pages) { + for (const viewport of config.viewports) { + for (const theme of config.themes) { + const url = `${baseUrl}/${page}`; + const cell = await scanCell( + client, + axeSource, + config, + page, + viewport, + theme, + url, + ); + axeVersion = axeVersion ?? cell.result?.testEngine?.version; + Deno.writeTextFileSync( + join(cellsDir, `${cellName(page, viewport.label, theme)}.json`), + JSON.stringify(cell, null, 2), + ); + cells.push(cell); + onCell?.(cell); + } + } + } + return { cells, axeVersion }; +} diff --git a/src/core/http-server.ts b/src/core/http-server.ts index 3c99ac51018..187c22edf01 100644 --- a/src/core/http-server.ts +++ b/src/core/http-server.ts @@ -9,6 +9,9 @@ export function handleHttpRequests( port?: number; hostname?: string; handler: (req: Request) => Promise; + // Deno.serve logs "Listening on ..." unless a callback is supplied; pass a + // no-op to serve quietly. + onListen?: (params: { hostname: string; port: number }) => void; }, ) { const abortController = new AbortController(); From 2c13f2c8b730e6ea95b48dd3e7468fc7e1f86172 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 10:10:33 -0700 Subject: [PATCH 03/43] Scan the whole document: drop the axe context excludes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither exclusion did any work here. Tabster's `[data-tabster-dummy]` sentinels arrive with the preview client's Fluent UI bundle, and `.quarto-axe-report` is the in-page panel axe-check.js builds when a site is rendered with `axe:` metadata — so neither appears in the plain rendered sites this command scans. Both were carried over from axe-check.js's own axe.run() call, along with a "won't fix upstream" claim that tabster#288 does not support: that issue is still open with no maintainer response. If someone does scan an `axe:`-rendered site, the overlay's own violations are information rather than noise. Results on the test site are unchanged: 12 cells, 12 ok, same rule ids. --- src/command/dev-call/axe/scan.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/command/dev-call/axe/scan.ts b/src/command/dev-call/axe/scan.ts index cbaa7ef3591..cbccb3d9d6f 100644 --- a/src/command/dev-call/axe/scan.ts +++ b/src/command/dev-call/axe/scan.ts @@ -374,12 +374,13 @@ export function vendoredAxePath(): string { return formatResourcePath("html", join("axe", "axe.min.js")); } -// axe context: keep our own dev chrome and the tabster shim out of results. -// (https://github.com/microsoft/tabster/issues/288 — won't fix upstream.) +// The whole document is in scope: what the scan sees is what the site ships. +// `document` is passed explicitly because axe.run() overloads its first +// argument between context and options. const kAxeRunExpression = ` (function () { return axe.run( - { exclude: ["[data-tabster-dummy]", ".quarto-axe-report"] }, + document, // v1 reports violations only, so axe can skip collecting full pass detail. { resultTypes: ["violations"] } ).then(function (result) { From 72ab8b0551832f41f6e874c26ee7b1ed11ae113f Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 10:44:41 -0700 Subject: [PATCH 04/43] M2: aggregate, report and the two contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the harness's aggregate and report stages, with the semantics the notes had already settled but the harness code had not caught up with. Signatures use the investigated "option 1" normalization: attribute values are kept with digit runs wildcarded, rather than stripped. Stripping turned `div[data-bs-target=".callout-4-contents"]` into the generic `div[data-bs-target]`, which every Bootstrap collapse, modal, tab and dropdown also matches — so one accepted callout defect silently suppressed unrelated conformance failures site-wide. The baseline is now the hand-written projection ledger: `signature` x `pages` x `impact`, empty `pages` meaning site-wide, a listed `pages` fail-closed at finding level, and escalation past the accepted impact re-alerting. `--update-baseline` is not ported; stale entries are reported and pruned by hand. Conformance labels come from `axe-check.js`'s own `axeConformanceLevel`, `impactRank` and `standardRank` rather than a second implementation, so a finding reads the same whether it came from a scan or the in-page report. That module guards its self-init on `typeof document`, which is what makes it importable here — the same property tests/unit/axe-*.test.ts rely on. `schemas.ts` holds Zod for both contracts. Baseline validation needed more than a schema: to Zod, a misspelled `impcat:` is an ignored unknown key plus a missing `impact`, so the report blamed `impact` while the culprit sat two characters away — and a `superRefine` never runs once a required field is missing. `parseBaseline` runs both checks and merges the issues, so the error names the typo and suggests the field it meant. `_axe-checks/` and `_axe-baseline.json` now anchor at the project root (nearest `_quarto.yml` at or above the site dir), falling back to the working directory for loose HTML. They sit beside the output dir, never inside it: a full render of a website or book deletes the output dir. Verified end-to-end on a rendered site: findings.json satisfies its own schema; all five baseline behaviours (site-wide accept, page-scoped accept, page-scoped re-alert on an unlisted page, impact-escalation re-alert, stale entry) land as expected; two hand-typo'd fields produce named errors with suggestions and exit 2. The report was driven in headless Chrome — row drill-down, column re-sort with row/detail pairing intact, the why-accepted column, the stale notice, and the copy-AI-briefing payloads. --- src/command/dev-call/axe/aggregate.ts | 508 ++++++++++++++++++++++++++ src/command/dev-call/axe/cmd.ts | 151 +++++++- src/command/dev-call/axe/report.ts | 300 +++++++++++++++ src/command/dev-call/axe/scan.ts | 11 + src/command/dev-call/axe/schemas.ts | 297 +++++++++++++++ 5 files changed, 1262 insertions(+), 5 deletions(-) create mode 100644 src/command/dev-call/axe/aggregate.ts create mode 100644 src/command/dev-call/axe/report.ts create mode 100644 src/command/dev-call/axe/schemas.ts diff --git a/src/command/dev-call/axe/aggregate.ts b/src/command/dev-call/axe/aggregate.ts new file mode 100644 index 00000000000..aa51a611aaf --- /dev/null +++ b/src/command/dev-call/axe/aggregate.ts @@ -0,0 +1,508 @@ +/* + * aggregate.ts + * + * The aggregate stage: per-cell axe payloads in, `findings.json` out. Every + * grouping and labelling decision lives here, so you can re-group without + * re-scanning and every consumer (report, agent, baseline) reads one contract. + * + * Grouping is by ROOT-CAUSE signature rather than exact DOM target, so one + * defect repeated by shared or generated code collapses into a single finding + * with a multiplicity count. + * + * Ported from the quarto-web harness (`_tools/axe/aggregate.mjs`) with the + * signature and baseline semantics settled in + * `aggregate-signature-breadth-investigation.md`: attribute values are kept + * with digit runs wildcarded (not stripped), and a baseline entry carries a + * `pages` scope. `--update-baseline` is gone: every entry exists because + * someone wrote it. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { warning } from "../../../deno_ral/log.ts"; +import { quartoConfig } from "../../../core/quarto.ts"; +import { md5HashSync } from "../../../core/hash.ts"; +import { AxeCell, AxeViolationNode } from "./scan.ts"; +import { AxeScanConfig } from "./config.ts"; +import { + AxeBaseline, + AxeFinding, + AxeFindings, + AxeImpact, + AxeOccurrence, + AxeStaleEntry, + kFindingsVersion, +} from "./schemas.ts"; + +// `axe-check.js` is the browser module behind the render-time `axe:` option. Its +// self-init is guarded on `typeof document`, so importing it here is +// side-effect-free — the same thing tests/unit/axe-*.test.ts already rely on. +// Reusing its labellers is deliberate: a finding's conformance label should read +// identically whether it came from a scan or from the in-page report. +import { + axeConformanceLevel as axeConformanceLevelJs, + impactRank as impactRankJs, + standardRank as standardRankJs, +} from "../../../resources/formats/html/axe/axe-check.js"; + +const axeConformanceLevel = axeConformanceLevelJs as (tags: string[]) => string; +/** Ascending: critical 0, serious 1, moderate 2, minor 3, unknown 4. */ +const impactRank = impactRankJs as (impact?: string | null) => number; +/** Ascending: WCAG A 0, AA 1, AAA 2, best-practice 3, obsolete 4, none 5. */ +const standardRank = standardRankJs as (tags: string[]) => number; + +/** Instances-or-pages threshold above which a finding is "systemic". */ +const kSystemicMinInstances = 3; + +// --------------------------------------------------------------------------- +// Signatures +// --------------------------------------------------------------------------- + +// Navigational and index attributes make otherwise-identical template output +// look distinct, so they're dropped entirely rather than value-normalized. +const kVolatileAttr = + /\[(?:href|data-original-href|data-index|id|name|style)(?:[~^$*|]?=(?:"[^"]*"|'[^']*'|[^\]]*))?\]/g; + +/** + * Normalize an axe target so repeated instances collapse but structure — and + * component identity — survives. + * + * Attribute *values* are kept with digit runs wildcarded. Stripping them (as + * the harness did) reduced `div[data-bs-target=".callout-4-contents"]` to the + * generic `div[data-bs-target]`, which every Bootstrap collapse, modal, tab and + * dropdown also matches — so one accepted callout defect silently suppressed + * unrelated conformance failures site-wide. Keeping the wildcarded value still + * collapses `.callout-4-contents`/`.callout-6-contents` into one signature. + */ +export function normalizeSelector(target: string[] | string): string { + const raw = Array.isArray(target) ? target.join(" > ") : String(target); + return raw + // positional: axe's nth-child varies with unrelated sibling edits + .replace(/:nth-(child|of-type|last-child)\(\d+\)/g, "") + .replace(kVolatileAttr, "") + .replace( + /\[([-\w]+)([~^$*|]?=)(?:"([^"]*)"|'([^']*)'|([^\]]*))\]/g, + (_match, name, op, dq, sq, bare) => { + const value = (dq ?? sq ?? bare ?? "").replace(/\d+/g, "*"); + return `[${name}${op}"${value}"]`; + }, + ) + // instance ids: #cb12-1 -> #cb, #fn3 -> #fn + .replace(/#([A-Za-z][\w-]*?)-?\d+(-\d+)*(?=[\s>~+.:#\[]|$)/g, "#$1") + .replace(/\s+/g, " ") + .trim(); +} + +interface AxeColorContrastData { + fgColor?: string; + bgColor?: string; + contrastRatio?: number; + expectedContrastRatio?: string; +} + +function colorContrastData( + node: AxeViolationNode, +): AxeColorContrastData | undefined { + const check = (node.any ?? []).find((entry) => entry.id === "color-contrast"); + return check?.data as AxeColorContrastData | undefined; +} + +/** + * Hybrid signature. For `color-contrast` the root cause is the colour pair, not + * where it appeared — every token sharing a colour pair is one defect in the + * theme. Everything else keys on the normalized selector. + */ +export function signatureOf(rule: string, node: AxeViolationNode): string { + if (rule === "color-contrast") { + const data = colorContrastData(node); + if (data?.fgColor && data?.bgColor) { + return `color-contrast :: ${data.fgColor} on ${data.bgColor}`; + } + } + return `${rule} :: ${normalizeSelector(node.target)}`; +} + +/** The contrast numbers, or the first real line of axe's failure summary. */ +function nodeDetail(rule: string, node: AxeViolationNode): string { + if (rule === "color-contrast") { + const data = colorContrastData(node); + if (data?.fgColor) { + return `${data.fgColor} on ${data.bgColor} = ${data.contrastRatio} ` + + `(needs ${data.expectedContrastRatio})`; + } + } + return (node.failureSummary ?? "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + // axe prefixes the summary with "Fix any of the following:" + .find((line) => !/^fix (any|all)/i.test(line)) ?? ""; +} + +// --------------------------------------------------------------------------- +// Grouping +// --------------------------------------------------------------------------- + +const kImpacts: AxeImpact[] = ["critical", "serious", "moderate", "minor"]; + +function asImpact(impact?: string | null): AxeImpact { + return kImpacts.includes(impact as AxeImpact) + ? (impact as AxeImpact) + : "minor"; +} + +/** The more severe of two impacts (impactRank is ascending). */ +function worstImpact(a: AxeImpact, b: AxeImpact): AxeImpact { + return impactRank(b) < impactRank(a) ? b : a; +} + +/** + * Stable, human-ish handle so an agent can be pointed at one finding. Derived + * from the signature, so it is recomputable and changes when the signature does. + */ +export function findingId(rule: string, signature: string): string { + return `${rule}-${md5HashSync(signature).slice(0, 6)}`; +} + +interface Occurrence { + page: string; + target: string; + html: string; + detail: string; + cells: Set; +} + +interface Group { + signature: string; + rule: string; + impact: AxeImpact; + tags: string[]; + help: string; + helpUrl: string; + detail: string | null; + /** distinct DOM elements: page + raw target */ + instances: Set; + pages: Set; + cells: Set; + viewports: Set; + themes: Set; + examples: Set; + occurrences: Map; +} + +function groupCells(cells: AxeCell[]): Map { + const groups = new Map(); + for (const cell of cells) { + for (const violation of cell.result!.violations) { + for (const node of violation.nodes) { + const signature = signatureOf(violation.id, node); + let group = groups.get(signature); + if (!group) { + group = { + signature, + rule: violation.id, + impact: asImpact(violation.impact), + tags: violation.tags, + help: violation.help, + helpUrl: violation.helpUrl, + detail: null, + instances: new Set(), + pages: new Set(), + cells: new Set(), + viewports: new Set(), + themes: new Set(), + examples: new Set(), + occurrences: new Map(), + }; + groups.set(signature, group); + } + group.impact = worstImpact(group.impact, asImpact(violation.impact)); + + const target = node.target.join(" > "); + const key = `${cell.page}##${target}`; + group.instances.add(key); + group.pages.add(cell.page); + group.cells.add(`${cell.page}|${cell.viewport}|${cell.theme}`); + group.viewports.add(cell.viewport); + group.themes.add(cell.theme); + if (group.examples.size < 4) { + group.examples.add(target); + } + + let occurrence = group.occurrences.get(key); + if (!occurrence) { + occurrence = { + page: cell.page, + target, + html: (node.html ?? "").trim().slice(0, 400), + detail: nodeDetail(violation.id, node), + cells: new Set(), + }; + group.occurrences.set(key, occurrence); + } + occurrence.cells.add(`${cell.viewport}·${cell.theme}`); + + if (!group.detail) { + const detail = nodeDetail(violation.id, node); + if (detail) { + group.detail = detail; + } + } + } + } + } + return groups; +} + +// --------------------------------------------------------------------------- +// Baseline reconciliation +// --------------------------------------------------------------------------- + +/** + * What the ledger accepts for one signature, merged across entries. + * + * `pages` scopes the acceptance and empty means site-wide. A page-scoped + * acceptance is fail-closed at finding level: the finding is known only while + * *every* page it occurs on is listed, so the same signature on one unlisted + * page re-alerts the whole finding, known occurrences included. + */ +interface Acceptance { + siteWide: boolean; + pages: Set; + /** Impact at acceptance: escalation past this re-alerts. */ + impact: AxeImpact; + notes: string[]; + entryIds: (string | undefined)[]; + rules: (string | undefined)[]; +} + +export function acceptances(baseline: AxeBaseline): Map { + const accepted = new Map(); + for (const entry of baseline.findings) { + const existing = accepted.get(entry.signature); + if (existing) { + // One entry per signature is the intended shape. Merging is defined + // (union the scope, take the most severe accepted impact) but ambiguous + // enough to be worth saying out loud rather than resolving silently. + warning( + `axe baseline: duplicate entries for signature '${entry.signature}' — ` + + `merging their pages and taking the most severe accepted impact. ` + + `Consider combining them into one entry.`, + ); + existing.siteWide = existing.siteWide || entry.pages.length === 0; + for (const page of entry.pages) { + existing.pages.add(page); + } + existing.impact = worstImpact(existing.impact, entry.impact); + existing.notes.push(entry.note); + existing.entryIds.push(entry.id); + existing.rules.push(entry.rule); + } else { + accepted.set(entry.signature, { + siteWide: entry.pages.length === 0, + pages: new Set(entry.pages), + impact: entry.impact, + notes: [entry.note], + entryIds: [entry.id], + rules: [entry.rule], + }); + } + } + return accepted; +} + +interface Reconciled { + baselined: boolean; + baselineNote: string | null; +} + +export function reconcile( + finding: { signature: string; impact: AxeImpact; pages: string[] }, + accepted: Map, +): Reconciled { + const acceptance = accepted.get(finding.signature); + if (!acceptance) { + return { baselined: false, baselineNote: null }; + } + const note = acceptance.notes.filter(Boolean).join(" · ") || null; + + // Escalation past the accepted impact re-alerts rather than hiding behind an + // old acceptance. + if (impactRank(finding.impact) < impactRank(acceptance.impact)) { + return { baselined: false, baselineNote: note }; + } + if (acceptance.siteWide) { + return { baselined: true, baselineNote: note }; + } + const covered = finding.pages.every((page) => acceptance.pages.has(page)); + return { baselined: covered, baselineNote: note }; +} + +/** + * Baseline entries not seen in this scan: site-wide entries whose signature + * didn't occur at all, and page-scoped entries whose signature didn't occur on + * any listed page. Reported, never auto-pruned — "resolved" is only confirmable + * on a full-site scan, since on a subset scan an entry may live on an unscanned + * page. + */ +export function staleEntries( + accepted: Map, + findings: { signature: string; pages: string[] }[], +): AxeStaleEntry[] { + const seen = new Map>(); + for (const finding of findings) { + let pages = seen.get(finding.signature); + if (!pages) { + pages = new Set(); + seen.set(finding.signature, pages); + } + for (const page of finding.pages) { + pages.add(page); + } + } + + const stale: AxeStaleEntry[] = []; + for (const [signature, acceptance] of accepted) { + const pages = seen.get(signature); + const wasSeen = acceptance.siteWide + ? pages !== undefined + : pages !== undefined && + [...acceptance.pages].some((page) => pages.has(page)); + if (!wasSeen) { + stale.push({ + signature, + id: acceptance.entryIds.find(Boolean) ?? null, + rule: acceptance.rules.find(Boolean) ?? null, + pages: [...acceptance.pages].sort(), + note: acceptance.notes.filter(Boolean).join(" · "), + }); + } + } + return stale.sort((a, b) => a.signature.localeCompare(b.signature)); +} + +// --------------------------------------------------------------------------- +// findings.json +// --------------------------------------------------------------------------- + +export interface AggregateOptions { + cells: AxeCell[]; + config: AxeScanConfig; + baseline: AxeBaseline; + baselineFile: string; + pages: string[]; + axeVersion?: string; +} + +export function aggregate(options: AggregateOptions): AxeFindings { + const { cells, config, baseline, baselineFile, pages } = options; + const okCells = cells.filter((cell) => cell.status === "ok" && cell.result); + const notOkCells = cells.filter((cell) => cell.status !== "ok"); + + const groups = groupCells(okCells); + const accepted = acceptances(baseline); + + const findings: AxeFinding[] = []; + for (const group of groups.values()) { + const conformance = axeConformanceLevel(group.tags); + // the version+level alone, without the success-criteria parenthetical + const standard = conformance.replace(/\s*\([^)]*\)\s*$/, "") || "—"; + const findingPages = [...group.pages].sort(); + const instances = group.instances.size; + const { baselined, baselineNote } = reconcile( + { signature: group.signature, impact: group.impact, pages: findingPages }, + accepted, + ); + + findings.push({ + id: findingId(group.rule, group.signature), + signature: group.signature, + rule: group.rule, + impact: group.impact, + conformance, + standard, + standardRank: standardRank(group.tags), + severityRank: impactRank(group.impact), + bestPractice: group.tags.includes("best-practice"), + help: group.help, + helpUrl: group.helpUrl, + detail: group.detail, + baselined, + baselineNote, + // systemic = repeated source: many elements, or more than one page + label: instances >= kSystemicMinInstances || group.pages.size >= 2 + ? "systemic" + : "localized", + instances, + pages: findingPages, + cells: group.cells.size, + viewports: [...group.viewports].sort(), + themes: [...group.themes].sort(), + examples: [...group.examples], + occurrences: [...group.occurrences.values()] + .map((occurrence): AxeOccurrence => ({ + page: occurrence.page, + target: occurrence.target, + html: occurrence.html, + detail: occurrence.detail, + cells: [...occurrence.cells].sort(), + })) + .sort((a, b) => + a.page.localeCompare(b.page) || a.target.localeCompare(b.target) + ), + }); + } + + // Default order: by standard (WCAG level first, best-practice and obsolete + // last), then severity, then reach. The HTML report can re-sort any column. + findings.sort((a, b) => + a.standardRank - b.standardRank || + a.severityRank - b.severityRank || + b.instances - a.instances || + b.pages.length - a.pages.length + ); + + const newCount = findings.filter((finding) => !finding.baselined).length; + + return { + version: kFindingsVersion, + generated: new Date().toISOString(), + quartoVersion: quartoConfig.version(), + axeVersion: options.axeVersion ?? + okCells[0]?.result?.testEngine?.version ?? null, + config: { + siteDir: config.siteDir, + viewports: config.viewports.map((viewport) => viewport.label), + themes: [...config.themes], + pages: config.pages ?? null, + maxPages: config.maxPages ?? null, + timeout: config.timeout, + settle: config.settle, + }, + pages: pages.map((page) => ({ output: page })), + cells: { + total: cells.length, + ok: okCells.length, + notOk: notOkCells.length, + }, + notOkCells: notOkCells.map((cell) => ({ + page: cell.page, + viewport: cell.viewport, + theme: cell.theme, + status: cell.status, + message: cell.message ?? null, + })), + pagesScanned: new Set(okCells.map((cell) => cell.page)).size, + baseline: { + file: baselineFile, + entries: accepted.size, + stale: staleEntries(accepted, findings), + }, + counts: { + total: findings.length, + new: newCount, + baselined: findings.length - newCount, + }, + findings, + }; +} diff --git a/src/command/dev-call/axe/cmd.ts b/src/command/dev-call/axe/cmd.ts index ca17251fd63..3fbcd1f9569 100644 --- a/src/command/dev-call/axe/cmd.ts +++ b/src/command/dev-call/axe/cmd.ts @@ -12,11 +12,18 @@ import { Command } from "cliffy/command/mod.ts"; import { error, info } from "../../../deno_ral/log.ts"; import { ensureDirSync, existsSync, walkSync } from "../../../deno_ral/fs.ts"; -import { globToRegExp, join, relative } from "../../../deno_ral/path.ts"; +import { + dirname, + globToRegExp, + join, + relative, + resolve, +} from "../../../deno_ral/path.ts"; import { pathWithForwardSlashes } from "../../../core/path.ts"; import { findOpenPort } from "../../../core/port.ts"; import { httpFileRequestHandler } from "../../../core/http.ts"; import { handleHttpRequests } from "../../../core/http-server.ts"; +import { projectConfigFile } from "../../../project/project-shared.ts"; import { AxeScanConfig, axeScanConfig, @@ -28,6 +35,9 @@ import { kDefaultViewports, } from "./config.ts"; import { AxeCell, launchScanBrowser, runAxeScan } from "./scan.ts"; +import { aggregate } from "./aggregate.ts"; +import { renderReport } from "./report.ts"; +import { AxeBaseline, AxeFindings, parseBaseline } from "./schemas.ts"; /** Scan complete: every cell produced an axe payload. */ const kExitComplete = 0; @@ -80,6 +90,82 @@ function cellLine(cell: AxeCell): string { } ${ids}`; } +/** + * Where `_axe-checks/` and `_axe-baseline.json` live: the nearest project root + * at or above the site dir, else the working directory. + * + * The artifacts sit *beside* the output dir, never inside it — a full render of + * a website or book deletes the output dir, and anything that survived there + * would be published. The cheap `_quarto.yml` check agrees with a real + * `ProjectContext.dir` wherever a project config exists; reading the project's + * own `output-dir` is what would need `projectContext()`, and that is deferred + * (see the design note's cut list). + */ +export function resolveAnchor(siteDir: string): string { + let dir = resolve(siteDir); + for (;;) { + if (projectConfigFile(dir)) { + return dir; + } + const parent = dirname(dir); + if (parent === dir) { + return Deno.cwd(); + } + dir = parent; + } +} + +/** + * Read the hand-written ledger. Missing is fine — that's the first run. A + * present-but-invalid ledger is an error: a misspelled field would otherwise + * mean "no impact recorded" or "site-wide", silently. + */ +export function readBaseline(file: string): AxeBaseline { + if (!existsSync(file)) { + return { findings: [] }; + } + let parsed: unknown; + try { + parsed = JSON.parse(Deno.readTextFileSync(file)); + } catch (e) { + throw new Error( + `${file} is not valid JSON: ${e instanceof Error ? e.message : e}`, + ); + } + const result = parseBaseline(parsed); + if (!result.success) { + const issues = result.issues.map((issue) => + ` ${issue.path}: ${issue.message}` + ).join("\n"); + throw new Error(`${file} is not a valid baseline:\n${issues}`); + } + return result.baseline; +} + +function summaryTable(results: AxeFindings): string[] { + if (results.findings.length === 0) { + return [" (no violations found)"]; + } + const rows = results.findings.map((finding) => [ + finding.id, + finding.impact, + finding.standard, + String(finding.instances), + String(finding.pages.length), + finding.label, + finding.baselined ? "known" : "new", + ]); + const header = ["ID", "IMPACT", "STANDARD", "N", "PAGES", "SCOPE", "STATUS"]; + const widths = header.map((cell, column) => + Math.max(cell.length, ...rows.map((row) => row[column].length)) + ); + const line = (cells: string[]) => + " " + + cells.map((cell, column) => cell.padEnd(widths[column])).join(" ") + .trimEnd(); + return [line(header), ...rows.map(line)]; +} + /** * Run the scan stage against `config.siteDir` and return the process exit code. */ @@ -103,9 +189,20 @@ export async function axeScan(config: AxeScanConfig): Promise { return kExitIncomplete; } - const cellsDir = join(kAxeOutputDir, "cells"); + const anchor = resolveAnchor(config.siteDir); + const outputDir = join(anchor, kAxeOutputDir); + const cellsDir = join(outputDir, "cells"); + const baselineFile = join(anchor, kAxeBaselineFile); ensureDirSync(cellsDir); + let baseline: AxeBaseline; + try { + baseline = readBaseline(baselineFile); + } catch (e) { + error(e instanceof Error ? e.message : String(e)); + return kExitIncomplete; + } + // Serve the site dir first, so the bound port can't be handed to Chrome next. const sitePort = findOpenPort(); const server = handleHttpRequests({ @@ -161,7 +258,49 @@ export async function axeScan(config: AxeScanConfig): Promise { `axe-core ${scan.axeVersion} (quarto-cli's vendored build, injected at scan time)`, ); } - info(`cells: ${cellsDir}`); + + const results = aggregate({ + cells: scan.cells, + config, + baseline, + baselineFile, + pages, + axeVersion: scan.axeVersion, + }); + + const findingsFile = join(outputDir, "findings.json"); + const reportFile = join(outputDir, "report.html"); + Deno.writeTextFileSync(findingsFile, JSON.stringify(results, null, 2)); + Deno.writeTextFileSync(reportFile, renderReport(results)); + + info(""); + for (const row of summaryTable(results)) { + info(row); + } + info(""); + info( + ` ${results.counts.total} finding${ + results.counts.total === 1 ? "" : "s" + } (${results.counts.new} new, ${results.counts.baselined} known)`, + ); + info(` findings: ${findingsFile}`); + info(` report: ${reportFile}`); + info(` cells: ${cellsDir}`); + info( + ` baseline: ${baselineFile} (${results.baseline.entries} entr` + + `${results.baseline.entries === 1 ? "y" : "ies"}` + + `${existsSync(baselineFile) ? "" : ", not present"})`, + ); + if (results.baseline.stale.length) { + info( + ` ${results.baseline.stale.length} baseline entr${ + results.baseline.stale.length === 1 ? "y" : "ies" + } not seen this scan — prune by hand after a full-site scan: ${ + results.baseline.stale.map((entry) => entry.id ?? entry.signature) + .join(", ") + }`, + ); + } if (notOk.length) { for (const cell of notOk) { @@ -187,8 +326,10 @@ export const axeCommand = new Command() "Scan a rendered site for accessibility violations with axe-core.\n\n" + "Prototype: scans every page in across the viewport x theme " + "matrix, groups violations by root-cause signature, reconciles " + - `${kAxeBaselineFile} in the working directory, and writes findings.json ` + - `plus report.html to ${kAxeOutputDir}/.`, + `${kAxeBaselineFile}, and writes findings.json plus report.html to ` + + `${kAxeOutputDir}/. Both sit at the project root (the nearest ` + + `_quarto.yml at or above ), or the working directory if ` + + `there is no project.`, ) .option( "--pages ", diff --git a/src/command/dev-call/axe/report.ts b/src/command/dev-call/axe/report.ts new file mode 100644 index 00000000000..1b2e1671f02 --- /dev/null +++ b/src/command/dev-call/axe/report.ts @@ -0,0 +1,300 @@ +/* + * report.ts + * + * The report stage: `findings.json` in, a self-contained `report.html` out. + * + * A dumb view — no grouping logic here, that all lives in aggregate.ts. Findings + * are ordered by standard then severity, each row expands to its occurrences, + * any column re-sorts, and every finding carries a copy-to-clipboard AI briefing + * so a fix can be handed straight to an agent. + * + * Ported from the quarto-web harness (`_tools/axe/report.mjs`), HTML format + * only: v1 keeps the drill-down report, and the console summary lives in cmd.ts. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { AxeFinding, AxeFindings } from "./schemas.ts"; + +function esc(value: unknown): string { + return String(value ?? "") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +function plural(count: number, singular = "", many = "s"): string { + return count === 1 ? singular : many; +} + +// How many occurrences an AI briefing lists before it starts summarizing. +const kOccurrenceCap = 15; + +const kBriefingPreamble = + "This is from an axe-core accessibility audit of a website generated by " + + "**Quarto**. Fixes usually belong in the Quarto *source* — a `.qmd`, " + + "`_quarto.yml`, `brand.yml`, or a theme `.scss` — not in the generated " + + "site HTML (it is overwritten every render). Some findings come from " + + "Quarto's shared templates (navbar, footer, listings); those may warrant an " + + "upstream Quarto issue rather than a local change."; + +/** A markdown briefing for one finding, sized to paste into a chat. */ +export function briefing(finding: AxeFinding): string { + const shown = finding.occurrences.slice(0, kOccurrenceCap).map((occurrence) => + `- \`${occurrence.page}\` [${occurrence.cells.join(", ")}] — selector ` + + `\`${occurrence.target}\`\n \`\`\`html\n ${occurrence.html}\n \`\`\`` + ).join("\n"); + const hidden = finding.occurrences.length - kOccurrenceCap; + const more = hidden > 0 ? `\n- …and ${hidden} more` : ""; + const fixNote = finding.label === "systemic" + ? " Fixing the shared source fixes all of them." + : ""; + return [ + `# Accessibility issue: ${finding.rule} — ${finding.standard} ` + + `(${finding.impact})`, + ``, + kBriefingPreamble, + ``, + `- **Rule:** \`${finding.rule}\`${ + finding.help ? ` — ${finding.help}` : "" + }`, + `- **Standard:** ${finding.conformance || "—"}`, + `- **Severity:** ${finding.impact}`, + `- **Scope:** ${finding.label} — ${finding.pages.length} page${ + plural(finding.pages.length) + }, ${finding.instances} occurrence${plural(finding.instances)}.${fixNote}`, + `- **Problem:** ${finding.detail || finding.occurrences[0]?.detail || ""}`, + ...(finding.helpUrl ? [`- **Reference:** ${finding.helpUrl}`] : []), + `- **Finding id:** \`${finding.id}\` (look up the full record in ` + + `\`findings.json\`)`, + ``, + `## Affected elements (${finding.occurrences.length}${ + hidden > 0 ? `, showing ${kOccurrenceCap}` : "" + })`, + shown + more, + ``, + `Please help me fix this in the Quarto source.`, + ].join("\n"); +} + +const kStyles = ` + :root { color-scheme: light dark; --br:#d0d3d9; --mut:#5b6470; --bg:#fff; --fg:#1a1d21; --card:#f6f7f9; --hd:#eef1f4; } + @media (prefers-color-scheme: dark){ :root{ --br:#333a44; --mut:#9aa4b0; --bg:#15171a; --fg:#e6e8eb; --card:#1e2126; --hd:#22262c; } } + body { font: 15px/1.5 system-ui, sans-serif; margin: 0 auto; max-width: 1180px; padding: 1.5rem; color:var(--fg); background:var(--bg); } + h1 { font-size: 1.5rem; margin: 0 0 .25rem; } + .sub { color: var(--mut); margin:0 0 1rem; } .mut { color: var(--mut); } + .warn { border-left:4px solid #c0392b; padding-left:.6rem; } + table.findings { width:100%; border-collapse:collapse; } + .findings > thead th { position:sticky; top:0; background:var(--hd); text-align:left; padding:.45rem .6rem; + font-size:.85em; border-bottom:2px solid var(--br); cursor:pointer; user-select:none; white-space:nowrap; } + .findings > thead th[data-key]::after { content:" \\2195"; color:var(--mut); font-size:.85em; } + .findings > thead th[aria-sort=ascending]::after { content:" \\2191"; color:var(--fg); } + .findings > thead th[aria-sort=descending]::after { content:" \\2193"; color:var(--fg); } + tr.f > td { padding:.4rem .6rem; border-bottom:1px solid var(--br); vertical-align:top; cursor:pointer; } + tr.f:hover > td { background:var(--card); } + tr.f.open > td { background:var(--card); } + td.std::before { content:"\\25B8 "; color:var(--mut); } + tr.f.open td.std::before { content:"\\25BE "; } + .rule { font-family: ui-monospace, monospace; font-weight:600; } + .badge { font-size:.72em; padding:.05rem .4rem; border-radius:99px; border:1px solid var(--br); text-transform:uppercase; letter-spacing:.03em; } + .badge.systemic { background:#fde8e8; color:#8a1c1c; border-color:#f5c2c2; } + .badge.localized { background:#eef1f4; color:#3a4048; } + .impact-critical,.impact-serious { background:#c0392b; color:#fff; border-color:#c0392b; } + .impact-moderate { background:#e0892e; color:#fff; border-color:#e0892e; } + .impact-minor { background:#8a939e; color:#fff; border-color:#8a939e; } + td.num, th.num { text-align:right; font-variant-numeric:tabular-nums; } + tr.d > td { padding:0 .6rem .6rem 1.6rem; background:var(--card); } + table.occ { width:100%; border-collapse:collapse; font-size:.86em; } + .occ th,.occ td { text-align:left; padding:.3rem .5rem; border-top:1px solid var(--br); vertical-align:top; } + .occ th { color:var(--mut); font-weight:600; } + .occ code { font-family: ui-monospace, monospace; font-size:.92em; word-break:break-all; } + .occ .snip { color:var(--mut); } .cells { white-space:nowrap; color:var(--mut); } + .findings > thead th:not([data-key]) { cursor:default; } + td.act { white-space:nowrap; text-align:right; } + .cp { font:inherit; font-size:.8em; padding:.1rem .45rem; border:1px solid var(--br); border-radius:6px; background:var(--bg); color:var(--fg); cursor:pointer; } + .cp:hover { background:var(--hd); } + .idchip { display:block; margin-top:.25rem; font-family:ui-monospace,monospace; font-size:.72em; color:var(--mut); cursor:pointer; } + .idchip:hover { color:var(--fg); text-decoration:underline; } + details.baselined { margin-top:1.5rem; border:1px solid var(--br); border-radius:8px; background:var(--card); } + details.baselined > summary { cursor:pointer; padding:.6rem .8rem; color:var(--mut); font-size:.9em; user-select:none; } + details.baselined[open] > summary { border-bottom:1px solid var(--br); } + details.baselined table.findings { padding:.2rem; } + details.baselined tr.f > td { opacity:.85; } + td.why { color:var(--mut); font-size:.9em; } + p.stale { border-left:4px solid #e0892e; padding-left:.6rem; font-size:.9em; } + p.stale code { font-family:ui-monospace,monospace; } +`; + +// Ranks are ascending (0 = worst), so severity and standard both default to +// ascending; reach columns default to biggest-first. +const kTableScript = ` + var AI = JSON.parse(document.getElementById('axe-ai').textContent); + function copyText(text, el){ + navigator.clipboard.writeText(text).then(function(){ + if (!el) return; var was = el.textContent; el.textContent = 'copied \\u2713'; + setTimeout(function(){ el.textContent = was; }, 1200); + }); + } + var tbody = document.querySelector('#findings tbody'); + function pairs(){ var out=[], cur=null; Array.prototype.forEach.call(tbody.children, function(tr){ + if (tr.classList.contains('f')) { cur = {f:tr, d:null}; out.push(cur); } else if (cur) { cur.d = tr; } }); return out; } + function onRowClick(e){ + var cp = e.target.closest('[data-copy]'); if (cp) { copyText(AI.briefings[cp.dataset.copy], cp); return; } + var ci = e.target.closest('[data-copyid]'); if (ci) { copyText(ci.dataset.copyid, ci); return; } + if (e.target.closest('a')) return; + var row = e.target.closest('tr.f'); if (!row) return; + var d = row.nextElementSibling; + if (d && d.classList.contains('d')) { d.hidden = !d.hidden; row.classList.toggle('open'); } + } + Array.prototype.forEach.call(document.querySelectorAll('table.findings tbody'), function(tb){ tb.addEventListener('click', onRowClick); }); + var DEFAULT_DIR = { standard:1, class:1, rule:1, severity:1, pages:-1, instances:-1 }; + var sortKey = null, dir = 1; + Array.prototype.forEach.call(document.querySelectorAll('#findings th[data-key]'), function(th){ + th.addEventListener('click', function(){ + var key = th.dataset.key, type = th.dataset.type || 'str'; + dir = (sortKey === key) ? -dir : DEFAULT_DIR[key]; sortKey = key; + var ps = pairs(); + ps.sort(function(a,b){ + var va = a.f.dataset[key], vb = b.f.dataset[key]; + if (type === 'num') return (parseFloat(va) - parseFloat(vb)) * dir; + return va.localeCompare(vb) * dir; + }); + ps.forEach(function(p){ tbody.appendChild(p.f); if (p.d) tbody.appendChild(p.d); }); + Array.prototype.forEach.call(document.querySelectorAll('#findings th'), function(h){ h.removeAttribute('aria-sort'); }); + th.setAttribute('aria-sort', dir > 0 ? 'ascending' : 'descending'); + }); + }); +`; + +function occurrenceRows(finding: AxeFinding): string { + return finding.occurrences.map((occurrence) => + `${esc(occurrence.page)}` + + `${occurrence.cells.map(esc).join("
")}` + + `${esc(occurrence.target)}` + + `${esc(occurrence.html)}` + + `${esc(occurrence.detail)}` + ).join(""); +} + +function findingRows(findings: AxeFinding[], showWhy: boolean): string { + return findings.map((finding) => { + const why = showWhy + ? `\n ${esc(finding.baselineNote ?? "")}` + : ""; + const columns = showWhy ? 9 : 8; + return ` + ${ + esc(finding.standard) + } + ${ + esc(finding.impact) + } + ${esc(finding.rule)} + ${finding.label} + ${finding.pages.length} + ${finding.instances} + ${esc(finding.detail || finding.examples[0])}${why} + ${esc(finding.id)} + + + ${occurrenceRows(finding)}
pagecells (width·theme)selectorelementdetail
`; + }).join("\n"); +} + +function tableHead(showWhy: boolean): string { + return ` + standard + severity + rule + class + pages + occurrences + detail${showWhy ? "\n why accepted" : ""} + AI +`; +} + +/** Render `findings.json` as a self-contained HTML report. */ +export function renderReport(results: AxeFindings): string { + const newFindings = results.findings.filter((finding) => !finding.baselined); + const baselined = results.findings.filter((finding) => finding.baselined); + + // JSON in a + + +`; +} diff --git a/src/command/dev-call/axe/scan.ts b/src/command/dev-call/axe/scan.ts index cbccb3d9d6f..98d1dac15af 100644 --- a/src/command/dev-call/axe/scan.ts +++ b/src/command/dev-call/axe/scan.ts @@ -29,12 +29,23 @@ import { AxeScanConfig, AxeTheme, AxeViewport } from "./config.ts"; /** Status of a single scanned cell. Anything but "ok" fails closed. */ export type AxeCellStatus = "ok" | "timeout" | "error" | "no-payload"; +/** One of axe's check results on a node; `data` carries rule-specific detail. */ +export interface AxeCheckResult { + id: string; + data?: unknown; +} + /** A single axe violation node, as axe-core reports it. */ export interface AxeViolationNode { html: string; target: string[]; failureSummary?: string; impact?: string | null; + // The scan keeps axe's nodes whole, so the check arrays survive to the + // aggregate stage — color-contrast reads its colour pair out of `any`. + any?: AxeCheckResult[]; + all?: AxeCheckResult[]; + none?: AxeCheckResult[]; } /** A single axe violation, as axe-core reports it. */ diff --git a/src/command/dev-call/axe/schemas.ts b/src/command/dev-call/axe/schemas.ts new file mode 100644 index 00000000000..ed1d6d1a630 --- /dev/null +++ b/src/command/dev-call/axe/schemas.ts @@ -0,0 +1,297 @@ +/* + * schemas.ts + * + * The two contracts `quarto dev-call axe` publishes: `findings.json` (what the + * aggregate stage writes, and what the report, agents and the eventual public + * command read) and `_axe-baseline.json` (the hand-written ledger of accepted + * findings). + * + * These are Zod rather than plain interfaces for two reasons: the aggregate + * stage can validate its own output in tests, and a hand-edited baseline is + * validated on read — a typo'd field name should be an error, not a silently + * ignored unknown key. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { z } from "zod"; + +/** Bump in step with any breaking change to `findings.json`. */ +export const kFindingsVersion = 1; + +// --------------------------------------------------------------------------- +// findings.json +// --------------------------------------------------------------------------- + +export const axeImpactSchema = z.enum([ + "critical", + "serious", + "moderate", + "minor", +]); + +export const axeOccurrenceSchema = z.object({ + page: z.string(), + /** CSS selector of one real instance. */ + target: z.string(), + /** Excerpt of the offending element. */ + html: z.string(), + detail: z.string(), + /** The matrix cells this instance reproduced in, e.g. `1440x900·dark`. */ + cells: z.array(z.string()), +}); + +export const axeFindingSchema = z.object({ + /** Stable handle: rule + hash of the signature. */ + id: z.string(), + /** Page-independent root-cause key, and the baseline's join key. */ + signature: z.string(), + rule: z.string(), + /** Escalated to the worst impact seen across occurrences. */ + impact: axeImpactSchema, + /** Full label including success criteria, e.g. `WCAG 2.0 AA 1.4.3`. */ + conformance: z.string(), + /** Version + level alone, e.g. `WCAG 2.0 AA`, for grouping. */ + standard: z.string(), + standardRank: z.number(), + severityRank: z.number(), + bestPractice: z.boolean(), + help: z.string(), + helpUrl: z.string(), + detail: z.string().nullable(), + /** The new/known switch: act on `false`. */ + baselined: z.boolean(), + /** + * The matching baseline entry's note. Present whenever an entry matched the + * signature — including when the acceptance did *not* hold (impact + * escalation, or an occurrence on an unlisted page), because "this was + * accepted at minor, and it's now serious" is the useful bit. + */ + baselineNote: z.string().nullable(), + label: z.enum(["systemic", "localized"]), + /** Distinct DOM elements affected. */ + instances: z.number(), + /** Every page it occurred on. */ + pages: z.array(z.string()), + /** Distinct matrix cells it occurred in. */ + cells: z.number(), + viewports: z.array(z.string()), + themes: z.array(z.string()), + examples: z.array(z.string()), + occurrences: z.array(axeOccurrenceSchema), +}); + +export const axeStaleEntrySchema = z.object({ + signature: z.string(), + id: z.string().nullable(), + rule: z.string().nullable(), + pages: z.array(z.string()), + note: z.string(), +}); + +export const axeNotOkCellSchema = z.object({ + page: z.string(), + viewport: z.string(), + theme: z.string(), + status: z.string(), + message: z.string().nullable(), +}); + +export const axeFindingsSchema = z.object({ + version: z.literal(kFindingsVersion), + generated: z.string(), + quartoVersion: z.string(), + /** Provenance: results move with axe upgrades. */ + axeVersion: z.string().nullable(), + /** Echo of what was scanned, for reproducibility. */ + config: z.object({ + siteDir: z.string(), + viewports: z.array(z.string()), + themes: z.array(z.string()), + pages: z.array(z.string()).nullable(), + maxPages: z.number().nullable(), + timeout: z.number(), + settle: z.number(), + }), + /** + * The pages scanned, as output paths. v1 has no `input`/`title`: source + * mapping needs Quarto's project index and is deferred to the CLI version. + */ + pages: z.array(z.object({ output: z.string() })), + cells: z.object({ + total: z.number(), + ok: z.number(), + notOk: z.number(), + }), + /** Fail-closed: failures are data, never passes. */ + notOkCells: z.array(axeNotOkCellSchema), + pagesScanned: z.number(), + baseline: z.object({ + file: z.string(), + entries: z.number(), + stale: z.array(axeStaleEntrySchema), + }), + counts: z.object({ + total: z.number(), + new: z.number(), + baselined: z.number(), + }), + findings: z.array(axeFindingSchema), +}); + +export type AxeFindings = z.infer; +export type AxeFinding = z.infer; +export type AxeOccurrence = z.infer; +export type AxeImpact = z.infer; +export type AxeStaleEntry = z.infer; +export type AxeNotOkCell = z.infer; + +// --------------------------------------------------------------------------- +// _axe-baseline.json +// --------------------------------------------------------------------------- + +/** + * A baseline entry is a projection of a finding: same field names, same + * meanings, nothing invented, plus a required `note` saying why it's accepted. + * + * `pages` is required even when empty, because the two cases mean very + * different things and an omission shouldn't silently pick the broader one: + * `[]` accepts the signature site-wide (right for chrome), while a listed + * `pages` accepts only those pages and re-alerts if the signature turns up + * anywhere else. + */ +export const axeBaselineEntrySchema = z.object({ + signature: z.string(), + pages: z.array(z.string()), + /** The impact *at acceptance*: escalation past this re-alerts. */ + impact: axeImpactSchema, + note: z.string(), + // reviewer context, not read by the scanner + id: z.string().optional(), + rule: z.string().optional(), + conformance: z.string().optional(), +}); + +export type AxeBaselineEntry = z.infer; + +/** + * Fields the tolerant reader ignores rather than rejects: everything a finding + * carries, so pasting a whole finding out of `findings.json` and adding a + * `note` works. Anything outside this set is treated as a misspelling. + */ +const kToleratedEntryFields = new Set([ + ...Object.keys(axeFindingSchema.shape), + ...Object.keys(axeBaselineEntrySchema.shape), +]); + +/** Levenshtein distance, for "did you mean" on a misspelled field. */ +function editDistance(a: string, b: string): number { + const rows = Array.from( + { length: a.length + 1 }, + (_, i) => [i, ...Array(b.length).fill(0)], + ); + for (let j = 0; j <= b.length; j++) { + rows[0][j] = j; + } + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + rows[i][j] = Math.min( + rows[i - 1][j] + 1, + rows[i][j - 1] + 1, + rows[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + } + } + return rows[a.length][b.length]; +} + +function didYouMean(key: string): string | undefined { + let best: string | undefined; + let bestDistance = Infinity; + for (const candidate of kToleratedEntryFields) { + const distance = editDistance(key.toLowerCase(), candidate.toLowerCase()); + if (distance < bestDistance) { + bestDistance = distance; + best = candidate; + } + } + // only suggest when it really looks like a typo rather than a new field + return bestDistance <= Math.max(1, Math.floor(key.length / 3)) + ? best + : undefined; +} + +/** + * The baseline as read from disk. Unknown-but-known-shaped keys pass through, + * so pasting a whole finding out of `findings.json` works. + */ +export const axeBaselineSchema = z.object({ + findings: z.array(axeBaselineEntrySchema.passthrough()), +}).passthrough(); + +export type AxeBaseline = z.infer; + +/** A single problem with a hand-edited baseline, ready to print. */ +export interface AxeBaselineIssue { + path: string; + message: string; +} + +export type ParsedBaseline = + | { success: true; baseline: AxeBaseline } + | { success: false; issues: AxeBaselineIssue[] }; + +/** + * Validate a hand-edited baseline, reporting misspelled field names alongside + * the shape errors they cause. + * + * Zod alone isn't enough here. A misspelled `impcat:` is, to a tolerant object + * schema, an unknown key (ignored) plus a missing `impact` (an error) — so the + * report says "impact: Required" while the culprit sits in plain sight two + * characters away. Zod also short-circuits: a `superRefine` on the entry never + * runs once a required field is missing, so the two checks can't be composed. + * Running them separately and merging the issues gives the reader both halves. + */ +export function parseBaseline(data: unknown): ParsedBaseline { + const issues: AxeBaselineIssue[] = []; + + const outer = z.object({ findings: z.array(z.record(z.unknown())) }) + .passthrough().safeParse(data); + if (!outer.success) { + return { + success: false, + issues: outer.error.issues.map((issue) => ({ + path: issue.path.join(".") || "(root)", + message: issue.message, + })), + }; + } + + outer.data.findings.forEach((entry, index) => { + for (const key of Object.keys(entry)) { + if (kToleratedEntryFields.has(key)) { + continue; + } + const suggestion = didYouMean(key); + issues.push({ + path: `findings[${index}].${key}`, + message: `unknown field '${key}'` + + (suggestion ? ` — did you mean '${suggestion}'?` : ""), + }); + } + const parsed = axeBaselineEntrySchema.passthrough().safeParse(entry); + if (!parsed.success) { + for (const issue of parsed.error.issues) { + issues.push({ + path: `findings[${index}].${issue.path.join(".")}`, + message: issue.message, + }); + } + } + }); + + if (issues.length) { + return { success: false, issues }; + } + return { success: true, baseline: axeBaselineSchema.parse(data) }; +} From cb5f740c47346343d17bbc38483b0d953ddd270c Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 11:11:25 -0700 Subject: [PATCH 05/43] Version the signature scheme, and pin the normalization with tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The signature is the most consequential value the scanner computes: too broad and one accepted defect suppresses unrelated conformance failures site-wide, too narrow and a baseline entry stops matching when a counter changes. Two gaps around it. First, nothing recorded which normalizer produced a signature. `version` covers findings.json's field shape, so a normalizer change would leave every field intact while re-keying every signature — reported as "N new findings, M baseline entries not seen", which is indistinguishable from "you fixed everything and broke an equal amount". The ledger's notes, the record of why each finding was accepted, would quietly stop applying. `signatureScheme` makes that a named error instead, and points at occurrences[].target as the unchanged raw selector to re-annotate from. It is optional in the baseline, so the first hand-written ledger needs no ceremony. Second, the normalization had no tests. tests/unit/axe-signature.test.ts covers it as explicit should-collapse / must-stay-distinct pairs over selectors axe really emits on Quarto output, rather than as assertions about the regexes. The must-stay-distinct half is the important one: it pins that a callout, a modal, a carousel and a tabset link do not share a signature just because they all hang off data-bs-target, which is exactly what the pre-option-1 normalizer got wrong. A third group asserts exact output so a scheme change shows up as a diff rather than as a changed collapse count. 30 tests, all passing. Also covers a case with no fixture yet: a color-contrast payload missing its check data must fall back to the selector rather than key on "undefined on undefined" and collapse every contrast finding in the site into one. --- src/command/dev-call/axe/aggregate.ts | 2 + src/command/dev-call/axe/schemas.ts | 50 ++++- tests/unit/axe-signature.test.ts | 252 ++++++++++++++++++++++++++ 3 files changed, 301 insertions(+), 3 deletions(-) create mode 100644 tests/unit/axe-signature.test.ts diff --git a/src/command/dev-call/axe/aggregate.ts b/src/command/dev-call/axe/aggregate.ts index aa51a611aaf..1fe99590f23 100644 --- a/src/command/dev-call/axe/aggregate.ts +++ b/src/command/dev-call/axe/aggregate.ts @@ -32,6 +32,7 @@ import { AxeOccurrence, AxeStaleEntry, kFindingsVersion, + kSignatureScheme, } from "./schemas.ts"; // `axe-check.js` is the browser module behind the render-time `axe:` option. Its @@ -466,6 +467,7 @@ export function aggregate(options: AggregateOptions): AxeFindings { return { version: kFindingsVersion, + signatureScheme: kSignatureScheme, generated: new Date().toISOString(), quartoVersion: quartoConfig.version(), axeVersion: options.axeVersion ?? diff --git a/src/command/dev-call/axe/schemas.ts b/src/command/dev-call/axe/schemas.ts index ed1d6d1a630..3cdf0dca935 100644 --- a/src/command/dev-call/axe/schemas.ts +++ b/src/command/dev-call/axe/schemas.ts @@ -16,9 +16,25 @@ import { z } from "zod"; -/** Bump in step with any breaking change to `findings.json`. */ +/** Bump in step with any breaking change to `findings.json`'s field shape. */ export const kFindingsVersion = 1; +/** + * Which signature-normalization scheme produced the signatures in this file. + * + * Separate from `kFindingsVersion` on purpose: a normalizer change leaves every + * field name intact but re-keys every signature, so a baseline written under an + * older scheme matches nothing. Without this, that is indistinguishable from + * "you fixed everything and introduced an equal number of new problems", and the + * ledger's notes — the record of *why* each finding was accepted — quietly stop + * applying. Bump it whenever `normalizeSelector` or `signatureOf` changes in a + * way that alters existing signatures. + * + * 1 — nth-child stripped; volatile attributes dropped; other attribute values + * kept with digit runs wildcarded; trailing instance ids collapsed. + */ +export const kSignatureScheme = 1; + // --------------------------------------------------------------------------- // findings.json // --------------------------------------------------------------------------- @@ -99,6 +115,7 @@ export const axeNotOkCellSchema = z.object({ export const axeFindingsSchema = z.object({ version: z.literal(kFindingsVersion), + signatureScheme: z.literal(kSignatureScheme), generated: z.string(), quartoVersion: z.string(), /** Provenance: results move with axe upgrades. */ @@ -226,6 +243,13 @@ function didYouMean(key: string): string | undefined { * so pasting a whole finding out of `findings.json` works. */ export const axeBaselineSchema = z.object({ + /** + * The signature scheme the entries were written against. Optional: a ledger + * without it is assumed to match the current scheme, which is right for the + * first one anybody writes. Once present, a mismatch is an error rather than + * a silent mass-invalidation. + */ + signatureScheme: z.number().optional(), findings: z.array(axeBaselineEntrySchema.passthrough()), }).passthrough(); @@ -255,8 +279,10 @@ export type ParsedBaseline = export function parseBaseline(data: unknown): ParsedBaseline { const issues: AxeBaselineIssue[] = []; - const outer = z.object({ findings: z.array(z.record(z.unknown())) }) - .passthrough().safeParse(data); + const outer = z.object({ + signatureScheme: z.number().optional(), + findings: z.array(z.record(z.unknown())), + }).passthrough().safeParse(data); if (!outer.success) { return { success: false, @@ -267,6 +293,24 @@ export function parseBaseline(data: unknown): ParsedBaseline { }; } + if ( + outer.data.signatureScheme !== undefined && + outer.data.signatureScheme !== kSignatureScheme + ) { + return { + success: false, + issues: [{ + path: "signatureScheme", + message: + `this baseline was written against signature scheme ${outer.data.signatureScheme}, ` + + `but this build emits scheme ${kSignatureScheme}. Every entry would read as ` + + `stale. Re-annotate the entries against the new signatures — the raw ` + + `selectors are unchanged in each finding's occurrences[].target — then ` + + `set "signatureScheme": ${kSignatureScheme}.`, + }], + }; + } + outer.data.findings.forEach((entry, index) => { for (const key of Object.keys(entry)) { if (kToleratedEntryFields.has(key)) { diff --git a/tests/unit/axe-signature.test.ts b/tests/unit/axe-signature.test.ts new file mode 100644 index 00000000000..99085ef9a9f --- /dev/null +++ b/tests/unit/axe-signature.test.ts @@ -0,0 +1,252 @@ +/* + * axe-signature.test.ts + * + * Tests the root-cause signature that `quarto dev-call axe` groups and + * baselines on: `normalizeSelector` and `signatureOf` in + * src/command/dev-call/axe/aggregate.ts. + * + * These are the most consequential lines in the scanner. A signature that is + * too broad makes one accepted defect suppress unrelated conformance failures + * across a whole site; one that is too narrow makes a baseline entry stop + * matching the moment a counter changes. So the tests are written as explicit + * should-collapse / must-stay-distinct pairs over selectors axe really emits on + * Quarto output, rather than as assertions about the regexes themselves. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assert, assertEquals, assertNotEquals } from "testing/asserts"; +import { + normalizeSelector, + signatureOf, +} from "../../src/command/dev-call/axe/aggregate.ts"; +import { AxeViolationNode } from "../../src/command/dev-call/axe/scan.ts"; + +function node(target: string[] | string): AxeViolationNode { + return { + html: "
", + target: Array.isArray(target) ? target : [target], + }; +} + +// --------------------------------------------------------------------------- +// Pairs that must collapse: the same defect, repeated by shared or generated +// code, has to be one finding with a count. +// --------------------------------------------------------------------------- + +const kMustCollapse: [string, string, string][] = [ + [ + "collapsible callouts differing only by index", + 'div[data-bs-target=".callout-4-contents"]', + 'div[data-bs-target=".callout-11-contents"]', + ], + [ + "sidebar sections differing only by index", + 'a[data-bs-target="#quarto-sidebar-section-1"]', + 'a[data-bs-target="#quarto-sidebar-section-2"]', + ], + [ + "code blocks: Pandoc's #cb- ids", + "#cb1-1 > code", + "#cb12-3 > code", + ], + [ + "footnote backrefs: #fn", + "#fn3 > p > a", + "#fn17 > p > a", + ], + [ + "nth-child position, which unrelated sibling edits change", + "p:nth-child(3) > a", + "p:nth-child(7) > a", + ], + [ + "href values, which differ per link but not per defect", + 'a[href="/docs/guide.html"]', + 'a[href="/docs/other.html"]', + ], + [ + "axe's bare-tag target versus the same tag with a position", + "h6", + "h6:nth-child(1)", + ], +]; + +for (const [label, a, b] of kMustCollapse) { + unitTest( + `axe signature - collapses: ${label}`, + // deno-lint-ignore require-await + async () => { + assertEquals( + normalizeSelector(a), + normalizeSelector(b), + `expected these to share a signature:\n ${a}\n ${b}`, + ); + }, + ); +} + +// --------------------------------------------------------------------------- +// Pairs that must stay distinct. Every one of these is a way for an accepted +// finding to silently suppress an unrelated one. +// --------------------------------------------------------------------------- + +const kMustStayDistinct: [string, string, string][] = [ + [ + "a callout and a modal both hang off data-bs-target", + 'div[data-bs-target=".callout-4-contents"]', + 'button[data-bs-target="#exampleModal"]', + ], + [ + "a callout and a carousel both hang off data-bs-target", + 'div[data-bs-target=".callout-4-contents"]', + 'button[data-bs-target="#carouselExampleControls"]', + ], + [ + "a sidebar link and a tabset link both hang off data-bs-target", + 'a[data-bs-target="#quarto-sidebar-section-1"]', + 'a[data-bs-target="#tabset-1-1"]', + ], + [ + "data-anchor-id values are content slugs, so headings are separate", + 'h4[data-anchor-id="markdown-syntax"]', + 'h4[data-anchor-id="latex-raw-blocks"]', + ], + [ + "the same slug at different heading levels is a different heading", + 'h4[data-anchor-id="markdown-syntax"]', + 'h6[data-anchor-id="markdown-syntax"]', + ], + [ + "an image in a layout cell versus a bare paragraph image", + ".quarto-layout-cell > p > .img-fluid", + "p > .img-fluid", + ], + [ + "different rules never share a signature even with one selector", + "h6", + "th", + ], + [ + "htmlwidget instances keep their hashes apart", + "#htmlwidget-9f0c1b2a3d4e", + "#htmlwidget-aa11bb22cc33", + ], +]; + +for (const [label, a, b] of kMustStayDistinct) { + unitTest( + `axe signature - keeps distinct: ${label}`, + // deno-lint-ignore require-await + async () => { + assertNotEquals( + normalizeSelector(a), + normalizeSelector(b), + `these must not share a signature:\n ${a}\n ${b}`, + ); + }, + ); +} + +// --------------------------------------------------------------------------- +// Exact output, so a scheme change is visible in a diff rather than inferred +// from a collapse count. Update these together with kSignatureScheme. +// --------------------------------------------------------------------------- + +const kExactOutput: [string, string][] = [ + [ + 'div[data-bs-target=".callout-4-contents"]', + 'div[data-bs-target=".callout-*-contents"]', + ], + ['a[data-bs-target="#tabset-1-1"]', 'a[data-bs-target="#tabset-*-*"]'], + [ + 'h4[data-anchor-id="markdown-syntax"]', + 'h4[data-anchor-id="markdown-syntax"]', + ], + ["#cb12-3 > code", "#cb > code"], + ["#fn17 > p > a", "#fn > p > a"], + ["p:nth-child(3) > a", "p > a"], + ['a[href="/docs/guide.html"]', "a"], + ['div[id="quarto-content"][data-index="3"]', "div"], + ['input[name="search-input"]', "input"], + ['img[src="elephant.png"][style="width:60px"]', 'img[src="elephant.png"]'], + ["#download-news > h6", "#download-news > h6"], +]; + +for (const [input, expected] of kExactOutput) { + unitTest( + `axe signature - normalizes ${input} -> ${expected}`, + // deno-lint-ignore require-await + async () => { + assertEquals(normalizeSelector(input), expected); + }, + ); +} + +unitTest( + "axe signature - axe's target array is joined into a descendant path", + // deno-lint-ignore require-await + async () => { + assertEquals( + normalizeSelector([".quarto-layout-cell", "p", ".img-fluid"]), + ".quarto-layout-cell > p > .img-fluid", + ); + }, +); + +// --------------------------------------------------------------------------- +// signatureOf: rule prefix, and the color-contrast special case +// --------------------------------------------------------------------------- + +unitTest( + "signatureOf - prefixes the rule id, so two rules never collide", + // deno-lint-ignore require-await + async () => { + assertEquals( + signatureOf("heading-order", node("h6")), + "heading-order :: h6", + ); + assertNotEquals( + signatureOf("heading-order", node("h6")), + signatureOf("empty-heading", node("h6")), + ); + }, +); + +unitTest( + "signatureOf - color-contrast keys on the colour pair, not the location", + // deno-lint-ignore require-await + async () => { + const contrastNode = (target: string): AxeViolationNode => ({ + html: "

", + target: [target], + any: [{ + id: "color-contrast", + data: { fgColor: "#767676", bgColor: "#ffffff" }, + }], + }); + // The root cause is one theme colour pair, so unrelated elements sharing it + // are one finding rather than one per element. + assertEquals( + signatureOf("color-contrast", contrastNode(".sidebar-link")), + "color-contrast :: #767676 on #ffffff", + ); + assertEquals( + signatureOf("color-contrast", contrastNode("p > code")), + signatureOf("color-contrast", contrastNode(".sidebar-link")), + ); + }, +); + +unitTest( + "signatureOf - color-contrast falls back to the selector without colour data", + // deno-lint-ignore require-await + async () => { + // A payload with no color-contrast check result must not produce + // "undefined on undefined" and collapse every contrast finding into one. + const signature = signatureOf("color-contrast", node(".sidebar-link")); + assertEquals(signature, "color-contrast :: .sidebar-link"); + assert(!signature.includes("undefined")); + }, +); From 945aa71ad227582b417819eb92ed55d42952c6b2 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 11:49:51 -0700 Subject: [PATCH 06/43] Let Zod's .strict() do the baseline error reporting The two-pass reader was built on a wrong premise. I had it that a schema can't report an unrecognized key and a missing required field together, because a `superRefine` never runs once a required field is missing. That is true of `superRefine`, but strict-key checking is part of the object parse itself, so `.strict()` reports both in one pass: findings.0.impact: Required findings.0: Unrecognized key(s) in object: 'impcat' Adjacent lines make the typo obvious, so the hand-rolled Levenshtein "did you mean" was polish sitting on top of ~60 lines that Zod already covers. Deleted, along with the tolerated-field set and the manual per-entry loop; schemas.ts drops from 341 to 275 lines. Keeping the paste-a-whole-finding affordance costs one expression rather than the machinery: declare every finding field as an ignorable optional and merge the entry schema over the top, so its required fields stay required. A finding pasted out of findings.json plus a note validates; an invented field is still rejected. `parseBaseline` survives only for the signature-scheme check, which needs to explain what a mismatch means rather than say "invalid literal". Behaviour change worth noting: the outer object is `.strict()` too, so an unknown top-level key is now an error. That catches a misspelled `findings` but also rejects a hand-added `"_comment"`, since JSON has nowhere else to put one. tests/unit/axe-baseline-parse.test.ts pins the deliberate choices: the paste shortcut validates, a typo names both halves, an invented field is rejected, `pages` and `note` are required so scope and rationale are never guessed, and a stale scheme is one named error rather than mass staleness. 42 axe unit tests passing. --- src/command/dev-call/axe/schemas.ts | 140 +++++----------- tests/unit/axe-baseline-parse.test.ts | 224 ++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 103 deletions(-) create mode 100644 tests/unit/axe-baseline-parse.test.ts diff --git a/src/command/dev-call/axe/schemas.ts b/src/command/dev-call/axe/schemas.ts index 3cdf0dca935..6a6f0a3e784 100644 --- a/src/command/dev-call/axe/schemas.ts +++ b/src/command/dev-call/axe/schemas.ts @@ -192,56 +192,23 @@ export const axeBaselineEntrySchema = z.object({ export type AxeBaselineEntry = z.infer; /** - * Fields the tolerant reader ignores rather than rejects: everything a finding - * carries, so pasting a whole finding out of `findings.json` and adding a - * `note` works. Anything outside this set is treated as a misspelling. + * The baseline as read from disk. + * + * `.strict()` is the whole error story: a hand-edited `impcat:` is reported as + * an unrecognized key *and* as a missing `impact` in the same pass, so the two + * lines sit next to each other and the typo is obvious. (Unlike `superRefine`, + * strict-key checking is part of the object parse, so a missing required field + * doesn't short-circuit it.) + * + * Every field a *finding* carries is declared as an ignorable optional, so + * pasting a whole finding out of `findings.json` and adding a `note` validates. + * Merge order matters: the entry schema goes last, so its required fields stay + * required rather than being softened to optional. */ -const kToleratedEntryFields = new Set([ - ...Object.keys(axeFindingSchema.shape), - ...Object.keys(axeBaselineEntrySchema.shape), -]); - -/** Levenshtein distance, for "did you mean" on a misspelled field. */ -function editDistance(a: string, b: string): number { - const rows = Array.from( - { length: a.length + 1 }, - (_, i) => [i, ...Array(b.length).fill(0)], - ); - for (let j = 0; j <= b.length; j++) { - rows[0][j] = j; - } - for (let i = 1; i <= a.length; i++) { - for (let j = 1; j <= b.length; j++) { - rows[i][j] = Math.min( - rows[i - 1][j] + 1, - rows[i][j - 1] + 1, - rows[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), - ); - } - } - return rows[a.length][b.length]; -} +const axeBaselineEntryReader = axeFindingSchema.partial() + .merge(axeBaselineEntrySchema) + .strict(); -function didYouMean(key: string): string | undefined { - let best: string | undefined; - let bestDistance = Infinity; - for (const candidate of kToleratedEntryFields) { - const distance = editDistance(key.toLowerCase(), candidate.toLowerCase()); - if (distance < bestDistance) { - bestDistance = distance; - best = candidate; - } - } - // only suggest when it really looks like a typo rather than a new field - return bestDistance <= Math.max(1, Math.floor(key.length / 3)) - ? best - : undefined; -} - -/** - * The baseline as read from disk. Unknown-but-known-shaped keys pass through, - * so pasting a whole finding out of `findings.json` works. - */ export const axeBaselineSchema = z.object({ /** * The signature scheme the entries were written against. Optional: a ledger @@ -250,8 +217,8 @@ export const axeBaselineSchema = z.object({ * a silent mass-invalidation. */ signatureScheme: z.number().optional(), - findings: z.array(axeBaselineEntrySchema.passthrough()), -}).passthrough(); + findings: z.array(axeBaselineEntryReader), +}).strict(); export type AxeBaseline = z.infer; @@ -266,43 +233,26 @@ export type ParsedBaseline = | { success: false; issues: AxeBaselineIssue[] }; /** - * Validate a hand-edited baseline, reporting misspelled field names alongside - * the shape errors they cause. + * Validate a hand-edited baseline. * - * Zod alone isn't enough here. A misspelled `impcat:` is, to a tolerant object - * schema, an unknown key (ignored) plus a missing `impact` (an error) — so the - * report says "impact: Required" while the culprit sits in plain sight two - * characters away. Zod also short-circuits: a `superRefine` on the entry never - * runs once a required field is missing, so the two checks can't be composed. - * Running them separately and merging the issues gives the reader both halves. + * The scheme check is separate from the schema because it needs to explain + * itself: a mismatch means every entry would read as stale, which is worth more + * than "invalid literal". It runs first, since the shape errors underneath it + * would be noise. */ export function parseBaseline(data: unknown): ParsedBaseline { - const issues: AxeBaselineIssue[] = []; - - const outer = z.object({ - signatureScheme: z.number().optional(), - findings: z.array(z.record(z.unknown())), - }).passthrough().safeParse(data); - if (!outer.success) { - return { - success: false, - issues: outer.error.issues.map((issue) => ({ - path: issue.path.join(".") || "(root)", - message: issue.message, - })), - }; - } - + const scheme = z.object({ signatureScheme: z.number().optional() }) + .passthrough().safeParse(data); if ( - outer.data.signatureScheme !== undefined && - outer.data.signatureScheme !== kSignatureScheme + scheme.success && scheme.data.signatureScheme !== undefined && + scheme.data.signatureScheme !== kSignatureScheme ) { return { success: false, issues: [{ path: "signatureScheme", message: - `this baseline was written against signature scheme ${outer.data.signatureScheme}, ` + + `this baseline was written against signature scheme ${scheme.data.signatureScheme}, ` + `but this build emits scheme ${kSignatureScheme}. Every entry would read as ` + `stale. Re-annotate the entries against the new signatures — the raw ` + `selectors are unchanged in each finding's occurrences[].target — then ` + @@ -311,31 +261,15 @@ export function parseBaseline(data: unknown): ParsedBaseline { }; } - outer.data.findings.forEach((entry, index) => { - for (const key of Object.keys(entry)) { - if (kToleratedEntryFields.has(key)) { - continue; - } - const suggestion = didYouMean(key); - issues.push({ - path: `findings[${index}].${key}`, - message: `unknown field '${key}'` + - (suggestion ? ` — did you mean '${suggestion}'?` : ""), - }); - } - const parsed = axeBaselineEntrySchema.passthrough().safeParse(entry); - if (!parsed.success) { - for (const issue of parsed.error.issues) { - issues.push({ - path: `findings[${index}].${issue.path.join(".")}`, - message: issue.message, - }); - } - } - }); - - if (issues.length) { - return { success: false, issues }; + const result = axeBaselineSchema.safeParse(data); + if (!result.success) { + return { + success: false, + issues: result.error.issues.map((issue) => ({ + path: issue.path.join(".") || "(root)", + message: issue.message, + })), + }; } - return { success: true, baseline: axeBaselineSchema.parse(data) }; + return { success: true, baseline: result.data }; } diff --git a/tests/unit/axe-baseline-parse.test.ts b/tests/unit/axe-baseline-parse.test.ts new file mode 100644 index 00000000000..a045212f421 --- /dev/null +++ b/tests/unit/axe-baseline-parse.test.ts @@ -0,0 +1,224 @@ +/* + * axe-baseline-parse.test.ts + * + * Tests reading a hand-edited `_axe-baseline.json`: `parseBaseline` in + * src/command/dev-call/axe/schemas.ts. + * + * The baseline is the one file in this feature a human writes by hand, so its + * failure modes are ergonomics, not just correctness. Two behaviours are chosen + * deliberately and pinned here: a whole finding pasted out of `findings.json` + * validates (that's the intended authoring shortcut), while a misspelled field + * name is an error rather than a silently ignored unknown key — because + * ignoring `impcat:` would mean "no impact recorded", which reads as an + * acceptance at the weakest impact. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assert, assertEquals } from "testing/asserts"; +import { + kSignatureScheme, + parseBaseline, +} from "../../src/command/dev-call/axe/schemas.ts"; + +const kTrimmedEntry = { + signature: "aria-allowed-role :: .navbar-toggler", + pages: [], + impact: "minor", + note: "tracked upstream", +}; + +function issuesFor(data: unknown): string[] { + const result = parseBaseline(data); + assert(!result.success, "expected parseBaseline to fail"); + return result.issues.map((issue) => `${issue.path}: ${issue.message}`); +} + +unitTest( + "parseBaseline - accepts the trimmed entry shape", + // deno-lint-ignore require-await + async () => { + const result = parseBaseline({ findings: [kTrimmedEntry] }); + assert(result.success, JSON.stringify(result)); + assertEquals(result.baseline.findings.length, 1); + assertEquals(result.baseline.findings[0].impact, "minor"); + }, +); + +unitTest( + "parseBaseline - accepts an empty ledger", + // deno-lint-ignore require-await + async () => { + const result = parseBaseline({ findings: [] }); + assert(result.success); + }, +); + +unitTest( + "parseBaseline - accepts a whole finding pasted from findings.json", + // deno-lint-ignore require-await + async () => { + // Every field a finding carries, so the authoring shortcut is "copy the + // finding, add a note" rather than "copy the finding, then delete 16 keys". + const pasted = { + id: "image-alt-4468cc", + signature: "image-alt :: img", + rule: "image-alt", + impact: "critical", + conformance: "WCAG 2.0 A (1.1.1)", + standard: "WCAG 2.0 A", + standardRank: 0, + severityRank: 0, + bestPractice: false, + help: "Images must have alternative text", + helpUrl: "https://dequeuniversity.com/rules/axe/4.10/image-alt", + detail: "Element does not have an alt attribute", + baselined: false, + baselineNote: null, + label: "systemic", + instances: 2, + pages: ["about.html", "index.html"], + cells: 4, + viewports: ["1440x900"], + themes: ["light"], + examples: ["img"], + occurrences: [], + note: "accepted while the upstream fix ships", + }; + const result = parseBaseline({ findings: [pasted] }); + assert(result.success, JSON.stringify(result)); + assertEquals(result.baseline.findings[0].pages, [ + "about.html", + "index.html", + ]); + }, +); + +unitTest( + "parseBaseline - a misspelled field is an error, next to the field it broke", + // deno-lint-ignore require-await + async () => { + const issues = issuesFor({ + findings: [{ ...kTrimmedEntry, impcat: "minor", impact: undefined }], + }); + // Both halves must be reported: the unknown key names the culprit, and the + // missing required field says what it cost. One without the other sends the + // reader looking in the wrong place. + assert( + issues.some((issue) => issue.includes("impcat")), + `expected the typo to be named: ${JSON.stringify(issues)}`, + ); + assert( + issues.some((issue) => issue.startsWith("findings.0.impact")), + `expected the missing field to be named: ${JSON.stringify(issues)}`, + ); + }, +); + +unitTest( + "parseBaseline - an invented field is rejected rather than ignored", + // deno-lint-ignore require-await + async () => { + const issues = issuesFor({ + findings: [{ ...kTrimmedEntry, expiresOn: "2027-01-01" }], + }); + assert( + issues.some((issue) => issue.includes("expiresOn")), + JSON.stringify(issues), + ); + }, +); + +unitTest( + "parseBaseline - pages is required, so scope is never guessed", + // deno-lint-ignore require-await + async () => { + // An omitted `pages` must not default to the broader reading. `[]` means + // site-wide and has to be written on purpose. + const { pages: _pages, ...withoutPages } = kTrimmedEntry; + const issues = issuesFor({ findings: [withoutPages] }); + assert( + issues.some((issue) => issue.startsWith("findings.0.pages")), + JSON.stringify(issues), + ); + }, +); + +unitTest( + "parseBaseline - note is required, so every entry says why", + // deno-lint-ignore require-await + async () => { + const { note: _note, ...withoutNote } = kTrimmedEntry; + const issues = issuesFor({ findings: [withoutNote] }); + assert( + issues.some((issue) => issue.startsWith("findings.0.note")), + JSON.stringify(issues), + ); + }, +); + +unitTest( + "parseBaseline - an unknown impact is rejected", + // deno-lint-ignore require-await + async () => { + const issues = issuesFor({ + findings: [{ ...kTrimmedEntry, impact: "catastrophic" }], + }); + assert( + issues.some((issue) => issue.startsWith("findings.0.impact")), + JSON.stringify(issues), + ); + }, +); + +unitTest( + "parseBaseline - a stale signature scheme is a named error, not mass staleness", + // deno-lint-ignore require-await + async () => { + const issues = issuesFor({ + signatureScheme: kSignatureScheme - 1, + findings: [kTrimmedEntry], + }); + assertEquals(issues.length, 1, JSON.stringify(issues)); + assert(issues[0].startsWith("signatureScheme:")); + // The message has to say what to do, since every entry silently ceasing to + // match looks exactly like "everything was fixed". + assert(issues[0].includes("occurrences[].target"), issues[0]); + }, +); + +unitTest( + "parseBaseline - a matching signature scheme is accepted", + // deno-lint-ignore require-await + async () => { + const result = parseBaseline({ + signatureScheme: kSignatureScheme, + findings: [kTrimmedEntry], + }); + assert(result.success, JSON.stringify(result)); + }, +); + +unitTest( + "parseBaseline - a ledger with no scheme is assumed current", + // deno-lint-ignore require-await + async () => { + // The first ledger anyone hand-writes shouldn't need the ceremony. + const result = parseBaseline({ findings: [kTrimmedEntry] }); + assert(result.success, JSON.stringify(result)); + assertEquals(result.baseline.signatureScheme, undefined); + }, +); + +unitTest( + "parseBaseline - findings must be present", + // deno-lint-ignore require-await + async () => { + const issues = issuesFor({}); + assert( + issues.some((issue) => issue.startsWith("findings")), + JSON.stringify(issues), + ); + }, +); From a1e159011dab36ed0f9e5b50f4d14d250ca06aee Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Thu, 20 Aug 2026 12:14:52 -0700 Subject: [PATCH 07/43] M3: fixture site, unit tests and a browser smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A purpose-built inaccessible Quarto site under tests/docs/axe-scan/site, with every violation planted on purpose and a manifest README recording intent. Source only: a committed render would carry site_libs/ and go stale, so the smoke test renders it in setup and removes it after. Writing the manifest first paid off twice, because two planted cases did not behave as designed. The tier-2 links were meant to prove that three `#panel-N` components collapse to one signature while a `#settings-dialog` stays separate. They all collapsed. axe had keyed them on `href`, which the normalizer drops as volatile, so the attribute never reached the selector. Switching to `", + "target": [ + "button[data-widget-target=\"#panel-1\"]" + ], + "failureSummary": "Fix any of the following:\n Element does not have inner text that is visible to screen readers\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute\n Element does not have an implicit (wrapped)