diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index 1488b51782..e920d7cd26 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -67,12 +67,19 @@ jobs: matrix: include: - project: mock + config: playwright.config.ts shard: 1 total: 2 - project: mock + config: playwright.config.ts shard: 2 total: 2 - project: seeded + config: playwright.config.ts + shard: 1 + total: 1 + - project: analytics + config: playwright.analytics.config.ts shard: 1 total: 1 outputs: @@ -119,8 +126,12 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium + - name: Build the isolated analytics frontend + if: matrix.project == 'analytics' + run: npm run build + - name: Run E2E shard (no credentials needed) - run: npx playwright test --fail-on-flaky-tests --project=${{ matrix.project }} --shard=${{ matrix.shard }}/${{ matrix.total }} + run: npx playwright test --fail-on-flaky-tests --config=${{ matrix.config }} --project=${{ matrix.project }} --shard=${{ matrix.shard }}/${{ matrix.total }} env: CI: true # Mock specs still use the backend for auth bootstrap and some API calls. diff --git a/doc/gui/0_gui.md b/doc/gui/0_gui.md index 6292b22a8d..90d8617ea3 100644 --- a/doc/gui/0_gui.md +++ b/doc/gui/0_gui.md @@ -35,7 +35,75 @@ To deploy an isolated instance for an external team, see [Deploy a New Instance] ## Views -CoPyRIT has three main views, accessible from the left sidebar: **Technique.execute** (Chat), **Attack History**, and **Target Configuration**. The **Theme** menu is available at the bottom of the sidebar. +Use the left sidebar to open **Home**, **Technique.execute** (Chat), **History**, **Analytics**, **Scanner**, +**Datasets**, or the **Registry**. **Configuration** is available to administrators. +The **Theme** menu is at the bottom of the sidebar. + +### Attack Results Analytics + +**Analytics** explores saved attack results using the +[SDK analytics API](../code/analytics/0_attack_results.md). Every distinct saved +AttackResult ID counts, including different IDs sharing a conversation and all +result roles. Selecting a scenario filters this same population, including its +retries. It does not switch to the scanner's latest-execution-unit statistics. + +Use **Operation**, **Operator**, **Objective target**, or **Add filter** to select +saved metadata. Additional dimensions include model, targeted harm category, +attack type, request or response converters, scenario run, and arbitrary label +keys. Targeted harms describe intended coverage, not harms detected by scorers. +Target options use persisted behavioral identities, not current registry +membership. The content hash in a matching row is for inspecting that saved +target, not a substitute for its filter key. + +Filter values load only when their editor is open, with debounced search and +bounded **First values** / **Next values** paging. Selections survive searches +and pages. Values within a filter are alternatives (**ANY**); converters also +support **ALL**. Separate filter chips are combined with **AND**, even when +they select the same dimension. Missing metadata, an empty converter pipeline, +and literal values such as `Unknown` remain distinct. + +**Last updated** accepts ISO 8601 timestamps with `Z` (UTC) or an explicit offset. +For example, `2026-11-01T01:30:00-07:00` and `2026-11-01T01:30:00-08:00` +are different instants during a daylight saving transition. After is inclusive; +before is exclusive. Microseconds are retained. These bounds describe last +updates, not attack execution times. + +The dashboard offers outcome-composition bars, success-rate bars, and a +two-axis heatmap. **ASR denominator** defaults to decided results (successes plus +failures). All-results ASR includes errors and undetermined outcomes. Both rates +come from the SDK. Counts are explicit: `100%, 3 success / 3 decided, 5 total` +means three successes and two results without a decision, not five successes. +An unavailable decided denominator displays **Unavailable**, not `0%`. +Outcome filters apply to the entire dashboard and mark **ASR*** with an +explanation, including the `100%*` produced by a successes-only cohort. + +Click a group, outcome segment, or heatmap cell to append the SDK's drill-down +predicates. Overlapping group totals must not be added together. When the +16-predicate or 500-value budget prevents another drill-down, the SDK's reason +is displayed; the current charts, results, and filter editing remain available. +Group and result pages stay bounded. **Show cell data** provides a textual +heatmap view. Matching result rows open the ordinary saved-attack view with a +click, Enter, or Space. + +Use **Reload** for a new report. **Last refreshed** shows the last successful +report time; paging matching results updates only **Page read**, not the report. +Presentation changes preserve pages and do not request new aggregates. +The dashboard does not poll, although the app's shared readiness checks continue. +Errors retain a same-cohort last successful report with a stale warning. Busy +read retries honor the server's cooldown; invalid result cursors can return to +the first results page without recalculating the report. + +Copy the browser URL to share filters and presentation. New links put the state +after `#analytics=`, so large filter values are not sent in HTTP request targets +and do not exceed the backend's 8 KiB URL limit on reload. Valid older query +links are migrated to this representation; invalid links require an explicit +reset rather than silently querying a broader cohort. Browser Back and the +Analytics sidebar button retain the exploration. Links contain the selected +metadata values, so share them only with appropriate recipients. + +Outcome fills are consistent across summaries, bars, swatches, and result +badges: success is green, failure red, execution error blue, and undetermined +gray. Text and keyboard actions remain available independently of color. ### Themes diff --git a/frontend/README.md b/frontend/README.md index 238edfbf35..35811b6669 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -60,6 +60,51 @@ a new attack, the draft is kept so you can retry after loading finishes. Copies and drafts saved to an existing attack remain available during defaults reloads and keep that attack's attribution. +## Attack results analytics + +Open **Analytics** to filter saved results, compare SDK-computed outcome +statistics, and drill into matching attacks. See the +[analytics GUI guide](../doc/gui/0_gui.md#attack-results-analytics) for cohort +semantics, denominators, time bounds, URL sharing, and refresh behavior. +The implementation follows +[part 5 of the analytics roadmap](https://gist.github.com/romanlutz/34230aba87f7fa07996772a4099e9a78). + +The shared `AttackResults/OutcomeSummaryBar` accepts `OutcomeStatistics`, not +result rows. This keeps it reusable without counting a paginated sample or +inferring historical scenario breakdowns from retries. + +### Isolated analytics browser checks + +From `frontend`, run: + +```bash +npm run test:e2e:analytics +``` + +This builds the current frontend and starts the real application routes and +compatibility/readiness middleware over a disposable local SQLite database. +Its synthetic results include repeated conversation IDs, all result roles, +converter memberships, and a saved retry scenario. It uses a private home, +explicit empty environment-file/initializer lists, and no provider calls, +shared database, or Azure credentials. The normal E2E configuration excludes +this fixture-specific spec; its dedicated configuration owns the server. +CI runs it as an additional isolated shard of the required Frontend E2E check, +using the same report upload, merge, and failure gate as the other projects. +Python dependencies must first be available through `uv sync` at the repository +root. `PYRIT_E2E_ANALYTICS_PORT` changes the loopback port when needed. +If Python is managed only by uv, point Vite's existing Python override at that +environment before running the command. In PowerShell: + +```powershell +$env:PYRIT_PYTHON = uv run python -c "import sys; print(sys.executable)" +npm run test:e2e:analytics +``` + +Set `PYRIT_ANALYTICS_SCREENSHOT_DIR` to an artifact directory to also capture the +verified overview in light/dark, a heatmap drill-down, and narrow controls/report views. +The browser checks do not mock analytics APIs. They do not repeat the separate +100,000-result backend performance workload. + ## Development ```bash @@ -206,10 +251,12 @@ E2E_LIVE_MODE=true npx playwright test The mock and seeded projects run in the **GitHub Actions** pull-request workflow. The live project is intended for a protected pipeline with an Entra identity or API keys. -CI runs two isolated mock shards alongside one serial seeded job, with one -worker per job. Each job starts its own Vite server and Python backend because -mock specs still depend on backend auth bootstrap and some API calls. The jobs -never share backend state. The required **Frontend E2E Tests** check merges their reports and fails +CI runs two isolated mock shards, one serial seeded job, and one isolated +analytics job, with one worker per job. Mock and seeded jobs start their own +Vite server and Python backend because mock specs still depend on backend auth +bootstrap and some API calls. The analytics job serves its production build +from the disposable SQLite fixture. Jobs never share backend state. +The required **Frontend E2E Tests** check merges their reports and fails if any shard fails, is cancelled, or is missing a report. Skipped and flaky tests also fail CI. diff --git a/frontend/e2e/analytics.spec.ts b/frontend/e2e/analytics.spec.ts new file mode 100644 index 0000000000..e99d41c1b0 --- /dev/null +++ b/frontend/e2e/analytics.spec.ts @@ -0,0 +1,349 @@ +import { mkdir } from 'node:fs/promises' +import path from 'node:path' + +import { expect, test } from '@playwright/test' +import type { Page, Request, Response } from '@playwright/test' + +import type { + AttackAnalyticsCell, AttackAnalyticsFilter, AttackAnalyticsOption, AttackAnalyticsReport, AttackAnalyticsResultRow, + AttackAnalyticsViewState, AttackOutcome, +} from '../src/types' +import { + DEFAULT_ANALYTICS_VIEW, analyticsViewToHash, analyticsViewToSearchParams, + formatAnalyticsCount, formatAnalyticsPercent, analyticsSuccessCountsLabel, analyticsValueKey, +} from '../src/utils/attackAnalytics' +import { compatibilityHeaders, getCompatibilityId } from './_compatibility' + +const BASELINE: AttackAnalyticsFilter = { + dimension: { name: 'operation' }, values: [{ kind: 'value', value: 'Synthetic baseline' }], match_mode: 'any', +} +const OVERVIEW: AttackAnalyticsFilter = { + dimension: { name: 'label', label_key: 'cohort' }, values: [{ kind: 'value', value: 'overview' }], match_mode: 'any', +} +const errors = new WeakMap() + +function reportResponse(response: Response): boolean { + return response.url().endsWith('/api/analytics/attacks/query') && response.request().method() === 'POST' +} + +async function readReport(response: Response): Promise { + expect(response.status()).toBe(200) + const report: AttackAnalyticsReport = await response.json() + expect(report.summary.success_rate_decided).toBe(report.summary.success_rate) + return report +} + +async function openReport(page: Page, view?: AttackAnalyticsViewState): Promise { + const pending = page.waitForResponse(reportResponse) + await page.goto(`/analytics${view ? analyticsViewToHash(view) : ''}`) + const report = await readReport(await pending) + await expect(page.getByRole('region', { name: 'Outcome summary' })).toBeVisible() + await expect(page.getByRole('row', { name: `Open result ${report.results.items[0].attack_result_id}` })).toBeVisible() + return report +} + +function observe(page: Page) { + const reports: Request[] = [] + const results: Request[] = [] + const facets: Request[] = [] + page.on('request', (request: Request) => { + if (request.url().endsWith('/analytics/attacks/query')) reports.push(request) + if (request.url().endsWith('/analytics/attacks/results')) results.push(request) + if (request.url().endsWith('/analytics/attacks/facets')) facets.push(request) + }) + return { reports, results, facets } +} + +async function capture(page: Page, name: string): Promise { + const directory = process.env.PYRIT_ANALYTICS_SCREENSHOT_DIR + if (!directory) return + await expect(page.getByRole('region', { name: 'Outcome summary' })).toBeVisible() + await expect(page.getByRole('button', { name: 'Reload', exact: true })).toBeEnabled() + await mkdir(directory, { recursive: true }) + await page.screenshot({ path: path.join(directory, name), fullPage: true }) +} + +function channels(color: string): number[] { + const values = color.match(/[\d.]+/g)?.slice(0, 3).map(Number) + if (!values || values.length !== 3) throw new Error(`Expected RGB color, got ${color}`) + return values +} + +function contrast(foreground: string, background: string): number { + const luminance = (color: string): number => { + const values = channels(color).map((channel: number) => { + const value = channel / 255 + return value <= 0.04045 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4 + }) + return values[0] * 0.2126 + values[1] * 0.7152 + values[2] * 0.0722 + } + const first = luminance(foreground) + const second = luminance(background) + return (Math.max(first, second) + 0.05) / (Math.min(first, second) + 0.05) +} + +test.beforeEach(async ({ page }: { page: Page }) => { + const messages: string[] = [] + errors.set(page, messages) + page.on('pageerror', (error: Error) => { messages.push(error.message) }) +}) + +test.afterEach(async ({ page }: { page: Page }) => { + expect(errors.get(page)).toEqual([]) +}) + +test('counts every saved ID and keeps result paging, presentation, and freshness independent', async ({ page }: { page: Page }) => { + const requests = observe(page) + const report = await openReport(page) + expect(report.summary).toMatchObject({ total_results: 204, successes: 83, failures: 41, errors: 20, undetermined: 60 }) + const summary = page.getByRole('region', { name: 'Outcome summary' }) + await expect(summary.getByText('204', { exact: true })).toBeVisible() + await expect(summary.getByText('83 success / 124 decided')).toBeVisible() + expect(requests.reports).toHaveLength(1) + expect(requests.facets).toHaveLength(0) + expect(requests.reports[0].headers()['pyrit-compatibility-id']).toBe(getCompatibilityId()) + const refreshed = await page.getByText(/Last refreshed:/).locator('time').getAttribute('datetime') + const results = page.getByRole('region', { name: 'Matching AttackResults' }) + await results.getByRole('button', { name: 'Next', exact: true }).click() + await expect(results.getByText('Page 2', { exact: true })).toBeVisible() + expect(requests.results).toHaveLength(1) + await page.getByLabel('Chart view', { exact: true }).selectOption('success-rate') + await page.getByLabel('ASR denominator', { exact: true }).selectOption('all') + await expect(summary.getByText(formatAnalyticsPercent(report.summary.success_rate_all), { exact: true })).toBeVisible() + await expect(summary.getByText('83 success / 204 all results')).toBeVisible() + await expect(results.getByText('Page 2', { exact: true })).toBeVisible() + await page.clock.install() + await page.clock.fastForward(300_000) + expect(requests.reports).toHaveLength(1) + expect(requests.results).toHaveLength(1) + expect(await page.getByText(/Last refreshed:/).locator('time').getAttribute('datetime')).toBe(refreshed) + const reloaded = page.waitForResponse(reportResponse) + await page.getByRole('button', { name: 'Reload', exact: true }).click() + await page.clock.fastForward(250) + await readReport(await reloaded) + await expect(results.getByText('Page 1', { exact: true })).toBeVisible() + expect(requests.reports).toHaveLength(2) +}) + +test('drills into a real heatmap cell and opens its saved result through the normal route', async ({ page }: { page: Page }) => { + await page.setViewportSize({ width: 1440, height: 1180 }) + const report = await openReport(page, { + ...DEFAULT_ANALYTICS_VIEW, chart: 'heatmap', filters: { ...DEFAULT_ANALYTICS_VIEW.filters, dimensions: [BASELINE] }, + }) + expect(report.summary.total_results).toBe(80) + const rowOption = report.rows.find((option: AttackAnalyticsOption) => option.label === 'Privacy') + const columnOption = report.columns.find((option: AttackAnalyticsOption) => option.label === 'PromptSendingAttack') + if (!rowOption || !columnOption) throw new Error('Synthetic heatmap axes are missing') + const cell = report.cells.find((item: AttackAnalyticsCell) => + analyticsValueKey(item.row) === analyticsValueKey(rowOption.key) && + analyticsValueKey(item.column) === analyticsValueKey(columnOption.key)) + expect(cell).toBeDefined() + if (!cell) throw new Error('Synthetic fixture is missing its Privacy / PromptSendingAttack cell') + const button = page.getByRole('button', { name: /^Privacy \/ PromptSendingAttack:/ }) + await expect(button.getByText(analyticsSuccessCountsLabel(cell.statistics), { exact: true })).toBeVisible() + await expect(button.getByText(`${formatAnalyticsCount(cell.statistics.total_results)} total`, { exact: true })).toBeVisible() + await page.getByRole('button', { name: 'Show cell data' }).click() + await expect(page.getByRole('region', { name: 'Heatmap aggregate data' })).toBeVisible() + await page.getByRole('button', { name: 'Hide cell data' }).click() + const next = page.waitForResponse(reportResponse) + await button.click() + const filtered = await readReport(await next) + expect(filtered.filters.dimensions).toEqual([...report.filters.dimensions, ...cell.drilldown_filters]) + expect(filtered.summary).toEqual(cell.statistics) + const id = filtered.results.items[0].attack_result_id + const row = page.getByRole('row', { name: `Open result ${id}` }) + await expect(row).toBeVisible() + await capture(page, 'analytics-heatmap-drilldown.png') + const sharedUrl = page.url() + await row.focus() + await page.keyboard.press('Enter') + await expect(page).toHaveURL(new RegExp(`/attacks/${id}$`)) + await expect(page.getByText('This is a safe, locally stored example response for analytics browser checks.')).toBeVisible() + await page.getByRole('button', { name: 'Analytics', exact: true }).click() + await expect(page).toHaveURL(sharedUrl) + await expect(page.getByRole('row', { name: `Open result ${id}` })).toBeVisible() + await page.getByRole('row', { name: `Open result ${id}` }).focus() + await page.keyboard.press('Space') + await expect(page).toHaveURL(new RegExp(`/attacks/${id}$`)) + await page.goBack() + await expect(page).toHaveURL(sharedUrl) + await expect(page.getByRole('row', { name: `Open result ${id}` })).toBeVisible() +}) + +test('keeps retries and all result roles when a saved scenario is selected', async ({ page }: { page: Page }) => { + await openReport(page) + await page.getByRole('button', { name: 'Add filter', exact: true }).click() + await page.getByLabel('Filter dimension', { exact: true }).selectOption('scenario') + await page.getByRole('checkbox', { name: /SyntheticRetryScenario/ }).check() + const pending = page.waitForResponse(reportResponse) + await page.getByRole('button', { name: 'Apply filter', exact: true }).click() + const report = await readReport(await pending) + expect(report.summary).toMatchObject({ total_results: 2, successes: 1, failures: 1, success_rate_decided: 0.5 }) + const ids = report.results.items.map((row: AttackAnalyticsResultRow) => row.attack_result_id) + expect(new Set(ids).size).toBe(2) + const conversations: string[] = [] + for (const id of ids) { + const response = await page.request.get(`/api/attacks/${id}`, { headers: compatibilityHeaders() }) + expect(response.status()).toBe(200) + const attack: { conversation_id: string } = await response.json() + conversations.push(attack.conversation_id) + } + expect(new Set(conversations).size).toBe(1) + const summary = page.getByRole('region', { name: 'Outcome summary' }) + await expect(summary.getByText('50%', { exact: true })).toBeVisible() + await page.getByRole('button', { name: 'Filter to success: 1', exact: true }).click() + await expect(summary.getByText('100%*', { exact: true })).toBeVisible() + await expect(page.getByRole('note', { name: 'Outcome-filtered ASR' })).toContainText('Outcome filter active') +}) + +test('loads paginated facets on demand and preserves selections across pages and search', async ({ page }: { page: Page }) => { + const requests = observe(page) + await openReport(page) + expect(requests.facets).toHaveLength(0) + await page.getByRole('button', { name: 'Operation', exact: true }).click() + await page.getByRole('checkbox', { name: 'Synthetic baseline', exact: true }).check() + await page.getByRole('button', { name: 'Next values', exact: true }).click() + await page.getByRole('checkbox', { name: 'Synthetic pilot', exact: true }).check() + await page.getByLabel('Search values', { exact: true }).fill('nothing-matches') + await expect(page.getByText(/No values match this search/)).toBeVisible() + await expect(page.getByRole('checkbox', { name: 'Synthetic baseline', exact: true })).toBeChecked() + await expect(page.getByRole('checkbox', { name: 'Synthetic pilot', exact: true })).toBeChecked() + const pending = page.waitForResponse(reportResponse) + await page.getByRole('button', { name: 'Apply filter', exact: true }).click() + const report = await readReport(await pending) + expect(report.summary.total_results).toBe(120) + expect(requests.facets).toHaveLength(3) + expect(requests.reports).toHaveLength(2) + await page.getByRole('button', { name: 'Add filter', exact: true }).click() + await page.getByLabel('Filter dimension', { exact: true }).selectOption('request_converters') + await page.getByLabel('Converter matching', { exact: true }).selectOption('all') + await page.getByRole('checkbox', { name: 'Base64Converter', exact: true }).check() + await page.getByRole('checkbox', { name: 'ROT13Converter', exact: true }).check() + const converters = page.waitForResponse(reportResponse) + await page.getByRole('button', { name: 'Apply filter', exact: true }).click() + const filtered = await readReport(await converters) + expect(filtered.filters.dimensions[1].match_mode).toBe('all') + expect(filtered.results.items.every((row: AttackAnalyticsResultRow) => row.request_converters.includes('Base64Converter') && + row.request_converters.includes('ROT13Converter'))).toBe(true) +}) + +test('large shared filters survive direct navigation, refresh, and Back within the server URL limit', async ({ page }: { page: Page }) => { + const view: AttackAnalyticsViewState = { + ...DEFAULT_ANALYTICS_VIEW, + filters: { ...DEFAULT_ANALYTICS_VIEW.filters, dimensions: ['a', 'b'].map((key: string): AttackAnalyticsFilter => ({ + dimension: { name: 'label', label_key: `long-${key}` }, + values: [{ kind: 'value', value: (key === 'a' ? 'x' : 'y').repeat(4096) }], match_mode: 'any', + })) }, + } + const requests = observe(page) + const targets: number[] = [] + page.on('request', (request: Request) => { + if (request.isNavigationRequest()) { + const url = new URL(request.url()) + targets.push(Buffer.byteLength(url.pathname + url.search)) + } + }) + expect(Buffer.byteLength(`/analytics?${analyticsViewToSearchParams(view)}`)).toBeGreaterThan(8192) + const report = await openReport(page, view) + expect(report.summary.total_results).toBe(2) + const pending = page.waitForResponse(reportResponse) + await page.reload() + expect((await readReport(await pending)).filters).toEqual(report.filters) + await page.getByLabel('Chart view', { exact: true }).selectOption('success-rate') + await page.goBack() + await expect(page.getByLabel('Chart view', { exact: true })).toHaveValue('outcomes') + await expect(page.getByRole('region', { name: 'Outcome summary' }).getByText('2 total', { exact: true })).toBeVisible() + expect(targets.every((bytes: number) => bytes <= 8192)).toBe(true) + expect(requests.reports).toHaveLength(2) + const oversized = await page.request.get(`/analytics?oversized=${'x'.repeat(8192)}`) + expect(oversized.status()).toBe(414) +}) + +test('recovers invalid links explicitly and migrates valid legacy query links without a duplicate read', async ({ page }: { page: Page }) => { + const requests = observe(page) + await page.goto('/analytics#analytics=invalid-json') + await expect(page.getByText(/Cannot restore this analytics link/)).toBeVisible() + expect(requests.reports).toHaveLength(0) + await expect(page.getByRole('button', { name: 'Reload', exact: true })).toBeDisabled() + await page.getByRole('button', { name: 'Reset analytics view' }).click() + await expect(page.getByRole('region', { name: 'Outcome summary' })).toBeVisible() + expect(requests.reports).toHaveLength(1) + await page.goto(`/analytics?${analyticsViewToSearchParams(DEFAULT_ANALYTICS_VIEW)}`) + await expect(page).toHaveURL(/\/analytics#analytics=/) + await expect(page.getByRole('region', { name: 'Outcome summary' })).toBeVisible() + expect(requests.reports).toHaveLength(2) +}) + +test('shows error-only ASR as unavailable by default rather than zero', async ({ page }: { page: Page }) => { + const report = await openReport(page, { ...DEFAULT_ANALYTICS_VIEW, filters: { + ...DEFAULT_ANALYTICS_VIEW.filters, outcomes: ['error'], + } }) + expect(report.summary).toMatchObject({ errors: 20, total_results: 20, total_decided: 0, success_rate_decided: null }) + const summary = page.getByRole('region', { name: 'Outcome summary' }) + await expect(summary.getByText('Unavailable*', { exact: true })).toBeVisible() + await expect(summary.getByText('0 success / 0 decided', { exact: true })).toBeVisible() +}) + +test('uses exactly matching, contrasting outcome fills in light and dark themes', async ({ page }: { page: Page }) => { + await page.setViewportSize({ width: 1440, height: 1180 }) + await openReport(page, { ...DEFAULT_ANALYTICS_VIEW, filters: { + ...DEFAULT_ANALYTICS_VIEW.filters, dimensions: [OVERVIEW], + } }) + const summary = page.getByRole('region', { name: 'Outcome summary' }) + for (const theme of ['Light', 'Dark']) { + await page.getByRole('button', { name: /^Theme:/ }).click() + await page.getByRole('menuitemradio', { name: theme, exact: true }).click() + await expect(page.locator('html')).toHaveAttribute('data-theme', theme.toLowerCase()) + const fills: Partial> = {} + for (const outcome of ['success', 'failure', 'error', 'undetermined'] as const) { + const badge = summary.getByText(outcome, { exact: true }) + const colors = await badge.evaluate((element: HTMLElement) => ({ + fill: getComputedStyle(element).backgroundColor, + foreground: getComputedStyle(element).color, + icon: getComputedStyle(element.querySelector('svg') ?? element).color, + })) + fills[outcome] = colors.fill + const segment = page.getByRole('button', { name: new RegExp(`^Synthetic baseline: ${outcome} segment;`) }) + const swatch = page.getByRole('button', { name: new RegExp(`^Synthetic baseline: \\d+ ${outcome};`) }) + .locator('[aria-hidden="true"]').first() + await expect(segment).toHaveCSS('background-color', colors.fill) + await expect(swatch).toHaveCSS('background-color', colors.fill) + await expect(page.getByRole('table', { name: 'Saved attack results' }).getByText(outcome, { exact: true }).first()) + .toHaveCSS('background-color', colors.fill) + expect(colors.icon).toBe(colors.foreground) + expect(contrast(colors.foreground, colors.fill)).toBeGreaterThanOrEqual(4.5) + await segment.hover() + await expect(segment).toHaveCSS('background-color', colors.fill) + } + const blue = channels(fills.error ?? '') + expect(blue[2]).toBeGreaterThan(blue[0]) + expect(blue[2]).toBeGreaterThan(blue[1]) + await page.getByRole('heading', { name: 'Analytics', exact: true }).hover() + await capture(page, `analytics-overview-${theme.toLowerCase()}.png`) + } +}) + +test.describe('narrow touch layout', () => { + test.use({ viewport: { width: 390, height: 844 }, hasTouch: true }) + + test('reflows controls and summary without page overflow and retains usable touch targets', async ({ page }: { page: Page }) => { + await openReport(page, { ...DEFAULT_ANALYTICS_VIEW, filters: { + ...DEFAULT_ANALYTICS_VIEW.filters, dimensions: [OVERVIEW], + } }) + await expect(page.getByRole('button', { name: 'More filters', exact: true })).toBeVisible() + await page.getByRole('button', { name: 'More filters', exact: true }).click() + await expect(page.getByRole('button', { name: 'Add filter', exact: true })).toBeVisible() + await page.getByRole('button', { name: 'Fewer filters', exact: true }).click() + for (const name of ['Reload', 'Operation', 'Operator', 'Objective target']) { + const box = await page.getByRole('button', { name, exact: true }).boundingBox() + expect(box?.height).toBeGreaterThanOrEqual(44) + } + expect(await page.evaluate(() => document.documentElement.scrollWidth <= document.documentElement.clientWidth)).toBe(true) + expect(await page.getByTestId('analytics-page').evaluate((element: HTMLElement) => element.scrollWidth <= element.clientWidth)).toBe(true) + await capture(page, 'analytics-narrow.png') + await page.getByRole('region', { name: 'Outcome summary' }).evaluate((element: HTMLElement) => { + element.scrollIntoView({ block: 'start' }) + }) + await capture(page, 'analytics-narrow-data.png') + }) +}) diff --git a/frontend/e2e/fixtures/analytics_backend.py b/frontend/e2e/fixtures/analytics_backend.py new file mode 100644 index 0000000000..27d2d959c4 --- /dev/null +++ b/frontend/e2e/fixtures/analytics_backend.py @@ -0,0 +1,248 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Real analytics routes and bundled UI over disposable, offline SQLite data.""" + +from __future__ import annotations + +import os +import tempfile +import uuid +from argparse import ArgumentParser +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING +from unittest.mock import patch + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from fastapi import FastAPI + + from pyrit.memory import SQLiteMemory + + +async def _seed_async(memory: SQLiteMemory) -> None: + from pyrit.models import ( + AtomicAttackIdentifier, + AttackIdentifier, + AttackOutcome, + AttackResult, + AttackResultMetadata, + AttackResultRole, + Conversation, + ConverterIdentifier, + MessagePiece, + ScenarioIdentifier, + ScenarioResult, + ScenarioRunState, + TargetIdentifier, + ) + + memory.disable_embedding() + start = datetime(2026, 10, 12, 18, tzinfo=UTC) + scenario_id = uuid.UUID(int=9000) + outcomes = [ + "failure", + "success", + "success", + "success", + "failure", + "success", + "error", + "undetermined", + "undetermined", + "undetermined", + ] + harms = ["Privacy", "Misinformation", "Cybersecurity"] + attack_names = ["PromptSendingAttack", "RedTeamingAttack", "CrescendoAttack"] + results: list[AttackResult] = [] + for index in range(204): + configuration = (index // 2) % 3 + target = TargetIdentifier( + class_name="OpenAIChatTarget", + class_module="pyrit.prompt_target", + model_name=f"Synthetic model {chr(65 + configuration)}", + ) + request_converters = ( + [] + if configuration == 2 + else [ + ConverterIdentifier(class_name="Base64Converter", class_module="pyrit.converter"), + ] + ) + if configuration == 1: + request_converters.append(ConverterIdentifier(class_name="ROT13Converter", class_module="pyrit.converter")) + identifier = AtomicAttackIdentifier.build( + attack_identifier=AttackIdentifier( + class_name=attack_names[configuration], + class_module="pyrit.executor.attack", + objective_target=target, + request_converters=request_converters, + response_converters=[] + if configuration != 1 + else [ + ConverterIdentifier(class_name="StringJoinConverter", class_module="pyrit.converter"), + ], + ), + ) + conversation_id = str(uuid.UUID(int=1000 + index // 2)) + if index % 2 == 0: + await memory.add_conversation_to_memory_async( + conversation=Conversation( + conversation_id=conversation_id, + target_identifier=target, + attack_result_id=str(uuid.UUID(int=index + 1)), + ) + ) + await memory.add_message_pieces_to_memory_async( + message_pieces=[ + MessagePiece( + role="user", + conversation_id=conversation_id, + sequence=0, + original_value=f"Synthetic evaluation example {index // 2 + 1}. No external model was called.", + ), + MessagePiece( + role="assistant", + conversation_id=conversation_id, + sequence=1, + original_value="This is a safe, locally stored example response for analytics browser checks.", + ), + ] + ) + operation = ( + "Synthetic baseline" + if index < 80 + else "Synthetic pilot" + if index < 120 + else "Synthetic follow-up" + if index < 144 + else f"Synthetic batch {index - 144:02}" + ) + role = (AttackResultRole.TARGET_FACING, AttackResultRole.ORCHESTRATION, AttackResultRole.UNKNOWN)[index % 3] + results.append( + AttackResult( + attack_result_id=str(uuid.UUID(int=index + 1)), + conversation_id=conversation_id, + objective=f"Synthetic {harms[configuration].lower()} check {index // 2 + 1}", + outcome=AttackOutcome(outcomes[index % len(outcomes)]), + atomic_attack_identifier=None if index % 19 == 18 else identifier, + operator=("Alex (synthetic)", "Sam (synthetic)", "Jordan (synthetic)")[configuration], + operation=operation, + timestamp=start + timedelta(minutes=index), + targeted_harm_categories=[harms[configuration]] + + ([harms[(configuration + 1) % 3]] if index % 5 == 0 else []), + labels={ + "source": "analytics-browser-fixture", + "team": ("research", "assurance")[index % 2], + "cohort": "overview" if index < 144 else "facet-pagination", + **({"long-a": "x" * 4096, "long-b": "y" * 4096} if index < 2 else {}), + }, + attribution_parent_id=str(scenario_id) if index < 2 else None, + attribution_data=None + if role is AttackResultRole.UNKNOWN + else AttackResultMetadata( + result_role=role, + attempt_index=index % 2, + ).to_metadata(), + ) + ) + await memory.add_attack_results_to_memory_async(attack_results=results) + await memory.add_scenario_results_to_memory_async( + scenario_results=[ + ScenarioResult( + id=scenario_id, + scenario_identifier=ScenarioIdentifier( + class_name="SyntheticRetryScenario", + class_module="analytics_browser_fixture", + version=1, + objective_target=TargetIdentifier( + class_name="OpenAIChatTarget", + class_module="pyrit.prompt_target", + model_name="Synthetic model A", + ), + ), + scenario_description="Synthetic failed attempt followed by success, sharing one conversation.", + attack_results={"PromptSendingAttack": results[:2]}, + scenario_run_state=ScenarioRunState.COMPLETED, + creation_time=start, + completion_time=start + timedelta(minutes=1), + ) + ] + ) + + +def _serve(*, directory: Path, port: int) -> None: + import uvicorn + + from pyrit.common import path as paths + + data = directory / "dbdata" + data.mkdir() + # Source checkouts otherwise default to /dbdata, regardless of HOME. + with patch.object(paths, "DB_DATA_PATH", data), patch.object(paths, "PYRIT_CACHE_PATH", directory): + from pyrit.backend.main import SPAStaticFiles, app, lifespan + from pyrit.memory import CentralMemory, SQLiteMemory + + frontend = Path(__file__).resolve().parents[2] / "dist" + if not (frontend / "index.html").is_file(): + raise RuntimeError("Build the frontend before starting the analytics browser fixture.") + + @asynccontextmanager + async def fixture_lifespan_async(application: FastAPI) -> AsyncGenerator[None, None]: + async with lifespan(application): + if application.state.runtime_lifecycle.state != "ready": + raise RuntimeError(application.state.runtime_lifecycle.message) + memory = CentralMemory.get_memory_instance() + if not isinstance(memory, SQLiteMemory) or not Path(memory.db_path).resolve().is_relative_to(directory): + raise RuntimeError("Refusing to seed anything except the fixture-owned SQLite database.") + await _seed_async(memory) + application.mount("/", SPAStaticFiles(directory=str(frontend), html=True), name="analytics-browser-ui") + print(f"Analytics browser fixture: {memory.db_path}", flush=True) + yield + + app.router.lifespan_context = fixture_lifespan_async + uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning") + + +def main() -> None: + """Serve only fixture-owned data with user configuration discovery disabled.""" + parser = ArgumentParser(description=__doc__) + parser.add_argument("--port", type=int, default=18496) + args = parser.parse_args() + root = Path(__file__).resolve().parents[3] + scratch = root / ".pyrit_cache" + scratch.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="analytics-e2e-", dir=scratch) as temporary: + directory = Path(temporary).resolve() + config = directory / "config.yaml" + config.write_text( + "memory_db_type: sqlite\nenv_files: []\nenv_akv_ref: []\ninitializers: []\n" + "initialization_scripts: []\nsilent: true\noperator: synthetic\noperation: synthetic\n", + encoding="utf-8", + ) + environment = { + key: value + for key, value in os.environ.items() + if key.upper() in {"PATH", "SYSTEMROOT", "WINDIR", "TEMP", "TMP", "PYTHONIOENCODING"} + } + environment.update( + { + "HOME": str(directory), + "USERPROFILE": str(directory), + "APPDATA": str(directory), + "LOCALAPPDATA": str(directory), + "XDG_CONFIG_HOME": str(directory), + "XDG_DATA_HOME": str(directory), + "PYRIT_CONFIG_FILE": str(config), + "PYRIT_DEV_MODE": "true", + } + ) + with patch.dict(os.environ, environment, clear=True): + _serve(directory=directory, port=args.port) + + +if __name__ == "__main__": + main() diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts index 315b8a0738..6d30d22f52 100644 --- a/frontend/e2e/scenario-history.spec.ts +++ b/frontend/e2e/scenario-history.spec.ts @@ -539,12 +539,13 @@ test.describe("Scenario catalog, history, and live run routing", () => { const primaryNavigation = page.getByRole("navigation", { name: "Primary" }); const primaryButtons = primaryNavigation.getByRole("button"); await expect(primaryNavigation.getByRole("button", { name: "Configuration" })).toBeVisible(); - await expect(primaryButtons).toHaveCount(7); + await expect(primaryButtons).toHaveCount(8); expect(await primaryButtons.evaluateAll((buttons) => buttons.map((button) => button.getAttribute("aria-label")))).toEqual([ "Home", "Technique.execute", "History", + "Analytics", "Scanner", "Datasets", "Registry", diff --git a/frontend/package.json b/frontend/package.json index 7a91079efa..07dc109836 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,6 +16,7 @@ "test:watch": "jest --watch", "test:coverage": "jest --coverage", "test:e2e": "playwright test", + "test:e2e:analytics": "npm run build && playwright test --config playwright.analytics.config.ts", "test:e2e:seeded": "playwright test --project seeded", "test:e2e:live": "E2E_LIVE_MODE=true playwright test --project live", "test:e2e:ui": "playwright test --ui", diff --git a/frontend/playwright.analytics.config.ts b/frontend/playwright.analytics.config.ts new file mode 100644 index 0000000000..037b6c5aa6 --- /dev/null +++ b/frontend/playwright.analytics.config.ts @@ -0,0 +1,39 @@ +import process from 'node:process' + +import { defineConfig, devices } from '@playwright/test' + +import base from './playwright.config' + +const port = Number(process.env.PYRIT_E2E_ANALYTICS_PORT ?? '18496') +if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error('PYRIT_E2E_ANALYTICS_PORT must be a valid TCP port.') +} +const url = `http://127.0.0.1:${port}` + +export default defineConfig({ + ...base, + testMatch: 'analytics.spec.ts', + testIgnore: [], + fullyParallel: false, + workers: 1, + timeout: 60_000, + use: { + ...base.use, + baseURL: url, + storageState: { + cookies: [], + origins: [{ origin: url, localStorage: [{ name: 'pyrit-tour-completed', value: 'true' }] }], + }, + }, + projects: [{ + name: 'analytics', + use: { ...devices['Desktop Chrome'], viewport: { width: 1440, height: 1050 } }, + }], + webServer: { + command: `cd .. && uv run --no-sync python -m frontend.e2e.fixtures.analytics_backend --port ${port}`, + env: { PYTHONIOENCODING: 'utf-8' }, + url: `${url}/api/health`, + reuseExistingServer: false, + timeout: 120_000, + }, +}) diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index e2e32cb65a..640cc3df0d 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -22,6 +22,7 @@ const E2E_BACKEND_URL = `http://127.0.0.1:${E2E_BACKEND_PORT}`; export default defineConfig({ testDir: "./e2e", + testIgnore: "**/analytics.spec.ts", fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index e856f9e0db..48fbe303a8 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -12,6 +12,7 @@ import App from "./App"; import { attacksApi, targetsApi } from "./services/api"; import { makeTarget } from "./test-utils/targetFixtures"; import { DEFAULT_USER_PREFERENCES, readUserPreferences, writeUserPreferences } from "./utils/userPreferences"; +import { DEFAULT_ANALYTICS_VIEW, analyticsViewToHash } from "./utils/attackAnalytics"; const mockGetActiveAccount = jest.fn(); let mockCreatedAttackRender: Promise | null = null; @@ -157,6 +158,7 @@ jest.mock("./components/Layout/MainLayout", () => { + @@ -302,6 +304,21 @@ jest.mock("./components/Configuration/Configuration", () => ({ default: () =>
Configuration
, })); +jest.mock("./components/Analytics/AnalyticsPage", () => { + const { useLocation } = jest.requireActual("react-router"); + function MockAnalyticsPage({ onOpenAttack }: { onOpenAttack: (id: string) => void }) { + const location = useLocation(); + return
+ {location.pathname}{location.search}{location.hash} + +
; + } + return { + __esModule: true, + default: MockAnalyticsPage, + }; +}); + jest.mock("./components/History/AttackHistory", () => { const MockAttackHistory = ({ onOpenAttack, @@ -514,6 +531,26 @@ describe("App", () => { window.localStorage.clear(); }); + it("opens Analytics and restores its fragment after inspecting a saved attack", async () => { + const user = userEvent.setup(); + const originalPath = `/analytics${analyticsViewToHash({ + ...DEFAULT_ANALYTICS_VIEW, + filters: { ...DEFAULT_ANALYTICS_VIEW.filters, outcomes: ["failure"] }, + })}`; + mockGetAttack.mockResolvedValue({ + attack_result_id: "analytics-result", conversation_id: "synthetic-conversation", + objective: "Safe saved check", outcome: "failure", labels: {}, related_conversation_ids: [], + }); + renderApp(originalPath); + expect(screen.getByTestId("main-layout")).toHaveAttribute("data-current-view", "analytics"); + expect(screen.getByLabelText("Analytics URL")).toHaveTextContent(originalPath); + await user.click(screen.getByRole("button", { name: "Open analytics result" })); + await screen.findByTestId("chat-window"); + await waitFor(() => expect(mockGetAttack).toHaveBeenCalledWith("analytics-result")); + await user.click(screen.getByRole("button", { name: "Analytics", exact: true })); + expect(screen.getByLabelText("Analytics URL")).toHaveTextContent(originalPath); + }); + it("renders with FluentProvider and MainLayout", () => { renderApp(); expect(screen.getByTestId("main-layout")).toBeInTheDocument(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index ccd71ceb99..bc1fea1e7e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -16,6 +16,7 @@ import TechniqueRegistry from './components/Registry/TechniqueRegistry' import RegistryLayout from './components/Registry/RegistryLayout' import Configuration from './components/Configuration/Configuration' import AttackHistory from './components/History/AttackHistory' +import AnalyticsPage from './components/Analytics/AnalyticsPage' import HistoryPage from './components/History/HistoryPage' import type { HistoryTab } from './components/History/HistoryPage' import ScenarioHistory from './components/History/ScenarioHistory' @@ -72,6 +73,7 @@ const VIEW_PATHS: Record = { home: '/', chat: '/chat', history: HISTORY_ATTACKS_PATH, + analytics: '/analytics', registry: '/registry/targets', scenarios: '/scanner', datasets: DATASETS_PATH, @@ -277,6 +279,7 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { ) const lastHistorySearch = useRef('') const lastScenarioHistorySearch = useRef('') + const lastAnalyticsLocation = useRef('') useEffect(() => { if (location.pathname === HISTORY_ATTACKS_PATH) { lastHistorySearch.current = location.search @@ -284,7 +287,10 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { if (location.pathname === HISTORY_SCANNER_PATH) { lastScenarioHistorySearch.current = location.search } - }, [location.pathname, location.search]) + if (location.pathname === VIEW_PATHS.analytics) { + lastAnalyticsLocation.current = location.search + location.hash + } + }, [location.pathname, location.search, location.hash]) const handleFiltersChange = useCallback((filters: HistoryFilters) => { setSearchParams(filtersToSearchParams(filters), { replace: true }) @@ -488,6 +494,10 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { navigate(VIEW_PATHS.history + lastHistorySearch.current) return } + if (view === 'analytics') { + navigate(VIEW_PATHS.analytics + lastAnalyticsLocation.current) + return + } navigate(VIEW_PATHS[view]) }, [navigate]) @@ -774,6 +784,7 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { } /> } /> } /> + } /> } /> void +} + +/** + * Built-in dimensions commit immediately; a custom label needs an explicit valid + * key before lookup. null means an uncommitted draft: filter editors hide facets, + * while chart controls retain their last valid axis until Use label is chosen. + * Callers key this control by committed dimension to restore drafts on URL changes. + */ +export default function AnalyticsDimensionPicker({ label, dimension, onChange }: AnalyticsDimensionPickerProps) { + const styles = useAnalyticsDimensionPickerStyles() + const [labelKey, setLabelKey] = useState(dimension?.name === 'label' ? dimension.label_key : '') + const [choosingLabel, setChoosingLabel] = useState(dimension?.name === 'label' || dimension === null) + const selected = choosingLabel ? 'label' + : dimension?.name === 'converter_type' ? `${dimension.converter_direction ?? 'request'}_converters` + : dimension?.name ?? 'label' + + function selectDimension(_event: React.ChangeEvent, data: SelectOnChangeData): void { + setChoosingLabel(data.value === 'label') + if (data.value === 'label') { + onChange(null) + return + } + const option = ANALYTICS_DIMENSIONS.find((item: typeof ANALYTICS_DIMENSIONS[number]) => item.value === data.value) + if (option) onChange(option.dimension) + } + + return ( +
+ + + + {choosingLabel && ( + <> + + , data: InputOnChangeData) => { + setLabelKey(data.value) + onChange(null) + }} + onKeyDown={(event: React.KeyboardEvent) => { + if (event.key === 'Enter' && isAnalyticsLabelKey(labelKey)) onChange({ name: 'label', label_key: labelKey }) + }} + /> + + + + )} +
+ ) +} diff --git a/frontend/src/components/Analytics/AnalyticsFilters.styles.ts b/frontend/src/components/Analytics/AnalyticsFilters.styles.ts new file mode 100644 index 0000000000..34863cfc91 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsFilters.styles.ts @@ -0,0 +1,96 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget, mobileTouchTargetHeight, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' + +export const useAnalyticsFiltersStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalS, + }, + row: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + }, + secondaryControls: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + }, + collapsed: { [NARROW_VIEWPORT_QUERY]: { display: 'none' } }, + disclosure: { + display: 'none', + [NARROW_VIEWPORT_QUERY]: { display: 'inline-flex' }, + ...mobileTouchTargetHeight, + }, + button: { ...mobileTouchTargetHeight }, + iconButton: { ...mobileTouchTarget }, + chip: { + display: 'inline-flex', + alignItems: 'center', + maxWidth: '100%', + border: `1px solid ${tokens.colorNeutralStroke1}`, + borderRadius: tokens.borderRadiusMedium, + backgroundColor: tokens.colorNeutralBackground1, + }, + chipLabel: { + overflowWrap: 'anywhere', + textAlign: 'start', + minWidth: 0, + ...mobileTouchTargetHeight, + }, + chipText: { + minWidth: 0, + maxWidth: '32rem', + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + }, + editor: { + backgroundColor: tokens.colorNeutralBackground1, + border: `1px solid ${tokens.colorNeutralStroke1}`, + borderRadius: tokens.borderRadiusMedium, + padding: tokens.spacingHorizontalL, + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + minWidth: 0, + [NARROW_VIEWPORT_QUERY]: { padding: tokens.spacingHorizontalS }, + }, + editorHeading: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + gap: tokens.spacingHorizontalM, + }, + facet: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalS, + maxWidth: '42rem', + }, + options: { + display: 'flex', + flexDirection: 'column', + overflowY: 'auto', + maxHeight: '16rem', + overflowWrap: 'anywhere', + }, + checkbox: { ...mobileTouchTargetHeight }, + field: { + minWidth: 0, + flex: 1, + maxWidth: '22rem', + }, + input: { ...mobileTouchTargetHeight }, + popover: { + display: 'flex', + flexDirection: 'column', + maxWidth: 'min(24rem, 80vw)', + gap: tokens.spacingVerticalS, + }, + note: { color: tokens.colorNeutralForeground2 }, +}) diff --git a/frontend/src/components/Analytics/AnalyticsFilters.test.tsx b/frontend/src/components/Analytics/AnalyticsFilters.test.tsx new file mode 100644 index 0000000000..52d9659752 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsFilters.test.tsx @@ -0,0 +1,221 @@ +import { useState } from 'react' + +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { act, render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { analyticsApi } from '@/services/api' +import { ANALYTICS_OPERATION_FILTER, makeAnalyticsFacets } from '@/test-utils/analyticsFixtures' +import type { AttackAnalyticsFacets, AttackAnalyticsFilters } from '@/types' + +import AnalyticsFilters from './AnalyticsFilters' + +jest.mock('@/services/api', () => ({ + analyticsApi: { facets: jest.fn() }, +})) +const mockFacets = jest.mocked(analyticsApi.facets) + +const TestWrapper: React.FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +) + +interface FiltersHarnessProps { + readonly initialFilters?: AttackAnalyticsFilters +} + +function FiltersHarness({ initialFilters = { dimensions: [], outcomes: [] } }: FiltersHarnessProps) { + const [filters, setFilters] = useState(initialFilters) + const [refreshVersion, setRefreshVersion] = useState(0) + return <> + { setFilters(next); return true }} /> + {JSON.stringify(filters)} + + +} + +describe('AnalyticsFilters', () => { + beforeEach(() => { + jest.clearAllMocks() + mockFacets.mockReset().mockResolvedValue(makeAnalyticsFacets()) + }) + afterEach(() => { jest.useRealTimers() }) + + it('should request only an opened facet, refresh it on Reload, and invalidate closed options', async () => { + const user = userEvent.setup() + render() + expect(mockFacets).not.toHaveBeenCalled() + await user.click(screen.getByRole('button', { name: 'Operation' })) + await screen.findByRole('checkbox', { name: 'Nightly' }) + expect(mockFacets).toHaveBeenCalledTimes(1) + expect(mockFacets).toHaveBeenLastCalledWith(expect.objectContaining({ dimension: { name: 'operation' }, search: '', offset: 0, limit: 50 }), expect.any(AbortSignal)) + await user.click(screen.getByRole('button', { name: 'Reload report' })) + await waitFor(() => { expect(mockFacets).toHaveBeenCalledTimes(2) }) + await user.click(screen.getByRole('button', { name: 'Close filter editor' })) + await user.click(screen.getByRole('button', { name: 'Reload report' })) + expect(mockFacets).toHaveBeenCalledTimes(2) + await user.click(screen.getByRole('button', { name: 'Operation' })) + await waitFor(() => { expect(mockFacets).toHaveBeenCalledTimes(3) }) + }) + + it('should debounce searches and retain typed selected values when no alternatives match', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + render() + await user.click(screen.getByRole('button', { name: 'Operation' })) + await act(async () => { await jest.advanceTimersByTimeAsync(250) }) + await user.click(screen.getByRole('checkbox', { name: 'Unknown' })) + await user.click(screen.getByRole('checkbox', { name: 'Unknown (missing metadata)' })) + mockFacets.mockResolvedValue(makeAnalyticsFacets({ items: [] })) + await user.type(screen.getByLabelText('Search values'), 'does-not-match') + expect(mockFacets).toHaveBeenCalledTimes(1) + await act(async () => { await jest.advanceTimersByTimeAsync(250) }) + expect(mockFacets).toHaveBeenCalledTimes(2) + expect(mockFacets).toHaveBeenLastCalledWith(expect.objectContaining({ search: 'does-not-match', offset: 0 }), expect.any(AbortSignal)) + const selected = screen.getByRole('group', { name: 'Selected filter values' }) + expect(within(selected).getByRole('checkbox', { name: 'Unknown' })).toBeChecked() + expect(within(selected).getByRole('checkbox', { name: 'Unknown (missing metadata)' })).toBeChecked() + await user.click(screen.getByRole('button', { name: 'Apply filter' })) + expect(screen.getByLabelText('Applied filters')).toHaveTextContent('"kind":"missing","value":null') + expect(screen.getByLabelText('Applied filters')).toHaveTextContent('"kind":"value","value":"Unknown"') + }) + + it('should show a fresh read when returning to an earlier facet search', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + let resolveOther: ((value: AttackAnalyticsFacets) => void) | undefined + mockFacets.mockResolvedValueOnce(makeAnalyticsFacets()) + .mockReturnValueOnce(new Promise((resolve: (value: AttackAnalyticsFacets) => void) => { resolveOther = resolve })) + .mockResolvedValueOnce(makeAnalyticsFacets({ items: [] })) + render() + await user.click(screen.getByRole('button', { name: 'Operation' })) + await act(async () => { await jest.advanceTimersByTimeAsync(250) }) + await user.type(screen.getByLabelText('Search values'), 'other') + await act(async () => { await jest.advanceTimersByTimeAsync(250) }) + await user.clear(screen.getByLabelText('Search values')) + expect(screen.getByRole('progressbar', { name: 'Loading filter values' })).toBeInTheDocument() + expect(screen.queryByRole('checkbox', { name: 'Nightly' })).not.toBeInTheDocument() + await act(async () => { + await jest.advanceTimersByTimeAsync(250) + resolveOther?.(makeAnalyticsFacets()) + }) + expect(screen.getByText(/No values match this search/)).toBeInTheDocument() + expect(mockFacets).toHaveBeenCalledTimes(3) + }) + + it('should require a valid explicit custom label key before looking up values', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + render() + await user.click(screen.getByRole('button', { name: 'Add filter' })) + await user.selectOptions(screen.getByLabelText('Filter dimension'), 'label') + await user.type(screen.getByLabelText('Filter dimension label key'), 'team.region') + await act(async () => { await jest.advanceTimersByTimeAsync(500) }) + expect(mockFacets).not.toHaveBeenCalled() + await user.click(screen.getByRole('button', { name: 'Use label' })) + await act(async () => { await jest.advanceTimersByTimeAsync(250) }) + expect(mockFacets).toHaveBeenCalledTimes(1) + expect(mockFacets).toHaveBeenCalledWith(expect.objectContaining({ dimension: { name: 'label', label_key: 'team.region' } }), expect.any(AbortSignal)) + }) + + it('should support response converters, ALL matching, known-empty pipelines and unknown metadata', async () => { + const user = userEvent.setup() + mockFacets.mockResolvedValue(makeAnalyticsFacets({ items: [ + { key: { kind: 'value', value: 'ConverterA' }, label: 'ConverterA' }, + { key: { kind: 'no_converters', value: null }, label: 'No converters' }, + { key: { kind: 'missing', value: null }, label: 'Unknown' }, + ] })) + render() + await user.click(screen.getByRole('button', { name: 'Add filter' })) + await user.selectOptions(screen.getByLabelText('Filter dimension'), 'response_converters') + await user.selectOptions(screen.getByLabelText('Converter matching'), 'all') + expect(screen.getByText(/All selected converters must match/)).toBeInTheDocument() + await user.click(await screen.findByRole('checkbox', { name: 'ConverterA' })) + await user.click(screen.getByRole('checkbox', { name: 'Unknown (missing metadata)' })) + expect(screen.getByRole('checkbox', { name: 'No converters (empty pipeline)' })).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Apply filter' })) + expect(screen.getByLabelText('Applied filters')).toHaveTextContent('"converter_direction":"response"') + expect(screen.getByLabelText('Applied filters')).toHaveTextContent('"match_mode":"all"') + const chip = screen.getByRole('button', { name: /^Response converter \(ALL\)/ }) + await user.click(chip) + expect(screen.getByLabelText('Converter matching')).toHaveValue('all') + expect(within(screen.getByRole('group', { name: 'Selected filter values' })).getByRole('checkbox', { name: 'ConverterA' })).toBeChecked() + }) + + it('should send all predicates for server-side facet exclusion and edit only the selected repeated dimension', async () => { + const user = userEvent.setup() + const filters: AttackAnalyticsFilters = { + dimensions: [ + ANALYTICS_OPERATION_FILTER, + { ...ANALYTICS_OPERATION_FILTER, values: [{ kind: 'value', value: 'Other operation' }] }, + ], + outcomes: ['failure'], + updated_after: '2026-01-01T00:00:00Z', + } + render() + await user.click(screen.getByRole('button', { name: 'Operation (ANY): Nightly' })) + await user.click(await screen.findByRole('checkbox', { name: 'Unknown' })) + expect(mockFacets).toHaveBeenCalledWith( + expect.objectContaining({ dimension: { name: 'operation' }, filters }), + expect.any(AbortSignal), + ) + await user.click(screen.getByRole('button', { name: 'Apply filter' })) + expect(screen.getByLabelText('Applied filters')).toHaveTextContent(JSON.stringify({ + ...filters, + dimensions: [ + { ...ANALYTICS_OPERATION_FILTER, values: [...ANALYTICS_OPERATION_FILTER.values, { kind: 'value', value: 'Unknown' }] }, + filters.dimensions[1], + ], + })) + }) + + it('should preserve untouched timestamp precision and time zone, and clear only an edited bound', async () => { + const user = userEvent.setup() + const filters: AttackAnalyticsFilters = { + dimensions: [], outcomes: [], + updated_after: '2026-01-01T08:00:05.123456-07:00', + updated_before: '2026-02-01T08:00:05.123456-07:00', + } + render() + await user.click(screen.getByRole('button', { name: 'Last updated' })) + await user.click(screen.getByRole('button', { name: 'Apply range' })) + expect(screen.getByLabelText('Applied filters')).toHaveTextContent(JSON.stringify(filters)) + await user.click(screen.getByRole('button', { name: 'Last updated' })) + await user.clear(screen.getByLabelText('Last updated after')) + await user.click(screen.getByRole('button', { name: 'Apply range' })) + expect(screen.getByLabelText('Applied filters')).toHaveTextContent(JSON.stringify({ ...filters, updated_after: null })) + expect(mockFacets).not.toHaveBeenCalled() + }) + + it('should page one facet with server offsets and surface errors without discarding selections', async () => { + const user = userEvent.setup() + mockFacets.mockResolvedValueOnce(makeAnalyticsFacets({ has_more: true, next_offset: 17 })) + .mockRejectedValueOnce(new Error('Facet read timed out')) + .mockResolvedValueOnce(makeAnalyticsFacets({ items: [] })) + render() + await user.click(screen.getByRole('button', { name: 'Operation' })) + await user.click(await screen.findByRole('checkbox', { name: 'Nightly' })) + await user.click(screen.getByRole('button', { name: 'Next values' })) + expect(await screen.findByText(/Could not load filter values.*Facet read timed out/)).toBeInTheDocument() + expect(mockFacets).toHaveBeenLastCalledWith(expect.objectContaining({ offset: 17 }), expect.any(AbortSignal)) + expect(screen.getByRole('checkbox', { name: 'Nightly' })).toBeChecked() + await user.click(screen.getByRole('button', { name: 'Retry values' })) + await screen.findByText(/No values match this search/) + expect(mockFacets).toHaveBeenCalledTimes(3) + }) + + it('should cancel a closed facet and ignore its late response in another dimension', async () => { + const user = userEvent.setup() + let resolveFirst: ((value: AttackAnalyticsFacets) => void) | undefined + mockFacets.mockReturnValueOnce(new Promise((resolve: (value: AttackAnalyticsFacets) => void) => { resolveFirst = resolve })) + .mockResolvedValueOnce(makeAnalyticsFacets({ items: [{ key: { kind: 'value', value: 'Alice' }, label: 'Alice' }] })) + render() + await user.click(screen.getByRole('button', { name: 'Operation' })) + await waitFor(() => { expect(mockFacets).toHaveBeenCalledTimes(1) }) + const signal = mockFacets.mock.calls[0][1] + await user.click(screen.getByRole('button', { name: 'Operator' })) + await screen.findByRole('checkbox', { name: 'Alice' }) + expect(signal?.aborted).toBe(true) + await act(async () => { resolveFirst?.(makeAnalyticsFacets()) }) + expect(screen.queryByRole('checkbox', { name: 'Nightly' })).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/Analytics/AnalyticsFilters.tsx b/frontend/src/components/Analytics/AnalyticsFilters.tsx new file mode 100644 index 0000000000..df448ce19d --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsFilters.tsx @@ -0,0 +1,393 @@ +import { useId, useState } from 'react' + +import { + Button, Checkbox, Field, Input, MessageBar, MessageBarBody, + Popover, PopoverSurface, PopoverTrigger, Select, Spinner, Text, mergeClasses, +} from '@fluentui/react-components' +import type { InputOnChangeData, SelectOnChangeData } from '@fluentui/react-components' +import { AddRegular, DismissRegular, FilterDismissRegular } from '@fluentui/react-icons' + +import { useAttackAnalyticsFacet } from '@/hooks/useAttackAnalyticsFacet' +import type { + AttackAnalyticsDimension, AttackAnalyticsFilter, AttackAnalyticsFilters, + AttackAnalyticsOption, AttackAnalyticsValue, AttackOutcome, +} from '@/types' +import { + ANALYTICS_MAX_VALUES, ANALYTICS_OUTCOMES, analyticsDateRangeError, analyticsDimensionKey, analyticsDimensionLabel, + analyticsOptionLabel, analyticsValueKey, analyticsValueLabel, formatAnalyticsTime, hasAnalyticsFilters, +} from '@/utils/attackAnalytics' + +import AnalyticsDimensionPicker from './AnalyticsDimensionPicker' +import { useAnalyticsFiltersStyles } from './AnalyticsFilters.styles' + +interface AnalyticsFacetPickerProps { + readonly dimension: AttackAnalyticsDimension + readonly filters: AttackAnalyticsFilters + readonly initial: AttackAnalyticsFilter | null + readonly refreshVersion: number + readonly knownLabels: Record + readonly onApply: (filter: AttackAnalyticsFilter, options: AttackAnalyticsOption[]) => void +} + +/** Friendly labels are cached per typed dimension/value, never used to identify or merge predicates. */ +function optionLabelKey(dimension: AttackAnalyticsDimension, value: AttackAnalyticsValue): string { + return JSON.stringify([analyticsDimensionKey(dimension), analyticsValueKey(value)]) +} + +/** Keep draft selections across facet searches/pages; only Apply changes the dashboard cohort. */ +function AnalyticsFacetPicker({ + dimension, filters, initial, refreshVersion, knownLabels, onApply, +}: AnalyticsFacetPickerProps) { + const styles = useAnalyticsFiltersStyles() + const facet = useAttackAnalyticsFacet(dimension, filters, refreshVersion) + const [selected, setSelected] = useState(() => + initial?.values.map((value: AttackAnalyticsValue) => ({ + key: value, + label: knownLabels[optionLabelKey(dimension, value)] ?? + (value.kind === 'missing' ? 'Unknown' : value.kind === 'no_converters' ? 'No converters' : analyticsValueLabel(value)), + })) ?? [], + ) + const [matchMode, setMatchMode] = useState(initial?.match_mode ?? 'any') + const tooMany = selected.length > ANALYTICS_MAX_VALUES + const selectedKeys = new Set(selected.map((option: AttackAnalyticsOption) => analyticsValueKey(option.key))) + + function toggle(option: AttackAnalyticsOption): void { + const key = analyticsValueKey(option.key) + setSelected((previous: AttackAnalyticsOption[]) => { + const remaining = previous.filter((item: AttackAnalyticsOption) => analyticsValueKey(item.key) !== key) + return remaining.length === previous.length ? [...previous, option] : remaining + }) + } + + return ( +
+ {dimension.name === 'converter_type' && ( + + + + )} + + , data: InputOnChangeData) => { facet.setSearch(data.value) }} + /> + + {selected.length > 0 && ( +
+ {selected.map((option: AttackAnalyticsOption) => ( + { toggle(option) }} + /> + ))} +
+ )} + {tooMany && Select at most 100 values in one filter.} + {!facet.ready ? Filter values will load when the runtime is ready. Selections are preserved. + : facet.loading ? : facet.error ? ( + + + Could not load filter values. {facet.error}{' '} + + + + ) : ( + <> +
+ {facet.items + .filter((option: AttackAnalyticsOption) => !selectedKeys.has(analyticsValueKey(option.key))) + .map((option: AttackAnalyticsOption) => ( + { toggle(option) }} + /> + ))} +
+ {facet.items.length === 0 && No values match this search. Selected values are retained.} +
+ + +
+ + )} + + {matchMode === 'all' ? 'All selected converters must match.' : 'Selected values are alternatives within this filter.'} + {' '}Separate filter chips are combined with AND. + +
+ +
+
+ ) +} + +interface AnalyticsFilterEditorProps { + readonly initial: AttackAnalyticsFilter | null + readonly startingDimension: AttackAnalyticsDimension | null + readonly filters: AttackAnalyticsFilters + readonly refreshVersion: number + readonly knownLabels: Record + readonly onApply: (filter: AttackAnalyticsFilter, options: AttackAnalyticsOption[]) => void + readonly onClose: () => void +} + +/** Changing dimension remounts the picker so old selections cannot be applied to a different metadata field. */ +function AnalyticsFilterEditor({ + initial, startingDimension, filters, refreshVersion, knownLabels, onApply, onClose, +}: AnalyticsFilterEditorProps) { + const styles = useAnalyticsFiltersStyles() + const [dimension, setDimension] = useState(startingDimension) + const matchingInitial = initial && dimension && + analyticsDimensionKey(initial.dimension) === analyticsDimensionKey(dimension) ? initial : null + return ( +
+
+ {initial ? 'Edit filter' : 'Add filter'} +
+ + {dimension ? ( + + ) : Enter a custom label key and choose Use label to look up its values.} +
+ ) +} + +interface AnalyticsDateFilterProps { + readonly filters: AttackAnalyticsFilters + readonly onChange: (filters: AttackAnalyticsFilters) => boolean + readonly onClose: () => void +} + +function AnalyticsDateFilter({ filters, onChange, onClose }: AnalyticsDateFilterProps) { + const styles = useAnalyticsFiltersStyles() + const [after, setAfter] = useState(filters.updated_after ?? '') + const [before, setBefore] = useState(filters.updated_before ?? '') + const [error, setError] = useState(null) + + function apply(event: React.FormEvent): void { + event.preventDefault() + const rangeError = analyticsDateRangeError(after || null, before || null) + if (rangeError) { + setError(rangeError) + return + } + if (onChange({ + ...filters, + updated_after: after || null, + updated_before: before || null, + })) onClose() + } + + return ( +
+
+ + , data: InputOnChangeData) => { setAfter(data.value) }} /> + + + , data: InputOnChangeData) => { setBefore(data.value) }} /> + +
+ + Use ISO 8601 with Z (UTC) or an explicit offset, such as 2026-11-01T01:30:00-07:00. + After is inclusive; before is exclusive. This filters last updates, not attack execution time. + Explicit offsets distinguish repeated local times during daylight saving changes. + + {error && {error}} +
+ + +
+
+ ) +} + +interface AnalyticsFiltersProps { + readonly filters: AttackAnalyticsFilters + readonly refreshVersion: number + readonly onChange: (filters: AttackAnalyticsFilters) => boolean +} + +interface FilterEditorState { + readonly dimension: AttackAnalyticsDimension | null + readonly index: number | null + readonly initial: AttackAnalyticsFilter | null +} + +/** + * Edits whole-cohort predicates while keeping drafts and fetched labels local. + * Existing chips edit one predicate; Add filter can append another on the same + * dimension. The boolean onChange result keeps rejected edits open for correction. + */ +export default function AnalyticsFilters({ filters, refreshVersion, onChange }: AnalyticsFiltersProps) { + const styles = useAnalyticsFiltersStyles() + const [editor, setEditor] = useState(null) + const [datesOpen, setDatesOpen] = useState(false) + const [knownLabels, setKnownLabels] = useState>({}) + const [expanded, setExpanded] = useState(false) + const moreFiltersId = useId() + // Browser Back can replace/remove a chip while its editor is open. Never apply + // that stale draft to whichever predicate now occupies its former array index. + const editorIsCurrent = editor?.index == null || + JSON.stringify(filters.dimensions[editor.index]) === JSON.stringify(editor.initial) + const outcomesLabel = filters.outcomes.length === 0 || filters.outcomes.length === 4 + ? 'All' : filters.outcomes.join(', ') + // Identical AND predicates are valid. Add an occurrence suffix for unique React + // keys rather than deduplicating filters or identifying all chips by dimension. + const occurrences = new Map() + + function openDimension(dimension: AttackAnalyticsDimension): void { + const index = filters.dimensions.findIndex( + (predicate: AttackAnalyticsFilter) => analyticsDimensionKey(predicate.dimension) === analyticsDimensionKey(dimension), + ) + setDatesOpen(false) + setEditor({ dimension, index: index < 0 ? null : index, initial: filters.dimensions[index] ?? null }) + } + + function apply(filter: AttackAnalyticsFilter, options: AttackAnalyticsOption[]): void { + const dimensions = editor?.index == null + ? [...filters.dimensions, filter] + : filters.dimensions.map((predicate: AttackAnalyticsFilter, index: number) => index === editor.index ? filter : predicate) + if (!onChange({ ...filters, dimensions })) return + setKnownLabels((previous: Record) => ({ + ...previous, + ...Object.fromEntries(options.map((option: AttackAnalyticsOption) => [ + optionLabelKey(filter.dimension, option.key), option.label, + ])), + })) + setEditor(null) + } + + return ( +
+
+ + + + +
+ + + + + + Select outcomes for the entire dashboard. No selection includes all outcomes. + {ANALYTICS_OUTCOMES.map((outcome: AttackOutcome) => ( + { + onChange({ ...filters, outcomes: filters.outcomes.includes(outcome) + ? filters.outcomes.filter((item: AttackOutcome) => item !== outcome) : [...filters.outcomes, outcome] }) + }} /> + ))} + + + + + + +
+
+
+ {filters.dimensions.map((predicate: AttackAnalyticsFilter, index: number) => { + const fingerprint = JSON.stringify(predicate) + const occurrence = occurrences.get(fingerprint) ?? 0 + occurrences.set(fingerprint, occurrence + 1) + const label = `${analyticsDimensionLabel(predicate.dimension)} (${predicate.match_mode.toUpperCase()}): ` + + predicate.values.map((value: AttackAnalyticsValue) => + knownLabels[optionLabelKey(predicate.dimension, value)] + ? analyticsOptionLabel({ key: value, label: knownLabels[optionLabelKey(predicate.dimension, value)] }) + : analyticsValueLabel(value), + ).join(', ') + return ( +
+ +
+ ) + })} + {filters.outcomes.map((outcome: AttackOutcome) => ( + + ))} + {filters.updated_after && ( + + )} + {filters.updated_before && ( + + )} +
+ {editor && !editorIsCurrent && The edited filter changed. Reopen a filter chip to edit the current selection.} + {editor && editorIsCurrent && ( + { setEditor(null) }} + /> + )} + {datesOpen && { setDatesOpen(false) }} />} +
+ ) +} diff --git a/frontend/src/components/Analytics/AnalyticsGroups.styles.ts b/frontend/src/components/Analytics/AnalyticsGroups.styles.ts new file mode 100644 index 0000000000..fbda14e604 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsGroups.styles.ts @@ -0,0 +1,56 @@ +import { makeStyles, shorthands, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' + +export const useAnalyticsGroupsStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalM, minWidth: 0 }, + groups: { + display: 'flex', + flexDirection: 'column', + overflowY: 'auto', + maxHeight: '26rem', + gap: tokens.spacingVerticalM, + padding: tokens.spacingHorizontalXS, + }, + group: { + display: 'grid', + gridTemplateColumns: 'minmax(8rem, 1fr) minmax(16rem, 3fr)', + alignItems: 'center', + gap: tokens.spacingHorizontalL, + paddingBottom: tokens.spacingVerticalM, + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + [NARROW_VIEWPORT_QUERY]: { gridTemplateColumns: 'minmax(0, 1fr)', gap: tokens.spacingVerticalS }, + }, + groupLabel: { display: 'flex', flexDirection: 'column', alignItems: 'flex-start', minWidth: 0 }, + groupButton: { + maxWidth: '100%', + overflowWrap: 'anywhere', + justifyContent: 'flex-start', + textAlign: 'start', + ...mobileTouchTargetHeight, + }, + visualization: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalXS, minWidth: 0 }, + bar: { + display: 'flex', + width: '100%', + height: tokens.spacingVerticalL, + backgroundColor: tokens.colorNeutralBackground4, + border: `1px solid ${tokens.colorNeutralStroke1}`, + ...mobileTouchTargetHeight, + }, + segment: { + display: 'block', + height: '100%', + '@media (forced-colors: active)': { borderRight: '1px solid currentColor', boxSizing: 'border-box' }, + }, + rateButton: { + minWidth: 0, + padding: 0, + borderRadius: 0, + justifyContent: 'flex-start', + }, + rate: { backgroundColor: tokens.colorBrandBackground }, + unavailable: { ...shorthands.borderStyle('dashed'), backgroundColor: tokens.colorNeutralBackground2 }, + number: { fontVariantNumeric: 'tabular-nums', overflowWrap: 'anywhere' }, + legend: { display: 'flex', alignItems: 'center', flexWrap: 'wrap', gap: tokens.spacingHorizontalS }, +}) diff --git a/frontend/src/components/Analytics/AnalyticsGroups.test.tsx b/frontend/src/components/Analytics/AnalyticsGroups.test.tsx new file mode 100644 index 0000000000..85879d1289 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsGroups.test.tsx @@ -0,0 +1,61 @@ +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { + ANALYTICS_OPERATION_FILTER, ANALYTICS_PREDICATE_LIMIT_REASON, makeAnalyticsReport, +} from '@/test-utils/analyticsFixtures' + +import AnalyticsGroups from './AnalyticsGroups' + +const TestWrapper: React.FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +) + +describe('AnalyticsGroups', () => { + beforeEach(() => { jest.clearAllMocks() }) + + it('should drill into an actual outcome segment using its server predicates and outcome', async () => { + const user = userEvent.setup() + const onDrilldown = jest.fn() + render() + expect(screen.queryByRole('table')).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Show aggregate data' })).not.toBeInTheDocument() + const segment = screen.getByRole('button', { name: 'Nightly: success segment; 4 results; filter dashboard' }) + segment.focus() + await user.keyboard('{Enter}') + expect(onDrilldown).toHaveBeenCalledWith([ANALYTICS_OPERATION_FILTER], 'success') + await user.click(screen.getByRole('button', { name: 'Nightly: 4 success; filter dashboard' })) + expect(onDrilldown).toHaveBeenCalledTimes(2) + }) + + it('should make the success-rate bar a keyboard-operable group drill-down', async () => { + const user = userEvent.setup() + const onDrilldown = jest.fn() + render() + expect(screen.queryByRole('table')).not.toBeInTheDocument() + const bar = screen.getByRole('button', { name: /Inspect Nightly success rate: 10 results; ASR 66.7%/ }) + expect(screen.getByText('66.7% (4 success / 6 decided, 10 total)')).toBeVisible() + bar.focus() + await user.keyboard(' ') + expect(onDrilldown).toHaveBeenCalledWith([ANALYTICS_OPERATION_FILTER]) + }) + + it.each([false, true])('should honor the SDK reason for every group action without hiding the chart (ASR: %s)', async (successRate: boolean) => { + const user = userEvent.setup() + const onDrilldown = jest.fn() + const report = makeAnalyticsReport({ drilldown_unavailable_reason: ANALYTICS_PREDICATE_LIMIT_REASON }) + render() + const groups = screen.getByRole('region', { name: successRate ? 'Success rate by Operation' : 'Outcome breakdown by Operation' }) + expect(screen.getByRole('note')).toHaveTextContent(ANALYTICS_PREDICATE_LIMIT_REASON) + expect(screen.getByRole('note')).toBeVisible() + expect(groups).toHaveAccessibleDescription(ANALYTICS_PREDICATE_LIMIT_REASON) + expect(within(groups).getByText('10 results')).toBeVisible() + for (const button of within(groups).getAllByRole('button')) { + expect(button).toBeDisabled() + expect(button).toHaveAccessibleDescription(/maximum 16 predicates/) + await user.click(button) + } + expect(onDrilldown).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/components/Analytics/AnalyticsGroups.tsx b/frontend/src/components/Analytics/AnalyticsGroups.tsx new file mode 100644 index 0000000000..aa387b7ac6 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsGroups.tsx @@ -0,0 +1,93 @@ +import { useId } from 'react' + +import { Button, mergeClasses, Text } from '@fluentui/react-components' + +import OutcomeSummaryBar from '@/components/AttackResults/OutcomeSummaryBar' +import type { AttackAnalyticsDenominator, AttackAnalyticsFilter, AttackAnalyticsGroup, AttackAnalyticsReport, AttackOutcome } from '@/types' +import { + analyticsDimensionLabel, analyticsOptionLabel, analyticsStatisticsLabel, analyticsSuccessCountsLabel, analyticsSuccessRate, analyticsValueKey, + formatAnalyticsCount, formatAnalyticsPercent, +} from '@/utils/attackAnalytics' + +import { AnalyticsAsrLabel } from './AnalyticsStats' +import { useAnalyticsGroupsStyles } from './AnalyticsGroups.styles' + +interface AnalyticsGroupsProps { + readonly report: AttackAnalyticsReport + readonly successRate: boolean + readonly denominator?: AttackAnalyticsDenominator + readonly onDrilldown: (filters: AttackAnalyticsFilter[], outcome?: AttackOutcome) => void +} + +/** + * Two presentations of the same server group page: outcome shares or ASR. + * Widths use SDK ratios, never a sum of visible rows. Both bars and labeled + * counts drill down with the supplied predicates; labels keep tiny segments usable. + */ +export default function AnalyticsGroups({ report, successRate, denominator = 'decided', onDrilldown }: AnalyticsGroupsProps) { + const styles = useAnalyticsGroupsStyles() + const reasonId = useId() + const drilldownUnavailable = report.drilldown_unavailable_reason !== null + const drilldownDescription = drilldownUnavailable ? reasonId : undefined + const marked = report.outcome_filter_applied + const title = `${successRate ? `Success rate${marked ? '*' : ''}` : 'Outcome breakdown'} by ${analyticsDimensionLabel(report.group_by)}` + + return ( +
+
+ {successRate ? ( + <>Fixed scale: 0% to 100%. Compare sample sizes, not rates alone. + ) : Bar width shows each outcome's share of all results in that group, not ASR.} +
+ {drilldownUnavailable && + {report.drilldown_unavailable_reason} + } +
+ {report.groups.map((group: AttackAnalyticsGroup) => { + const label = analyticsOptionLabel(group) + const statistics = group.statistics + const rate = analyticsSuccessRate(statistics, denominator) + return ( +
+
+ + {formatAnalyticsCount(statistics.total_results)} results +
+
+ {successRate && ( + <> + + + {formatAnalyticsPercent(rate)}{marked ? '*' : ''}{' '} + ({analyticsSuccessCountsLabel(statistics, denominator)}, {formatAnalyticsCount(statistics.total_results)} total) + + + )} + { onDrilldown(group.drilldown_filters, outcome) }} /> +
+
+ ) + })} + {report.groups.length === 0 && No groups on this page. Return to the first groups or narrow the filters.} +
+
+ ) +} diff --git a/frontend/src/components/Analytics/AnalyticsHeatmap.styles.ts b/frontend/src/components/Analytics/AnalyticsHeatmap.styles.ts new file mode 100644 index 0000000000..ec9e734294 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsHeatmap.styles.ts @@ -0,0 +1,47 @@ +import { makeStyles, shorthands, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight } from '@/styles/touchTargets' + +export const useAnalyticsHeatmapStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalS, minWidth: 0 }, + scroll: { overflow: 'auto', maxHeight: '28rem', minWidth: 0 }, + table: { tableLayout: 'auto', width: 'max-content', minWidth: '100%' }, + caption: { + paddingBlock: tokens.spacingVerticalS, + fontWeight: tokens.fontWeightSemibold, + textAlign: 'start', + }, + heading: { minWidth: '9rem', maxWidth: '16rem', overflowWrap: 'anywhere' }, + cell: { padding: tokens.spacingHorizontalXXS, minWidth: '9rem' }, + button: { + display: 'flex', + flexDirection: 'column', + width: '100%', + minWidth: '9rem', + gap: tokens.spacingVerticalXXS, + padding: tokens.spacingHorizontalS, + fontVariantNumeric: 'tabular-nums', + border: `1px solid ${tokens.colorNeutralStroke1}`, + borderRadius: tokens.borderRadiusSmall, + ...mobileTouchTargetHeight, + }, + level0: { backgroundColor: tokens.colorNeutralBackground1, color: tokens.colorNeutralForeground1 }, + level1: { backgroundColor: tokens.colorBrandBackground2, color: tokens.colorBrandForeground2 }, + level2: { backgroundColor: tokens.colorPaletteBlueBackground2, color: tokens.colorPaletteBlueForeground2 }, + level3: { backgroundColor: tokens.colorBrandBackground, color: tokens.colorNeutralForegroundOnBrand }, + level4: { backgroundColor: tokens.colorBrandBackgroundPressed, color: tokens.colorNeutralForegroundOnBrand }, + unavailable: { + backgroundColor: tokens.colorNeutralBackground3, + color: tokens.colorNeutralForeground2, + ...shorthands.borderStyle('dashed'), + }, + empty: { + backgroundColor: tokens.colorNeutralBackground2, + color: tokens.colorNeutralForeground2, + ...shorthands.borderStyle('dotted'), + }, + legend: { display: 'flex', alignItems: 'center', flexWrap: 'wrap', gap: tokens.spacingHorizontalS }, + scrollTable: { overflowX: 'auto' }, + dataTable: { minWidth: '55rem' }, + touchTarget: { ...mobileTouchTargetHeight }, +}) diff --git a/frontend/src/components/Analytics/AnalyticsHeatmap.test.tsx b/frontend/src/components/Analytics/AnalyticsHeatmap.test.tsx new file mode 100644 index 0000000000..26ee8f0a3e --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsHeatmap.test.tsx @@ -0,0 +1,160 @@ +import { + createHighContrastTheme, FluentProvider, webDarkTheme, webLightTheme, +} from '@fluentui/react-components' +import type { Theme } from '@fluentui/react-components' +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { + ANALYTICS_EMPTY_STATISTICS, ANALYTICS_PREDICATE_LIMIT_REASON, ANALYTICS_VALUE_LIMIT_REASON, makeAnalyticsReport, +} from '@/test-utils/analyticsFixtures' +import type { AttackAnalyticsFilter, AttackAnalyticsReport } from '@/types' + +import AnalyticsHeatmap from './AnalyticsHeatmap' + +const ROW_FILTER: AttackAnalyticsFilter = { + dimension: { name: 'targeted_harm_category' }, + values: [{ kind: 'value', value: 'category-A' }], + match_mode: 'any', +} +const COLUMN_FILTER: AttackAnalyticsFilter = { + dimension: { name: 'attack_type' }, + values: [{ kind: 'value', value: 'A' }], + match_mode: 'any', +} +const REPORT: AttackAnalyticsReport = makeAnalyticsReport({ + group_by: ROW_FILTER.dimension, + compare_by: COLUMN_FILTER.dimension, + outcome_filter_applied: true, + axes_truncated: true, + rows: [ + { key: { kind: 'value', value: 'category-A' }, label: 'category-A' }, + { key: { kind: 'missing', value: null }, label: 'Unknown' }, + ], + columns: [ + { key: { kind: 'value', value: 'A' }, label: 'A' }, + { key: { kind: 'value', value: 'B' }, label: 'B' }, + ], + cells: [ + { + row: { kind: 'value', value: 'category-A' }, column: { kind: 'value', value: 'A' }, + statistics: { + ...ANALYTICS_EMPTY_STATISTICS, total_results: 5, failures: 5, total_decided: 5, + success_rate: 0, success_rate_decided: 0, success_rate_all: 0, decided_share: 1, + outcome_shares: { success: 0, failure: 1, error: 0, undetermined: 0 }, + }, + drilldown_filters: [ROW_FILTER, COLUMN_FILTER], + }, + { + row: { kind: 'value', value: 'category-A' }, column: { kind: 'value', value: 'B' }, + statistics: { + ...ANALYTICS_EMPTY_STATISTICS, total_results: 3, errors: 2, undetermined: 1, + success_rate_all: 0, decided_share: 0, + outcome_shares: { success: 0, failure: 0, error: 2 / 3, undetermined: 1 / 3 }, + }, + drilldown_filters: [], + }, + { + row: { kind: 'missing', value: null }, column: { kind: 'value', value: 'A' }, + statistics: ANALYTICS_EMPTY_STATISTICS, drilldown_filters: [], + }, + ], +}) + +const TestWrapper: React.FC<{ children: React.ReactNode; theme?: Theme }> = ({ children, theme = webLightTheme }) => ( + {children} +) + +describe('AnalyticsHeatmap', () => { + beforeEach(() => { jest.clearAllMocks() }) + + it.each([ + ['light', webLightTheme], + ['dark', webDarkTheme], + ['high contrast', createHighContrastTheme()], + ] as const)('should distinguish missing, empty, no-decided and zero-success cells in %s', (_name: string, theme: Theme) => { + render() + const heatmap = screen.getByRole('region', { name: 'Heatmap' }) + expect(within(heatmap).getByText('0%*')).toBeInTheDocument() + expect(within(heatmap).getByText('Unavailable*')).toBeInTheDocument() + expect(within(heatmap).getByText('No results')).toBeInTheDocument() + expect(within(heatmap).getByText('No cell data')).toBeInTheDocument() + expect(within(heatmap).getByRole('rowheader', { name: 'Unknown (missing metadata)' })).toBeInTheDocument() + expect(screen.getByText(/Some values are omitted/)).toBeInTheDocument() + expect(screen.getByRole('button', { name: /ASR\*:.*Outcome filter active/ })).toBeInTheDocument() + }) + + it('should drill down via keyboard using the supplied cell predicates and expose exact counts', async () => { + const user = userEvent.setup() + const onDrilldown = jest.fn() + render() + const cell = screen.getByRole('button', { name: /category-A \/ A: 5 results; ASR\* 0%/ }) + cell.focus() + await user.keyboard('{Enter}') + expect(onDrilldown).toHaveBeenCalledWith([ROW_FILTER, COLUMN_FILTER]) + const empty = screen.getByRole('button', { name: /Unknown \(missing metadata\) \/ A: 0 results/ }) + expect(empty).toHaveAttribute('aria-disabled', 'true') + await user.click(empty) + expect(onDrilldown).toHaveBeenCalledTimes(1) + await user.click(screen.getByRole('button', { name: 'Show cell data' })) + expect(screen.getByRole('region', { name: 'Heatmap aggregate data' })).toBeInTheDocument() + expect(screen.getByText('Cell outcome counts and denominators')).toBeInTheDocument() + }) + + it('should show count mode independently of unavailable rates', () => { + render() + const noDecided = screen.getByRole('button', { name: /category-A \/ B: 3 results/ }) + expect(within(noDecided).getByText('3 total')).toBeInTheDocument() + expect(within(noDecided).getByText('0 success / 0 decided')).toBeInTheDocument() + expect(screen.getByText(/Count colors: 1-9, 10-99, 100-999, 1,000\+/)).toBeInTheDocument() + }) + + it.each([ANALYTICS_PREDICATE_LIMIT_REASON, ANALYTICS_VALUE_LIMIT_REASON])( + 'should make blocked cells keyboard-readable but inert and retain the data table: %s', + async (reason: string) => { + const user = userEvent.setup() + const onDrilldown = jest.fn() + render() + expect(screen.getByRole('note')).toHaveTextContent(reason) + expect(screen.getByRole('note')).toBeVisible() + const heatmap = screen.getByRole('region', { name: 'Heatmap' }) + expect(heatmap).toHaveAccessibleDescription(reason) + const cell = within(heatmap).getByRole('button', { name: /category-A \/ A: 5 results/ }) + expect(cell).toHaveAttribute('aria-disabled', 'true') + expect(cell).toHaveAccessibleDescription(/maximum (16 predicates|500 values)/) + cell.focus() + expect(cell).toHaveFocus() + await user.keyboard('{Enter} ') + await user.click(cell) + expect(onDrilldown).not.toHaveBeenCalled() + expect(within(heatmap).getByText('0%*')).toBeVisible() + await user.click(screen.getByRole('button', { name: 'Show cell data' })) + expect(screen.getByRole('region', { name: 'Heatmap aggregate data' })).toBeVisible() + }, + ) + + it('labels successes and decided results separately from the total population', () => { + const report = { + ...REPORT, + outcome_filter_applied: false, + cells: [{ + ...REPORT.cells[0], + statistics: { + ...ANALYTICS_EMPTY_STATISTICS, + total_results: 5, successes: 3, total_decided: 3, undetermined: 2, + success_rate: 1, success_rate_decided: 1, success_rate_all: 0.6, decided_share: 0.6, + outcome_shares: { success: 0.6, failure: 0, error: 0, undetermined: 0.4 }, + }, + }], + } + render() + const cell = screen.getByRole('button', { name: /category-A \/ A: 5 results; ASR 100%/ }) + expect(within(cell).getByText('100%', { exact: true })).toBeVisible() + expect(within(cell).getByText('3 success / 3 decided')).toBeVisible() + expect(within(cell).getByText('5 total')).toBeVisible() + expect(within(cell).queryByText('3 / 3 decided')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/Analytics/AnalyticsHeatmap.tsx b/frontend/src/components/Analytics/AnalyticsHeatmap.tsx new file mode 100644 index 0000000000..4ea1221312 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsHeatmap.tsx @@ -0,0 +1,165 @@ +import { useId, useState } from 'react' + +import { + Button, mergeClasses, Table, TableBody, TableCell, TableHeader, + TableHeaderCell, TableRow, Text, Tooltip, +} from '@fluentui/react-components' + +import type { + AttackAnalyticsCell, AttackAnalyticsDenominator, AttackAnalyticsFilter, AttackAnalyticsHeatmapMetric, + AttackAnalyticsOption, AttackAnalyticsReport, AttackAnalyticsValue, +} from '@/types' +import { + ANALYTICS_AXIS_LIMIT, analyticsDimensionLabel, analyticsOptionLabel, analyticsStatisticsLabel, analyticsSuccessCountsLabel, analyticsSuccessRate, analyticsValueKey, + formatAnalyticsCount, formatAnalyticsPercent, +} from '@/utils/attackAnalytics' + +import { AnalyticsAsrLabel } from './AnalyticsStats' +import { useAnalyticsHeatmapStyles } from './AnalyticsHeatmap.styles' + +interface AnalyticsHeatmapProps { + readonly report: AttackAnalyticsReport + readonly metric: AttackAnalyticsHeatmapMetric + readonly denominator?: AttackAnalyticsDenominator + readonly onDrilldown: (filters: AttackAnalyticsFilter[]) => void +} + +/** Join typed axis identities, not friendly labels; different absence kinds may share a displayed label. */ +function cellKey(row: AttackAnalyticsValue, column: AttackAnalyticsValue): string { + return JSON.stringify([analyticsValueKey(row), analyticsValueKey(column)]) +} + +/** + * Render the SDK's bounded matrix without regrouping results or filling absent + * cells with invented zeroes. The color metric and optional data table are local + * presentation; every drill-down uses the cell's server-provided predicates. + */ +export default function AnalyticsHeatmap({ report, metric, denominator = 'decided', onDrilldown }: AnalyticsHeatmapProps) { + const styles = useAnalyticsHeatmapStyles() + const reasonId = useId() + const drilldownUnavailable = report.drilldown_unavailable_reason !== null + const drilldownDescription = drilldownUnavailable ? reasonId : undefined + const [showData, setShowData] = useState(false) + const marked = report.outcome_filter_applied + const cells = new Map(report.cells.map((cell: AttackAnalyticsCell) => [cellKey(cell.row, cell.column), cell])) + const rowLabels = new Map(report.rows.map((option: AttackAnalyticsOption) => [analyticsValueKey(option.key), analyticsOptionLabel(option)])) + const columnLabels = new Map(report.columns.map((option: AttackAnalyticsOption) => [analyticsValueKey(option.key), analyticsOptionLabel(option)])) + const rowName = analyticsDimensionLabel(report.group_by) + const columnName = report.compare_by ? analyticsDimensionLabel(report.compare_by) : 'Comparison' + + /** Fixed bands stay comparable across cohorts; empty, unavailable, and zero remain distinct. */ + function cellColorClass(cell: AttackAnalyticsCell): string { + const statistics = cell.statistics + if (statistics.total_results === 0) return styles.empty + const value = metric === 'success_rate' ? analyticsSuccessRate(statistics, denominator) : statistics.total_results + if (value === null) return styles.unavailable + if (value === 0) return styles.level0 + if (value <= (metric === 'success_rate' ? 0.25 : 9)) return styles.level1 + if (value <= (metric === 'success_rate' ? 0.5 : 99)) return styles.level2 + if (value <= (metric === 'success_rate' ? 0.75 : 999)) return styles.level3 + return styles.level4 + } + + return ( +
+
+ {metric === 'success_rate' ? ( + <>Darker: higher ASR{marked ? '*' : ''}, on a fixed 0% to 100% scale. + ) : Count colors: 1-9, 10-99, 100-999, 1,000+ results. Darker means more results.} + No results, unavailable ASR, and 0% ASR are distinct. The selected denominator applies to every cell. +
+ {drilldownUnavailable && + {report.drilldown_unavailable_reason} + } +
+ + + + {rowName} / {columnName} + {report.columns.map((column: AttackAnalyticsOption) => ( + + {analyticsOptionLabel(column)} + + ))} + + + {report.rows.map((row: AttackAnalyticsOption) => ( + + {analyticsOptionLabel(row)} + {report.columns.map((column: AttackAnalyticsOption) => { + const cell = cells.get(cellKey(row.key, column.key)) + const label = `${analyticsOptionLabel(row)} / ${analyticsOptionLabel(column)}` + const description = cell ? analyticsStatisticsLabel(cell.statistics, marked, denominator) : 'No cell data' + const value = !cell ? 'No cell data' : cell.statistics.total_results === 0 ? 'No results' + : metric === 'success_rate' ? `${formatAnalyticsPercent(analyticsSuccessRate(cell.statistics, denominator))}${marked ? '*' : ''}` + : `${formatAnalyticsCount(cell.statistics.total_results)} total` + return ( + + + + + + ) + })} + + ))} + +
{rowName} by {columnName}
+
+ {report.axes_truncated && Axes are limited to {ANALYTICS_AXIS_LIMIT} values each. Some values are omitted; narrow the filters to inspect them.} +
+ +
+ {showData && ( +
+ + + + RowColumn + Total results + DecidedDecided share + SuccessesFailures + ErrorsUndetermined + + + {report.cells.map((cell: AttackAnalyticsCell) => ( + + {rowLabels.get(analyticsValueKey(cell.row))} + {columnLabels.get(analyticsValueKey(cell.column))} + {formatAnalyticsCount(cell.statistics.total_results)} + {formatAnalyticsPercent(analyticsSuccessRate(cell.statistics, denominator))}{marked ? '*' : ''} + {formatAnalyticsCount(cell.statistics.total_decided)} + {formatAnalyticsPercent(cell.statistics.decided_share)} + {formatAnalyticsCount(cell.statistics.successes)} + {formatAnalyticsCount(cell.statistics.failures)} + {formatAnalyticsCount(cell.statistics.errors)} + {formatAnalyticsCount(cell.statistics.undetermined)} + + ))} + +
Cell outcome counts and denominators
+
+ )} +
+ ) +} diff --git a/frontend/src/components/Analytics/AnalyticsPage.styles.ts b/frontend/src/components/Analytics/AnalyticsPage.styles.ts new file mode 100644 index 0000000000..a4fa7c8d1a --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsPage.styles.ts @@ -0,0 +1,68 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' +import { WORKSPACE_CANVAS_BACKGROUND } from '@/styles/workspaceBackground' + +export const useAnalyticsPageStyles = makeStyles({ + root: { + height: '100%', + overflowY: 'auto', + minWidth: 0, + backgroundColor: WORKSPACE_CANVAS_BACKGROUND, + }, + header: { + display: 'flex', + flexWrap: 'wrap', + alignItems: 'center', + justifyContent: 'space-between', + gap: tokens.spacingHorizontalM, + padding: `${tokens.spacingVerticalM} ${tokens.spacingHorizontalXXL}`, + backgroundColor: tokens.colorNeutralBackground3, + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + [NARROW_VIEWPORT_QUERY]: { paddingInline: tokens.spacingHorizontalM }, + }, + title: { margin: 0 }, + row: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalM, + }, + body: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalL, + padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalXXL}`, + [NARROW_VIEWPORT_QUERY]: { paddingInline: tokens.spacingHorizontalM }, + }, + section: { + minWidth: 0, + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + }, + toolbar: { + display: 'flex', + flexWrap: 'wrap', + alignItems: 'end', + gap: tokens.spacingHorizontalL, + }, + note: { + color: tokens.colorNeutralForeground2, + overflowWrap: 'anywhere', + }, + control: { minWidth: 0, maxWidth: '100%', [NARROW_VIEWPORT_QUERY]: { width: '100%' } }, + button: { minWidth: 0, maxWidth: '100%', ...mobileTouchTargetHeight }, + empty: { + padding: `${tokens.spacingVerticalXXL} ${tokens.spacingHorizontalL}`, + display: 'flex', + flexDirection: 'column', + alignItems: 'flex-start', + gap: tokens.spacingVerticalM, + }, + skeleton: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalL, + }, +}) diff --git a/frontend/src/components/Analytics/AnalyticsPage.test.tsx b/frontend/src/components/Analytics/AnalyticsPage.test.tsx new file mode 100644 index 0000000000..8d8b2d0a6e --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsPage.test.tsx @@ -0,0 +1,632 @@ +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { act, render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { MemoryRouter, useLocation, useNavigate } from 'react-router' + +import { analyticsApi } from '@/services/api' +import { useRuntime } from '@/hooks/useRuntime' +import { + ANALYTICS_EMPTY_STATISTICS, ANALYTICS_OPERATION_FILTER, ANALYTICS_PAGE_TIME, + ANALYTICS_PREDICATE_LIMIT_REASON, ANALYTICS_TEST_STATISTICS, ANALYTICS_TEST_TIME, ANALYTICS_VALUE_LIMIT_REASON, + makeAnalyticsFacets, makeAnalyticsReport, makeAnalyticsResults, makeAnalyticsRow, +} from '@/test-utils/analyticsFixtures' +import type { + AttackAnalyticsFilter, AttackAnalyticsFilters, AttackAnalyticsQuery, AttackAnalyticsReport, + AttackAnalyticsValue, AttackAnalyticsViewState, +} from '@/types' +import { + ANALYTICS_ASR_NOTE, ANALYTICS_DEBOUNCE_MS, DEFAULT_ANALYTICS_VIEW, analyticsViewToHash, formatAnalyticsTime, +} from '@/utils/attackAnalytics' + +import AnalyticsPage from './AnalyticsPage' + +jest.mock('@/services/api', () => ({ + analyticsApi: { query: jest.fn(), results: jest.fn(), facets: jest.fn() }, +})) +jest.mock('@/hooks/useRuntime', () => ({ useRuntime: jest.fn() })) +const mockRuntime = jest.mocked(useRuntime) + +const mockQuery = jest.mocked(analyticsApi.query) +const mockResults = jest.mocked(analyticsApi.results) +const mockFacets = jest.mocked(analyticsApi.facets) + +const TestWrapper: React.FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +) + +function NavigationProbe() { + const navigate = useNavigate() + const location = useLocation() + return <>{location.search}{location.hash} +} + +function renderPage(view?: AttackAnalyticsViewState, onOpenAttack: (id: string) => void = jest.fn()) { + const path = view ? `/analytics${analyticsViewToHash(view)}` : '/analytics' + return render( + + + , + ) +} + +function deferred() { + let resolve: ((value: T) => void) | undefined + let reject: ((reason: unknown) => void) | undefined + const promise = new Promise((onResolve: (value: T | PromiseLike) => void, onReject: (reason: unknown) => void) => { + resolve = onResolve + reject = onReject + }) + return { + promise, + resolve: (value: T): void => { if (resolve) resolve(value) }, + reject: (reason: unknown): void => { if (reject) reject(reason) }, + } +} + +function echoQuery(query: AttackAnalyticsQuery): AttackAnalyticsReport { + return makeAnalyticsReport({ + filters: query.filters ?? { dimensions: [], outcomes: [] }, + group_by: query.group_by ?? { name: 'operation' }, + compare_by: query.compare_by ?? null, + }) +} + +interface DrilldownBoundaryCase { + readonly chart: 'outcomes' | 'heatmap' + readonly budget: 'predicates' | 'values' + readonly initialSize: number +} + +describe('AnalyticsPage', () => { + beforeEach(() => { + jest.clearAllMocks() + mockRuntime.mockReturnValue({ ready: true, state: 'ready', generation: 'first-runtime' }) + mockQuery.mockReset().mockImplementation(async (query: AttackAnalyticsQuery) => echoQuery(query)) + mockResults.mockReset().mockResolvedValue(makeAnalyticsResults()) + mockFacets.mockReset().mockResolvedValue(makeAnalyticsFacets()) + }) + + afterEach(() => { jest.useRealTimers() }) + + it.each([ + { chart: 'outcomes', budget: 'predicates', initialSize: 15 }, + { chart: 'heatmap', budget: 'predicates', initialSize: 14 }, + { chart: 'outcomes', budget: 'values', initialSize: 499 }, + { chart: 'heatmap', budget: 'values', initialSize: 498 }, + ])( + 'should allow the final valid $chart drill-down at the $budget limit without disabling the report', + async ({ chart, budget, initialSize }: DrilldownBoundaryCase) => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const reason = budget === 'predicates' ? ANALYTICS_PREDICATE_LIMIT_REASON : ANALYTICS_VALUE_LIMIT_REASON + const filters: AttackAnalyticsFilters = { + ...DEFAULT_ANALYTICS_VIEW.filters, + dimensions: budget === 'predicates' + ? Array.from({ length: initialSize }, () => ANALYTICS_OPERATION_FILTER) + : Array.from({ length: 5 }, (_unused: unknown, index: number): AttackAnalyticsFilter => ({ + ...ANALYTICS_OPERATION_FILTER, + values: Array.from({ length: index === 4 ? initialSize - 400 : 100 }, (_value: unknown, value: number): AttackAnalyticsValue => ({ + kind: 'value', value: value === 0 ? 'Nightly' : `operation-${index}-${value}`, + })), + })), + } + const row: AttackAnalyticsFilter = { + dimension: DEFAULT_ANALYTICS_VIEW.heatmapRow, + values: [{ kind: 'value', value: 'category-A' }], match_mode: 'any', + } + const column: AttackAnalyticsFilter = { + dimension: DEFAULT_ANALYTICS_VIEW.heatmapColumn, + values: [{ kind: 'value', value: 'PromptSendingAttack' }], match_mode: 'any', + } + const additions = chart === 'heatmap' ? [row, column] : [ANALYTICS_OPERATION_FILTER] + const reportForQuery = (query: AttackAnalyticsQuery, unavailableReason: string | null): AttackAnalyticsReport => makeAnalyticsReport({ + ...echoQuery(query), + drilldown_unavailable_reason: unavailableReason, + has_more_groups: true, next_group_offset: 15, + results: makeAnalyticsResults({ has_more: true, next_cursor: 'page-at-limit' }), + ...(chart === 'heatmap' ? { + rows: [{ key: row.values[0], label: 'category-A' }], + columns: [{ key: column.values[0], label: 'PromptSendingAttack' }], + cells: [{ + row: row.values[0], column: column.values[0], + statistics: ANALYTICS_TEST_STATISTICS, drilldown_filters: additions, + }], + } : {}), + }) + mockQuery.mockImplementation(async (query: AttackAnalyticsQuery) => reportForQuery(query, reason)) + .mockImplementationOnce(async (query: AttackAnalyticsQuery) => reportForQuery(query, null)) + mockResults.mockResolvedValue(makeAnalyticsResults({ + items: [makeAnalyticsRow({ objective_preview: 'Paged at the limit' })], + })) + renderPage({ ...DEFAULT_ANALYTICS_VIEW, chart, filters }) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + const actionName = chart === 'outcomes' ? 'Filter to Nightly' : /category-A \/ PromptSendingAttack: 10 results/ + await user.click(screen.getByRole('button', { name: actionName })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + + expect(mockQuery).toHaveBeenCalledTimes(2) + expect(mockQuery.mock.calls[1][0].filters?.dimensions).toEqual([...filters.dimensions, ...additions]) + const notice = screen.getByText(reason, { selector: '[role="note"]' }) + expect(notice).toBeVisible() + expect(screen.getByRole('region', { name: 'Outcome summary' })).toBeVisible() + const chartRegion = screen.getByRole('region', { name: chart === 'outcomes' ? 'Outcome breakdown by Operation' : 'Heatmap' }) + expect(chartRegion).toBeVisible() + expect(chartRegion).toHaveAccessibleDescription(reason) + for (const button of within(chartRegion).getAllByRole('button')) { + if (chart === 'outcomes') expect(button).toBeDisabled() + else expect(button).toHaveAttribute('aria-disabled', 'true') + } + const url = screen.getByLabelText('Current URL').textContent + await user.click(screen.getByRole('button', { name: actionName })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + expect(screen.getByLabelText('Current URL').textContent).toBe(url) + expect(mockQuery).toHaveBeenCalledTimes(2) + + const results = screen.getByRole('region', { name: 'Matching AttackResults' }) + await user.click(within(results).getByRole('button', { name: 'Next' })) + expect(await within(results).findByText('Paged at the limit')).toBeInTheDocument() + expect(within(results).getByText('Page 2')).toBeInTheDocument() + expect(mockResults).toHaveBeenCalledTimes(1) + expect(mockQuery).toHaveBeenCalledTimes(2) + if (chart === 'outcomes') expect(screen.getByRole('button', { name: 'Next groups' })).toBeEnabled() + + await user.click(screen.getByRole('button', { name: 'Filter to success: 4' })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + expect(mockQuery).toHaveBeenCalledTimes(3) + expect(mockQuery.mock.calls[2][0].filters).toMatchObject({ + dimensions: [...filters.dimensions, ...additions], outcomes: ['success'], + }) + expect(screen.getByText(reason, { selector: '[role="note"]' })).toBeVisible() + }, + ) + + it('should load all saved results once, use server statistics and reuse the included first page', async () => { + mockQuery.mockResolvedValue(makeAnalyticsReport({ + summary: { + success_rate: 0.625, success_rate_decided: 0.625, success_rate_all: 15 / 64, + successes: 15, failures: 9, errors: 10, undetermined: 30, total_decided: 24, + total_results: 64, decided_share: 0.375, + outcome_shares: { success: 15 / 64, failure: 9 / 64, error: 10 / 64, undetermined: 30 / 64 }, + }, + })) + const onOpenAttack = jest.fn() + const user = userEvent.setup() + renderPage(undefined, onOpenAttack) + expect(screen.getByRole('status', { name: 'Loading analytics for current filters' })).toBeInTheDocument() + expect(await screen.findByText('Inspect a saved response')).toBeInTheDocument() + const summary = screen.getByRole('region', { name: 'Outcome summary' }) + expect(within(summary).getByText('62.5%')).toBeInTheDocument() + expect(within(summary).getByText('37.5%')).toBeInTheDocument() + expect(within(summary).getByText('15 success / 24 decided')).toBeInTheDocument() + expect(within(summary).getByText('64 total')).toBeInTheDocument() + expect(mockQuery).toHaveBeenCalledTimes(1) + expect(mockQuery).toHaveBeenCalledWith(expect.objectContaining({ + filters: DEFAULT_ANALYTICS_VIEW.filters, + group_by: { name: 'operation' }, compare_by: null, result_limit: 25, + }), expect.any(AbortSignal)) + expect(mockResults).not.toHaveBeenCalled() + expect(mockFacets).not.toHaveBeenCalled() + await user.click(screen.getByRole('row', { name: 'Open result result-1' })) + expect(onOpenAttack).toHaveBeenCalledWith('result-1') + }) + + it('should paginate only results, keep report freshness, and reset the table on Reload', async () => { + const user = userEvent.setup() + mockQuery.mockResolvedValue(makeAnalyticsReport({ + results: makeAnalyticsResults({ has_more: true, next_cursor: 'opaque-next' }), + })) + mockResults.mockResolvedValue(makeAnalyticsResults({ + items: [makeAnalyticsRow({ attack_result_id: 'result-2', objective_preview: 'Second saved result' })], + computed_at: ANALYTICS_PAGE_TIME, + })) + renderPage() + await screen.findByText('Inspect a saved response') + const refreshed = screen.getByText(/Last refreshed:/).textContent + const results = screen.getByRole('region', { name: 'Matching AttackResults' }) + await user.click(within(results).getByRole('button', { name: 'Next' })) + expect(await screen.findByText('Second saved result')).toBeInTheDocument() + expect(within(results).getByText('Page 2')).toBeInTheDocument() + expect(screen.getByText(/Last refreshed:/).textContent).toBe(refreshed) + expect(within(results).getByText(formatAnalyticsTime(ANALYTICS_PAGE_TIME))).toBeInTheDocument() + expect(mockResults).toHaveBeenCalledWith({ + filters: { dimensions: [], outcomes: [] }, cursor: 'opaque-next', limit: 25, + }, expect.any(AbortSignal)) + expect(mockQuery).toHaveBeenCalledTimes(1) + expect(mockFacets).not.toHaveBeenCalled() + await user.click(screen.getByRole('button', { name: 'Reload' })) + expect(await screen.findByText('Inspect a saved response')).toBeInTheDocument() + await waitFor(() => { expect(mockQuery).toHaveBeenCalledTimes(2) }) + expect(within(results).getByText('Page 1')).toBeInTheDocument() + expect(mockResults).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['heatmap color', 'heatmap', 'Cell color', 'total_results'], + ['bar presentation', 'outcomes', 'Chart view', 'success-rate'], + ['ASR denominator', 'outcomes', 'ASR denominator', 'all'], + ['heatmap denominator', 'heatmap', 'ASR denominator', 'all'], + ] as const)( + 'should change only %s without requests or resetting result, group, or facet pages', + async (_name: string, chart: 'heatmap' | 'outcomes', control: string, value: string) => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + mockQuery.mockImplementation(async (query: AttackAnalyticsQuery) => makeAnalyticsReport({ + ...echoQuery(query), + has_more_groups: true, next_group_offset: 15, + results: makeAnalyticsResults({ has_more: true, next_cursor: 'page-2' }), + })) + mockResults.mockResolvedValue(makeAnalyticsResults({ + items: [makeAnalyticsRow({ objective_preview: 'Second saved result' })], + computed_at: ANALYTICS_PAGE_TIME, + })) + mockFacets.mockResolvedValue(makeAnalyticsFacets({ has_more: true, next_offset: 50 })) + renderPage(chart === 'heatmap' ? { ...DEFAULT_ANALYTICS_VIEW, chart } : undefined) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + if (chart === 'outcomes') { + await user.click(screen.getByRole('button', { name: 'Next groups' })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + } + const reportRequests = chart === 'outcomes' ? 2 : 1 + const results = screen.getByRole('region', { name: 'Matching AttackResults' }) + await user.click(within(results).getByRole('button', { name: 'Next' })) + expect(await within(results).findByText('Page 2')).toBeInTheDocument() + const refreshed = screen.getByText(/Last refreshed:/).textContent + await user.click(screen.getByRole('button', { name: 'Operation' })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + await user.click(screen.getByRole('button', { name: 'Next values' })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + + await user.selectOptions(screen.getByLabelText(control), value) + expect(screen.queryByRole('status', { name: 'Reloading saved results' })).not.toBeInTheDocument() + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS * 2) }) + + expect(mockQuery).toHaveBeenCalledTimes(reportRequests) + expect(mockResults).toHaveBeenCalledTimes(1) + expect(mockFacets).toHaveBeenCalledTimes(2) + expect(within(results).getByText('Page 2')).toBeInTheDocument() + expect(within(results).getByText('Second saved result')).toBeInTheDocument() + expect(within(results).getByText(formatAnalyticsTime(ANALYTICS_PAGE_TIME))).toBeInTheDocument() + expect(screen.getByText(/Last refreshed:/).textContent).toBe(refreshed) + expect(screen.getByRole('button', { name: 'First values' })).toBeEnabled() + if (chart === 'outcomes') expect(screen.getByRole('button', { name: 'First groups' })).toBeEnabled() + }, + ) + + it('should not abort an in-flight results page when only the heatmap color changes', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const pending = deferred>() + mockQuery.mockResolvedValue(makeAnalyticsReport({ + results: makeAnalyticsResults({ has_more: true, next_cursor: 'page-2' }), + })) + mockResults.mockReturnValueOnce(pending.promise) + renderPage({ ...DEFAULT_ANALYTICS_VIEW, chart: 'heatmap' }) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + const results = screen.getByRole('region', { name: 'Matching AttackResults' }) + await user.click(within(results).getByRole('button', { name: 'Next' })) + const signal = mockResults.mock.calls[0][1] + await user.selectOptions(screen.getByLabelText('Cell color'), 'total_results') + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS * 2) }) + expect(signal?.aborted).toBe(false) + expect(mockQuery).toHaveBeenCalledTimes(1) + await act(async () => { + pending.resolve(makeAnalyticsResults({ items: [makeAnalyticsRow({ objective_preview: 'Second saved result' })] })) + }) + expect(within(results).getByText('Page 2')).toBeInTheDocument() + expect(within(results).getByText('Second saved result')).toBeInTheDocument() + }) + + it.each([false, true])( + 'should treat returning to an earlier query as a new read without reviving its result page (intervening read settled: %s)', + async (interveningReadSettled: boolean) => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const otherReport = deferred() + const returnedReport = deferred() + mockQuery.mockResolvedValueOnce(makeAnalyticsReport({ + results: makeAnalyticsResults({ has_more: true, next_cursor: 'old-page-2' }), + })).mockReturnValueOnce(otherReport.promise).mockReturnValueOnce(returnedReport.promise) + mockResults.mockResolvedValue(makeAnalyticsResults({ + items: [makeAnalyticsRow({ objective_preview: 'Old second page' })], + })) + renderPage() + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + await user.click(within(screen.getByRole('region', { name: 'Matching AttackResults' })).getByRole('button', { name: 'Next' })) + await screen.findByText('Page 2') + await user.selectOptions(screen.getByLabelText('Group by'), 'model') + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + if (interveningReadSettled) { + await act(async () => { otherReport.resolve(makeAnalyticsReport({ group_by: { name: 'model' } })) }) + } + + await user.click(screen.getByRole('button', { name: 'Browser back' })) + expect(screen.getByRole('status', { + name: interveningReadSettled ? 'Loading analytics for current filters' : 'Reloading saved results', + })).toBeInTheDocument() + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + await act(async () => { + returnedReport.resolve(makeAnalyticsReport({ + computed_at: ANALYTICS_PAGE_TIME, + results: makeAnalyticsResults({ items: [makeAnalyticsRow({ objective_preview: 'Fresh original cohort' })] }), + })) + otherReport.resolve(makeAnalyticsReport({ group_by: { name: 'model' } })) + }) + expect(screen.getByText('Fresh original cohort')).toBeInTheDocument() + expect(screen.getByText('Page 1')).toBeInTheDocument() + expect(screen.queryByText('Old second page')).not.toBeInTheDocument() + expect(mockQuery).toHaveBeenCalledTimes(3) + expect(mockResults).toHaveBeenCalledTimes(1) + }, + ) + + it('should keep a failed Reload visibly stale with its original timestamp and support Retry', async () => { + const user = userEvent.setup() + mockQuery.mockResolvedValueOnce(makeAnalyticsReport()) + .mockRejectedValueOnce(new Error('Analytics is busy')) + .mockResolvedValueOnce(makeAnalyticsReport({ computed_at: ANALYTICS_PAGE_TIME })) + renderPage() + await screen.findByText('Inspect a saved response') + await user.click(screen.getByRole('button', { name: 'Reload' })) + expect(await screen.findByText(/Reload failed.*data may be stale/)).toBeInTheDocument() + expect(screen.getByText('Inspect a saved response')).toBeInTheDocument() + expect(screen.getByText(/Last refreshed:/)).toHaveTextContent(formatAnalyticsTime(ANALYTICS_TEST_TIME)) + await user.click(screen.getByRole('button', { name: 'Retry' })) + await waitFor(() => { expect(screen.getByText(/Last refreshed:/)).toHaveTextContent(formatAnalyticsTime(ANALYTICS_PAGE_TIME)) }) + expect(screen.queryByText(/data may be stale/)).not.toBeInTheDocument() + }) + + it('should never show old results under changed filters, including when the new cohort fails', async () => { + const user = userEvent.setup() + const pending = deferred() + mockQuery.mockResolvedValueOnce(makeAnalyticsReport()).mockReturnValueOnce(pending.promise) + renderPage() + await screen.findByText('Inspect a saved response') + await user.click(screen.getByRole('button', { name: 'Filter to error: 1' })) + expect(screen.queryByText('Inspect a saved response')).not.toBeInTheDocument() + expect(screen.queryByRole('region', { name: 'Outcome summary' })).not.toBeInTheDocument() + await waitFor(() => { expect(mockQuery).toHaveBeenCalledTimes(2) }) + await act(async () => { pending.reject(new Error('Cannot read this cohort')) }) + expect(await screen.findByText(/Could not load analytics for these filters/)).toBeInTheDocument() + expect(screen.queryByText('Inspect a saved response')).not.toBeInTheDocument() + }) + + it('should abort superseded reports and ignore a late successful response', async () => { + const user = userEvent.setup() + const pending = deferred() + mockQuery.mockReturnValueOnce(pending.promise).mockResolvedValueOnce(makeAnalyticsReport({ + group_by: { name: 'model' }, + results: makeAnalyticsResults({ items: [makeAnalyticsRow({ objective_preview: 'Latest cohort' })] }), + })) + renderPage() + await waitFor(() => { expect(mockQuery).toHaveBeenCalledTimes(1) }) + const signal = mockQuery.mock.calls[0][1] + await user.selectOptions(screen.getByLabelText('Group by'), 'model') + expect(await screen.findByText('Latest cohort')).toBeInTheDocument() + expect(signal?.aborted).toBe(true) + await act(async () => { pending.resolve(makeAnalyticsReport()) }) + expect(screen.getByText('Latest cohort')).toBeInTheDocument() + expect(screen.queryByText('Inspect a saved response')).not.toBeInTheDocument() + }) + + it('should abort an in-flight results page on Reload so it cannot replace the refreshed first page', async () => { + const user = userEvent.setup() + const pending = deferred>() + mockQuery.mockResolvedValue(makeAnalyticsReport({ + results: makeAnalyticsResults({ has_more: true, next_cursor: 'page-2' }), + })) + mockResults.mockReturnValueOnce(pending.promise) + renderPage() + await screen.findByText('Inspect a saved response') + await user.click(within(screen.getByRole('region', { name: 'Matching AttackResults' })).getByRole('button', { name: 'Next' })) + const signal = mockResults.mock.calls[0][1] + await user.click(screen.getByRole('button', { name: 'Reload' })) + await waitFor(() => { expect(mockQuery).toHaveBeenCalledTimes(2) }) + expect(signal?.aborted).toBe(true) + await act(async () => { + pending.resolve(makeAnalyticsResults({ items: [makeAnalyticsRow({ objective_preview: 'Obsolete page' })] })) + }) + expect(screen.queryByText('Obsolete page')).not.toBeInTheDocument() + expect(screen.getByText('Page 1')).toBeInTheDocument() + }) + + it('should preserve report and current rows on a results-only error, then retry that cursor', async () => { + const user = userEvent.setup() + mockQuery.mockResolvedValue(makeAnalyticsReport({ + results: makeAnalyticsResults({ has_more: true, next_cursor: 'retry-cursor' }), + })) + mockResults.mockRejectedValueOnce(new Error('Page deadline exceeded')).mockResolvedValueOnce(makeAnalyticsResults()) + renderPage() + await screen.findByText('Inspect a saved response') + await user.click(within(screen.getByRole('region', { name: 'Matching AttackResults' })).getByRole('button', { name: 'Next' })) + expect(await screen.findByText(/Still showing page 1/)).toBeInTheDocument() + expect(screen.getByText('Inspect a saved response')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Retry results' })) + await screen.findByText('Page 2') + expect(mockResults.mock.calls[1][0].cursor).toBe('retry-cursor') + expect(mockQuery).toHaveBeenCalledTimes(1) + }) + + it('should apply typed group predicates in addition to existing converter, outcome and date filters', async () => { + const user = userEvent.setup() + const converters: AttackAnalyticsFilter = { + dimension: { name: 'converter_type', converter_direction: 'response' }, + values: [{ kind: 'value', value: 'A' }, { kind: 'value', value: 'B' }], + match_mode: 'any', + } + const added: AttackAnalyticsFilter = { ...converters, values: [{ kind: 'missing', value: null }] } + const view: AttackAnalyticsViewState = { + ...DEFAULT_ANALYTICS_VIEW, + groupBy: converters.dimension, + filters: { + dimensions: [ANALYTICS_OPERATION_FILTER, converters], outcomes: ['success', 'failure'], + updated_after: '2026-01-01T00:00:00Z', updated_before: '2026-10-01T00:00:00Z', + }, + } + mockQuery.mockImplementation(async (query: AttackAnalyticsQuery) => makeAnalyticsReport({ + ...echoQuery(query), outcome_filter_applied: true, groups_overlap: true, + groups: [{ key: { kind: 'missing', value: null }, label: 'Unknown', statistics: ANALYTICS_TEST_STATISTICS, drilldown_filters: [added] }], + })) + renderPage(view) + await user.click(await screen.findByRole('button', { name: 'Filter to Unknown (missing metadata)' })) + await waitFor(() => { + expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ + filters: { ...view.filters, dimensions: [...view.filters.dimensions, added] }, + }), expect.any(AbortSignal)) + }) + expect(screen.getByText(/Groups overlap/)).toBeInTheDocument() + }) + + it('should clear a rejected edit message when browser Back restores a valid view', async () => { + const user = userEvent.setup() + renderPage() + await screen.findByText('Inspect a saved response') + await user.selectOptions(screen.getByLabelText('Chart view'), 'heatmap') + await user.selectOptions(screen.getByLabelText('Compare by'), 'targeted_harm_category') + expect(screen.getByText('Choose different heatmap row and column dimensions.')).toBeVisible() + await user.click(screen.getByRole('button', { name: 'Browser back' })) + expect(screen.getByLabelText('Chart view')).toHaveValue('outcomes') + expect(screen.queryByText('Choose different heatmap row and column dimensions.')).not.toBeInTheDocument() + }) + + it('should not revive an old group offset after an axis round trip', async () => { + const user = userEvent.setup() + mockQuery.mockImplementation(async (query: AttackAnalyticsQuery) => makeAnalyticsReport({ + ...echoQuery(query), has_more_groups: true, next_group_offset: 15, + })) + renderPage() + await user.click(await screen.findByRole('button', { name: 'Next groups' })) + await waitFor(() => { expect(screen.getByRole('button', { name: 'First groups' })).toBeEnabled() }) + await user.selectOptions(screen.getByLabelText('Group by'), 'model') + await waitFor(() => { expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ + group_by: { name: 'model' }, group_offset: 0, + }), expect.any(AbortSignal)) }) + await user.click(screen.getByRole('button', { name: 'Browser back' })) + await waitFor(() => { expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ + group_by: { name: 'operation' }, group_offset: 0, + }), expect.any(AbortSignal)) }) + expect(mockQuery).toHaveBeenCalledTimes(4) + }) + + it('should restore filter and chart choices with browser Back', async () => { + const user = userEvent.setup() + renderPage() + await user.click(await screen.findByRole('button', { name: 'Filter to success: 4' })) + await user.selectOptions(screen.getByLabelText('Chart view'), 'success-rate') + expect(screen.getByLabelText('Chart view')).toHaveValue('success-rate') + await user.click(screen.getByRole('button', { name: 'Browser back' })) + expect(screen.getByLabelText('Chart view')).toHaveValue('outcomes') + expect(screen.getByRole('button', { name: 'Remove outcome success' })).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Browser back' })) + expect(screen.queryByRole('button', { name: 'Remove outcome success' })).not.toBeInTheDocument() + await waitFor(() => { expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ filters: DEFAULT_ANALYTICS_VIEW.filters }), expect.any(AbortSignal)) }) + }) + + it('should invalidate a stale filter editor when browser Back restores a different predicate', async () => { + const user = userEvent.setup() + renderPage() + await user.click(await screen.findByRole('button', { name: 'Filter to Nightly' })) + await user.click(screen.getByRole('button', { name: /^Operation \(ANY\): Nightly/ })) + expect(screen.getByRole('region', { name: 'Filter editor' })).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Browser back' })) + expect(screen.queryByRole('region', { name: 'Filter editor' })).not.toBeInTheDocument() + expect(screen.getByText(/The edited filter changed/)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Operation' })) + expect(screen.queryByRole('group', { name: 'Selected filter values' })).not.toBeInTheDocument() + }) + + it('should browse every group with server offsets and not mistake a group page for a results page', async () => { + const user = userEvent.setup() + mockQuery.mockResolvedValue(makeAnalyticsReport({ has_more_groups: true, next_group_offset: 15 })) + renderPage() + await screen.findByRole('region', { name: 'Outcome breakdown by Operation' }) + expect(screen.queryByRole('button', { name: 'Show all groups' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Show aggregate data' })).not.toBeInTheDocument() + expect(screen.getByRole('table', { name: 'Saved attack results' })).toBeInTheDocument() + expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ group_limit: 15, group_offset: 0 }), expect.any(AbortSignal)) + await waitFor(() => { expect(screen.getByRole('button', { name: 'Next groups' })).toBeEnabled() }) + await user.click(screen.getByRole('button', { name: 'Next groups' })) + await waitFor(() => { expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ group_limit: 15, group_offset: 15 }), expect.any(AbortSignal)) }) + await waitFor(() => { expect(screen.getByRole('button', { name: 'First groups' })).toBeEnabled() }) + await user.click(screen.getByRole('button', { name: 'First groups' })) + await waitFor(() => { expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ group_limit: 15, group_offset: 0 }), expect.any(AbortSignal)) }) + expect(mockResults).not.toHaveBeenCalled() + }) + + it('should reset group pagination on Reload while keeping a failed refresh coherent', async () => { + const user = userEvent.setup() + mockQuery.mockResolvedValue(makeAnalyticsReport({ has_more_groups: true, next_group_offset: 15 })) + renderPage() + await user.click(await screen.findByRole('button', { name: 'Next groups' })) + await waitFor(() => { expect(screen.getByRole('button', { name: 'First groups' })).toBeEnabled() }) + mockQuery.mockRejectedValueOnce(new Error('Reload deadline exceeded')) + await user.click(screen.getByRole('button', { name: 'Reload' })) + await screen.findByText(/Reload failed.*data may be stale/) + expect(mockQuery).toHaveBeenLastCalledWith(expect.objectContaining({ group_offset: 0 }), expect.any(AbortSignal)) + expect(screen.getByText('Inspect a saved response')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'First groups' })).toBeEnabled() + }) + + it('should discard an aborted results request even when the superseding group request fails', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const pending = deferred>() + mockQuery.mockResolvedValueOnce(makeAnalyticsReport({ + has_more_groups: true, next_group_offset: 15, + results: makeAnalyticsResults({ has_more: true, next_cursor: 'page-2' }), + })).mockRejectedValueOnce(new Error('Group page timed out')) + mockResults.mockReturnValueOnce(pending.promise) + renderPage() + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + const results = screen.getByRole('region', { name: 'Matching AttackResults' }) + await user.click(within(results).getByRole('button', { name: 'Next' })) + const signal = mockResults.mock.calls[0][1] + await user.click(screen.getByRole('button', { name: 'Next groups' })) + await act(async () => { await jest.advanceTimersByTimeAsync(ANALYTICS_DEBOUNCE_MS) }) + expect(signal?.aborted).toBe(true) + expect(screen.getByText(/Reload failed.*data may be stale/)).toBeInTheDocument() + expect(within(results).getByText('Page 1')).toBeInTheDocument() + expect(within(results).queryByText('Loading results page')).not.toBeInTheDocument() + expect(within(results).getByRole('button', { name: 'Next' })).toBeEnabled() + await act(async () => { + pending.resolve(makeAnalyticsResults({ items: [makeAnalyticsRow({ objective_preview: 'Obsolete page' })] })) + }) + expect(screen.queryByText('Obsolete page')).not.toBeInTheDocument() + }) + + it.each(['error', 'undetermined'] as const)('should show unavailable ASR* for %s-only results', async (outcome: 'error' | 'undetermined') => { + mockQuery.mockResolvedValue(makeAnalyticsReport({ + outcome_filter_applied: true, + summary: { ...ANALYTICS_EMPTY_STATISTICS, total_results: 2, [outcome === 'error' ? 'errors' : 'undetermined']: 2 }, + })) + renderPage({ ...DEFAULT_ANALYTICS_VIEW, filters: { dimensions: [], outcomes: [outcome] } }) + const summary = await screen.findByRole('region', { name: 'Outcome summary' }) + expect(within(summary).getByText('Unavailable*')).toBeInTheDocument() + expect(within(summary).queryByText('0%')).not.toBeInTheDocument() + expect(screen.getByRole('note', { name: 'Outcome-filtered ASR' })).toHaveTextContent(ANALYTICS_ASR_NOTE) + expect(screen.getByRole('note', { name: 'Outcome-filtered ASR' })).toBeVisible() + expect(within(summary).getByRole('button', { name: /ASR\*:.*Outcome filter active/ })).toBeInTheDocument() + }) + + it.each([false, true])('should distinguish empty storage from a filtered empty cohort (%s)', async (filtered: boolean) => { + mockQuery.mockResolvedValue(makeAnalyticsReport({ + summary: ANALYTICS_EMPTY_STATISTICS, groups: [], results: makeAnalyticsResults({ items: [] }), + })) + renderPage(filtered ? { ...DEFAULT_ANALYTICS_VIEW, filters: { dimensions: [ANALYTICS_OPERATION_FILTER], outcomes: [] } } : undefined) + expect(await screen.findByRole('heading', { name: filtered ? 'No results match these filters' : 'No saved AttackResults yet' })).toBeInTheDocument() + }) + + it('should render backend warnings as text and refuse malformed shared URLs', async () => { + const warning = ' SQLite reads may block writes.' + mockQuery.mockResolvedValue(makeAnalyticsReport({ warnings: [warning] })) + const { unmount } = renderPage() + expect(await screen.findByText(warning)).toBeInTheDocument() + expect(screen.queryByRole('img')).not.toBeInTheDocument() + unmount() + mockQuery.mockClear() + render( + + ) + expect(screen.getByText(/Cannot restore this analytics link/)).toBeInTheDocument() + expect(mockQuery).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/components/Analytics/AnalyticsPage.tsx b/frontend/src/components/Analytics/AnalyticsPage.tsx new file mode 100644 index 0000000000..370f26805b --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsPage.tsx @@ -0,0 +1,262 @@ +import { useEffect, useMemo, useState } from 'react' + +import { + Button, Field, MessageBar, MessageBarBody, Select, Skeleton, SkeletonItem, Text, +} from '@fluentui/react-components' +import type { SelectOnChangeData } from '@fluentui/react-components' +import { ArrowSyncRegular } from '@fluentui/react-icons' +import { useLocation, useNavigate } from 'react-router' + +import { ErrorBoundary } from '@/components/ErrorBoundary' +import { useAttackAnalytics } from '@/hooks/useAttackAnalytics' +import { useRuntime } from '@/hooks/useRuntime' +import type { AttackAnalyticsDimension, AttackAnalyticsFilter, AttackAnalyticsFilters, AttackAnalyticsViewState, AttackOutcome } from '@/types' +import { + ANALYTICS_ASR_NOTE, DEFAULT_ANALYTICS_VIEW, analyticsAsrDefinition, + analyticsDimensionKey, analyticsQuery, analyticsQueryKey, analyticsViewError, analyticsViewFromLocation, + analyticsViewToHash, appendAnalyticsDrilldown, formatAnalyticsTime, hasAnalyticsFilters, +} from '@/utils/attackAnalytics' + +import AnalyticsDimensionPicker from './AnalyticsDimensionPicker' +import AnalyticsFilters from './AnalyticsFilters' +import AnalyticsGroups from './AnalyticsGroups' +import AnalyticsHeatmap from './AnalyticsHeatmap' +import AnalyticsResultsTable from './AnalyticsResultsTable' +import AnalyticsStats from './AnalyticsStats' +import { useAnalyticsPageStyles } from './AnalyticsPage.styles' + +interface AnalyticsPageProps { + readonly onOpenAttack: (attackResultId: string) => void +} + +interface GroupPageState { + readonly query: { readonly key: string } + readonly offset: number +} + +/** + * Coordinates URL-owned filters/presentation and local pagination. SDK reports + * supply every aggregate; child charts only render them or return drill-down + * predicates. Filter drafts and cursors are deliberately not shareable URL state. + */ +export default function AnalyticsPage({ onOpenAttack }: AnalyticsPageProps) { + const styles = useAnalyticsPageStyles() + const location = useLocation() + const navigate = useNavigate() + const runtime = useRuntime() + const parsed = useMemo(() => analyticsViewFromLocation({ search: location.search, hash: location.hash }), + [location.search, location.hash]) + const view = parsed.view + // Presentation-only controls must not reset the group page. The first-page + // server query identifies this scope independently of chart colors/rendering. + const groupQueryKey = JSON.stringify([analyticsQueryKey(analyticsQuery(view)), runtime.generation]) + const groupQuery = useMemo(() => ({ key: groupQueryKey }), [groupQueryKey]) + const [groupPage, setGroupPage] = useState(null) + const [rejectedEdit, setRejectedEdit] = useState<{ locationKey: string; message: string } | null>(null) + const interactionError = rejectedEdit?.locationKey === location.key ? rejectedEdit.message : null + const offset = groupPage?.query === groupQuery ? groupPage.offset : 0 + const query = parsed.error ? null : analyticsQuery(view, offset) + const analytics = useAttackAnalytics(query) + const report = analytics.report + const filtersActive = hasAnalyticsFilters(view.filters) + + useEffect(() => { + if (!parsed.error && new URLSearchParams(location.search).has('analytics') && !location.hash) { + void navigate({ pathname: location.pathname, search: '', hash: analyticsViewToHash(view) }, { replace: true }) + } + }, [parsed.error, view, location.pathname, location.search, location.hash, navigate]) + + /** Validate before writing browser history so rejected edits keep the current cohort intact. */ + function changeView(next: AttackAnalyticsViewState): boolean { + const error = analyticsViewError(next) + setRejectedEdit(error ? { locationKey: location.key, message: error } : null) + if (error) return false + const hash = analyticsViewToHash(next) + if (hash !== location.hash || location.search) { + void navigate({ pathname: location.pathname, search: '', hash }) + } + return true + } + + function changeFilters(filters: AttackAnalyticsFilters): boolean { + return changeView({ ...view, filters }) + } + + function drilldown(predicates: AttackAnalyticsFilter[], outcome?: AttackOutcome): void { + changeFilters(appendAnalyticsDrilldown(view.filters, predicates, outcome)) + } + + function reload(): void { + setGroupPage({ query: groupQuery, offset: 0 }) + analytics.reload() + } + + function changeGroupPage(nextOffset: number): void { + setGroupPage({ query: groupQuery, offset: nextOffset }) + // After a failure, the requested offset can differ from the still-visible + // report's offset. Selecting that same request again must actually retry it. + if (offset === nextOffset && analytics.error) analytics.reload() + } + + function changeDimension( + field: 'groupBy' | 'heatmapRow' | 'heatmapColumn', + dimension: AttackAnalyticsDimension | null, + ): void { + if (dimension) changeView({ ...view, [field]: dimension }) + } + + return ( +
+
+ Analytics +
+ + Last refreshed: {report + ? + : 'Not loaded'} + + +
+
+
+ {parsed.error ? ( + + {parsed.error}{' '} + + + ) : ( + <> + + Every saved AttackResult ID, across operations, targets, and result roles. + Scenario filters retain this population, including retries. Filters apply to the entire dashboard. + + + {interactionError && {interactionError}} +
+ + + + + + + {view.chart === 'heatmap' ? ( + <> + { changeDimension('heatmapRow', dimension) }} /> + { changeDimension('heatmapColumn', dimension) }} /> + + + + + ) : ( + { changeDimension('groupBy', dimension) }} /> + )} +
+ {!runtime.ready && + Analytics is waiting for the runtime to be ready. Your filters are preserved. + } + {analytics.error && ( + + {report ? 'Reload failed. Showing the last successful report; data may be stale.' : 'Could not load analytics for these filters.'} + {' '}{analytics.error}{' '} + + + )} + {analytics.loading && ( +
+ {report ? 'Reloading saved results...' : 'Loading analytics for current filters...'} + {!report && + + } +
+ )} + {report && ( + <> + {report.warnings.map((warning: string) => ( + {warning} + ))} + { changeFilters({ ...view.filters, outcomes: [outcome] }) }} /> + + {report.outcome_filter_applied && `${ANALYTICS_ASR_NOTE} `}{analyticsAsrDefinition(view.denominator)} + + {report.summary.total_results === 0 ? ( +
+ + {filtersActive ? 'No results match these filters' : 'No saved AttackResults yet'} + + {filtersActive + ? 'Remove a filter or widen the last updated range to explore other saved results.' + : 'Analytics reads persisted results only. Save attack results, then use Reload to see them here.'} + {filtersActive && } +
+ ) : ( + +
+ {report.groups_overlap && + Groups overlap: a result can appear in multiple groups or cells. Do not add their totals together. + } + {view.chart === 'heatmap' + ? + : <> + +
+ + + {report.has_more_groups && More groups are available. No combined "Other" bucket is used.} +
+ } +
+ + )} + {analytics.results && } + + )} + + )} +
+
+ ) +} diff --git a/frontend/src/components/Analytics/AnalyticsResultsTable.styles.ts b/frontend/src/components/Analytics/AnalyticsResultsTable.styles.ts new file mode 100644 index 0000000000..8fc8aeeda6 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsResultsTable.styles.ts @@ -0,0 +1,25 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { MINIMUM_TOUCH_TARGET_SIZE, mobileTouchTargetHeight, TOUCH_INPUT_QUERY } from '@/styles/touchTargets' + +export const useAnalyticsResultsTableStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalS, minWidth: 0 }, + heading: { margin: 0 }, + header: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', justifyContent: 'space-between', gap: tokens.spacingHorizontalM }, + scroll: { overflow: 'auto', maxHeight: '28rem', minWidth: 0 }, + table: { minWidth: '65rem', tableLayout: 'auto' }, + clickableRow: { + cursor: 'pointer', + ':hover': { backgroundColor: tokens.colorNeutralBackground1Hover }, + ':focus-visible': { + outline: `2px solid ${tokens.colorStrokeFocus2}`, + outlineOffset: '-2px', + }, + [TOUCH_INPUT_QUERY]: { height: MINIMUM_TOUCH_TARGET_SIZE }, + }, + objective: { minWidth: '15rem', maxWidth: '25rem', overflowWrap: 'anywhere' }, + metadata: { maxWidth: '18rem', overflowWrap: 'anywhere' }, + secondary: { display: 'block', color: tokens.colorNeutralForeground2, overflowWrap: 'anywhere' }, + date: { whiteSpace: 'nowrap' }, + button: { ...mobileTouchTargetHeight }, +}) diff --git a/frontend/src/components/Analytics/AnalyticsResultsTable.test.tsx b/frontend/src/components/Analytics/AnalyticsResultsTable.test.tsx new file mode 100644 index 0000000000..8c05f9fbe1 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsResultsTable.test.tsx @@ -0,0 +1,78 @@ +import type { ComponentProps, ReactNode } from 'react' + +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { makeAnalyticsResults, makeAnalyticsRow } from '@/test-utils/analyticsFixtures' + +import AnalyticsResultsTable from './AnalyticsResultsTable' + +function TestWrapper({ children }: { readonly children: ReactNode }) { + return {children} +} + +function renderTable(overrides: Partial> = {}) { + const props: ComponentProps = { + results: makeAnalyticsResults(), + totalResults: 10, + page: 0, + loading: false, + disabled: false, + error: null, + onFirst: jest.fn(), + onNext: jest.fn(), + onRetry: jest.fn(), + onOpenAttack: jest.fn(), + ...overrides, + } + render() + return props +} + +describe('AnalyticsResultsTable', () => { + beforeEach(() => { jest.clearAllMocks() }) + + it('opens the correct result when any cell in its row is clicked', async () => { + const user = userEvent.setup() + const { onOpenAttack } = renderTable({ + results: makeAnalyticsResults({ + items: [makeAnalyticsRow(), makeAnalyticsRow({ attack_result_id: 'result-2', operator: 'Bob' })], + }), + }) + const row = screen.getByRole('row', { name: 'Open result result-2' }) + await user.click(within(row).getByRole('cell', { name: 'Bob' })) + expect(onOpenAttack).toHaveBeenCalledTimes(1) + expect(onOpenAttack).toHaveBeenCalledWith('result-2') + expect(screen.queryByRole('button', { name: /^Open result / })).not.toBeInTheDocument() + expect(screen.queryByRole('columnheader', { name: 'Open' })).not.toBeInTheDocument() + }) + + it.each(['{Enter}', ' '])('opens a focused row using %s', async (key: string) => { + const user = userEvent.setup() + const { onOpenAttack } = renderTable() + const row = screen.getByRole('row', { name: 'Open result result-1' }) + row.focus() + expect(row).toHaveFocus() + expect(row).toHaveAccessibleDescription('Inspect a saved response') + await user.keyboard(key) + expect(onOpenAttack).toHaveBeenCalledTimes(1) + expect(onOpenAttack).toHaveBeenCalledWith('result-1') + }) + + it('does not activate a row for navigation keys', async () => { + const user = userEvent.setup() + const { onOpenAttack } = renderTable() + screen.getByRole('row', { name: 'Open result result-1' }).focus() + await user.keyboard('{ArrowDown}') + expect(onOpenAttack).not.toHaveBeenCalled() + }) + + it('keeps the retry control separate from row navigation', async () => { + const user = userEvent.setup() + const { onRetry, onOpenAttack } = renderTable({ error: 'Retry this request.' }) + await user.click(screen.getByRole('button', { name: 'Retry results' })) + expect(onRetry).toHaveBeenCalledTimes(1) + expect(onOpenAttack).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/components/Analytics/AnalyticsResultsTable.tsx b/frontend/src/components/Analytics/AnalyticsResultsTable.tsx new file mode 100644 index 0000000000..5656543363 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsResultsTable.tsx @@ -0,0 +1,114 @@ +import type { KeyboardEvent } from 'react' + +import { + Button, MessageBar, MessageBarBody, Spinner, Table, TableBody, + TableCell, TableHeader, TableHeaderCell, TableRow, Text, +} from '@fluentui/react-components' + +import HistoryPagination from '@/components/History/HistoryPagination' +import OutcomeBadge from '@/components/OutcomeBadge' +import type { AttackAnalyticsResultRow, AttackAnalyticsResults } from '@/types' +import { formatAnalyticsCount, formatAnalyticsTime } from '@/utils/attackAnalytics' + +import { useAnalyticsResultsTableStyles } from './AnalyticsResultsTable.styles' + +interface AnalyticsResultsTableProps { + readonly results: AttackAnalyticsResults + readonly totalResults: number + readonly page: number + readonly loading: boolean + readonly disabled: boolean + readonly error: string | null + readonly invalidCursor?: boolean + readonly onFirst: () => void + readonly onNext: () => void + readonly onRetry: () => void + readonly onOpenAttack: (attackResultId: string) => void +} + +/** + * Present one independent rows-only read alongside the report's cohort size. + * Failed page requests keep the last displayed page/time. Row activation delegates + * to the normal attack loader so analytics never bypasses its read-only guards. + */ +export default function AnalyticsResultsTable({ + results, totalResults, page, loading, disabled, error, invalidCursor = false, onFirst, onNext, onRetry, onOpenAttack, +}: AnalyticsResultsTableProps) { + const styles = useAnalyticsResultsTableStyles() + return ( +
+
+ Matching AttackResults + {formatAnalyticsCount(totalResults)} results in the report +
+ {error && ( + + Could not load the requested results page. Still showing page {page + 1}. {error}{' '} + + + )} + {loading && } +
+ + + OutcomeObjective + OperationOperator + Attack type / targeted harmsTarget model / identity + Last updated + + + {results.items.map((result: AttackAnalyticsResultRow) => ( + { onOpenAttack(result.attack_result_id) }} + onKeyDown={(event: KeyboardEvent) => { + if (event.target === event.currentTarget && (event.key === 'Enter' || event.key === ' ')) { + event.preventDefault() + onOpenAttack(result.attack_result_id) + } + }} + > + + + + {result.objective_preview || '(No objective recorded)'} + + {result.attack_result_id} + + {result.operation ?? '(Missing operation)'} + {result.operator ?? '(Missing operator)'} + + {result.attack_type ?? '(Missing attack type)'} + {result.targeted_harm_categories.join(', ') || '(No targeted harms recorded)'} + + + {result.target_model ?? '(Missing model)'} + {result.target_identifier_hash ?? '(Missing target identity)'} + + + + ))} + +
+
+ {results.items.length === 0 && No results on this page. Return to the first page or reload the report.} + + Page read: . + {' '}Each page is a fresh read; saved results may change between requests. + + +
+ ) +} diff --git a/frontend/src/components/Analytics/AnalyticsStats.styles.ts b/frontend/src/components/Analytics/AnalyticsStats.styles.ts new file mode 100644 index 0000000000..15f5796a0c --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsStats.styles.ts @@ -0,0 +1,37 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' + +export const useAnalyticsStatsStyles = makeStyles({ + root: { + display: 'flex', + alignItems: 'stretch', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalXXL, + paddingBlock: tokens.spacingVerticalM, + borderTop: `1px solid ${tokens.colorNeutralStroke1}`, + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + [NARROW_VIEWPORT_QUERY]: { gap: tokens.spacingHorizontalM }, + }, + metric: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXS, + justifyContent: 'center', + }, + number: { fontVariantNumeric: 'tabular-nums' }, + outcomes: { + display: 'flex', + flex: 1, + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + }, + button: { columnGap: tokens.spacingHorizontalXS, ...mobileTouchTargetHeight }, + asrLabel: { + paddingInline: tokens.spacingHorizontalXS, + minWidth: 0, + justifyContent: 'flex-start', + ...mobileTouchTargetHeight, + }, +}) diff --git a/frontend/src/components/Analytics/AnalyticsStats.tsx b/frontend/src/components/Analytics/AnalyticsStats.tsx new file mode 100644 index 0000000000..3cd8f48093 --- /dev/null +++ b/frontend/src/components/Analytics/AnalyticsStats.tsx @@ -0,0 +1,88 @@ +import { Button, Text, Tooltip } from '@fluentui/react-components' +import { InfoRegular } from '@fluentui/react-icons' + +import OutcomeBadge from '@/components/OutcomeBadge' +import type { AttackAnalyticsDenominator, AttackAnalyticsStatistics, AttackOutcome } from '@/types' +import { + ANALYTICS_ASR_NOTE, analyticsAsrDefinition, analyticsSuccessCountsLabel, analyticsSuccessRate, + formatAnalyticsCount, formatAnalyticsPercent, +} from '@/utils/attackAnalytics' + +import { useAnalyticsStatsStyles } from './AnalyticsStats.styles' + +interface AnalyticsAsrLabelProps { + readonly marked: boolean + readonly denominator?: AttackAnalyticsDenominator +} + +/** A focusable explanation shared by summary/charts; the SDK's outcome-filter flag controls the asterisk. */ +export function AnalyticsAsrLabel({ marked, denominator = 'decided' }: AnalyticsAsrLabelProps) { + const styles = useAnalyticsStatsStyles() + const explanation = `${marked ? `${ANALYTICS_ASR_NOTE} ` : ''}${analyticsAsrDefinition(denominator)}` + return ( + + + + ) +} + +interface AnalyticsStatsProps { + readonly statistics: AttackAnalyticsStatistics + readonly marked: boolean + readonly denominator?: AttackAnalyticsDenominator + readonly onOutcome: (outcome: AttackOutcome) => void +} + +/** Display SDK population statistics; outcome buttons change the whole cohort, not just this summary. */ +export default function AnalyticsStats({ statistics, marked, denominator = 'decided', onOutcome }: AnalyticsStatsProps) { + const styles = useAnalyticsStatsStyles() + const outcomes: Array<{ outcome: AttackOutcome; count: number }> = [ + { outcome: 'success', count: statistics.successes }, + { outcome: 'failure', count: statistics.failures }, + { outcome: 'error', count: statistics.errors }, + { outcome: 'undetermined', count: statistics.undetermined }, + ] + return ( +
+
+ Total results + {formatAnalyticsCount(statistics.total_results)} +
+
+ + + {formatAnalyticsPercent(analyticsSuccessRate(statistics, denominator))}{marked ? '*' : ''} + + {analyticsSuccessCountsLabel(statistics, denominator)} + {formatAnalyticsCount(statistics.total_results)} total +
+
+ Decided share + {formatAnalyticsPercent(statistics.decided_share)} + {formatAnalyticsCount(statistics.total_decided)} decided / {formatAnalyticsCount(statistics.total_results)} total +
+
+ {outcomes.map((item: { outcome: AttackOutcome; count: number }) => ( + + ))} +
+
+ ) +} diff --git a/frontend/src/components/AttackResults/OutcomeSummaryBar.styles.ts b/frontend/src/components/AttackResults/OutcomeSummaryBar.styles.ts new file mode 100644 index 0000000000..c0e1f10c3e --- /dev/null +++ b/frontend/src/components/AttackResults/OutcomeSummaryBar.styles.ts @@ -0,0 +1,39 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { OUTCOME_PALETTE } from '@/styles/outcomePalette' +import { mobileTouchTargetHeight } from '@/styles/touchTargets' + +function outcomeFill(color: string) { + return { + backgroundColor: color, + ':hover': { backgroundColor: color }, + ':active': { backgroundColor: color }, + } +} + +export const useOutcomeSummaryBarStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalXS, minWidth: 0 }, + bar: { + display: 'flex', + width: '100%', + height: tokens.spacingVerticalL, + border: `1px solid ${tokens.colorNeutralStroke1}`, + backgroundColor: tokens.colorNeutralBackground4, + ...mobileTouchTargetHeight, + }, + segment: { + minWidth: 0, + height: '100%', + padding: 0, + border: 0, + borderRadius: 0, + '@media (forced-colors: active)': { borderRight: '1px solid currentColor' }, + }, + legend: { display: 'flex', flexWrap: 'wrap', gap: tokens.spacingHorizontalXS }, + swatch: { width: tokens.spacingHorizontalS, height: tokens.spacingVerticalS, border: '1px solid currentColor' }, + button: { columnGap: tokens.spacingHorizontalXS, ...mobileTouchTargetHeight }, + success: outcomeFill(OUTCOME_PALETTE.success.color), + failure: outcomeFill(OUTCOME_PALETTE.failure.color), + error: outcomeFill(OUTCOME_PALETTE.error.color), + undetermined: outcomeFill(OUTCOME_PALETTE.undetermined.color), +}) diff --git a/frontend/src/components/AttackResults/OutcomeSummaryBar.test.tsx b/frontend/src/components/AttackResults/OutcomeSummaryBar.test.tsx new file mode 100644 index 0000000000..7f90b2e839 --- /dev/null +++ b/frontend/src/components/AttackResults/OutcomeSummaryBar.test.tsx @@ -0,0 +1,32 @@ +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { ANALYTICS_TEST_STATISTICS } from '@/test-utils/analyticsFixtures' + +import OutcomeSummaryBar from './OutcomeSummaryBar' + +const TestWrapper: React.FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +) + +describe('OutcomeSummaryBar', () => { + beforeEach(() => { jest.clearAllMocks() }) + + it('should use supplied statistics and make a tiny outcome available through a labeled action', async () => { + const user = userEvent.setup() + const onOutcome = jest.fn() + render( + + ) + await user.click(screen.getByRole('button', { name: 'Synthetic run: 1 error; filter dashboard' })) + expect(onOutcome).toHaveBeenCalledWith('error') + expect(screen.getByRole('button', { name: 'Synthetic run: success segment; 4 results; filter dashboard' })).toBeVisible() + }) + + it('should show unavailable historical statistics without inferring counts from retries or errors', () => { + render() + expect(screen.getByText(/Outcome breakdown unavailable/)).toBeVisible() + expect(screen.queryByRole('button')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/AttackResults/OutcomeSummaryBar.tsx b/frontend/src/components/AttackResults/OutcomeSummaryBar.tsx new file mode 100644 index 0000000000..e4771a5402 --- /dev/null +++ b/frontend/src/components/AttackResults/OutcomeSummaryBar.tsx @@ -0,0 +1,70 @@ +import { Button, mergeClasses, Text, Tooltip } from '@fluentui/react-components' + +import type { AttackOutcome, OutcomeStatistics } from '@/types' +import { formatAnalyticsCount, formatAnalyticsPercent } from '@/utils/attackAnalytics' + +import { useOutcomeSummaryBarStyles } from './OutcomeSummaryBar.styles' + +interface OutcomeSummaryBarProps { + readonly statistics: OutcomeStatistics | null + readonly label: string + readonly showBar?: boolean + readonly disabled?: boolean + readonly disabledReason?: string | null + readonly describedBy?: string + readonly onOutcome?: (outcome: AttackOutcome) => void +} + +/** Presentation only: callers supply SDK statistics, never paginated result rows. */ +export default function OutcomeSummaryBar({ + statistics, label, showBar = true, disabled = false, disabledReason, describedBy, onOutcome, +}: OutcomeSummaryBarProps) { + const styles = useOutcomeSummaryBarStyles() + if (statistics === null) return Outcome breakdown unavailable for this historical result. + const outcomes: Array<{ outcome: AttackOutcome; count: number }> = [ + { outcome: 'success', count: statistics.successes }, + { outcome: 'failure', count: statistics.failures }, + { outcome: 'error', count: statistics.errors }, + { outcome: 'undetermined', count: statistics.undetermined }, + ] + const inactive = disabled || !onOutcome + return ( +
+ {showBar && ( +
+ {outcomes.map(({ outcome, count }: { outcome: AttackOutcome; count: number }) => + statistics.outcome_shares[outcome] > 0 && ( + +
+ )} +
+ {outcomes.map(({ outcome, count }: { outcome: AttackOutcome; count: number }) => ( + + + + ))} +
+
+ ) +} diff --git a/frontend/src/components/History/HistoryPagination.test.tsx b/frontend/src/components/History/HistoryPagination.test.tsx index 1acda1a284..6b3bcd74b3 100644 --- a/frontend/src/components/History/HistoryPagination.test.tsx +++ b/frontend/src/components/History/HistoryPagination.test.tsx @@ -1,5 +1,6 @@ import { render, screen, fireEvent } from '@testing-library/react' import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import userEvent from '@testing-library/user-event' import HistoryPagination from './HistoryPagination' jest.mock('./AttackHistory.styles', () => ({ @@ -22,6 +23,19 @@ describe('HistoryPagination', () => { jest.clearAllMocks() }) + it('should disable both cursor actions during a read', async () => { + const user = userEvent.setup() + render() + const first = screen.getByRole('button', { name: 'First' }) + const next = screen.getByRole('button', { name: 'Next' }) + expect(first).toBeDisabled() + expect(next).toBeDisabled() + await user.click(first) + await user.click(next) + expect(defaultProps.onPrevPage).not.toHaveBeenCalled() + expect(defaultProps.onNextPage).not.toHaveBeenCalled() + }) + it('should render page number (1-indexed)', () => { render( diff --git a/frontend/src/components/History/HistoryPagination.tsx b/frontend/src/components/History/HistoryPagination.tsx index 635e7d3898..2fdf6ac8d8 100644 --- a/frontend/src/components/History/HistoryPagination.tsx +++ b/frontend/src/components/History/HistoryPagination.tsx @@ -13,9 +13,10 @@ interface HistoryPaginationProps { isLastPage: boolean onPrevPage: () => void onNextPage: () => void + disabled?: boolean } -export default function HistoryPagination({ page, isLastPage, onPrevPage, onNextPage }: HistoryPaginationProps) { +export default function HistoryPagination({ page, isLastPage, onPrevPage, onNextPage, disabled = false }: HistoryPaginationProps) { const styles = useAttackHistoryStyles() return ( @@ -24,7 +25,7 @@ export default function HistoryPagination({ page, isLastPage, onPrevPage, onNext className={styles.touchTargetHeight} appearance="subtle" icon={} - disabled={page === 0} + disabled={disabled || page === 0} onClick={onPrevPage} data-testid="prev-page-btn" > @@ -36,7 +37,7 @@ export default function HistoryPagination({ page, isLastPage, onPrevPage, onNext appearance="subtle" icon={} iconPosition="after" - disabled={isLastPage} + disabled={disabled || isLastPage} onClick={onNextPage} data-testid="next-page-btn" > diff --git a/frontend/src/components/Home/Home.tsx b/frontend/src/components/Home/Home.tsx index 520331c50f..82f9b9f0d4 100644 --- a/frontend/src/components/Home/Home.tsx +++ b/frontend/src/components/Home/Home.tsx @@ -6,19 +6,15 @@ import { MessageBarBody, Spinner, Text, - tokens, } from '@fluentui/react-components' import { ArrowRightRegular, - CheckmarkCircleRegular, - DismissCircleRegular, - ErrorCircleRegular, - QuestionCircleRegular, TagMultipleRegular, TargetRegular, } from '@fluentui/react-icons' import { attacksApi } from '../../services/api' import { toApiError } from '../../services/errors' +import { OutcomeIcon } from '@/components/OutcomeBadge' import type { AttackSummary, TargetInstance } from '../../types' import { targetEndpoint, targetModelName, targetType } from '../../utils/targetIdentity' import type { ViewName } from '../Sidebar/Navigation' @@ -29,13 +25,6 @@ const MAX_OPERATIONS = 5 const MAX_ATTACKS_PER_OPERATION = 3 const NO_OPERATION_KEY = '__no_operation__' -const OUTCOME_ICONS: Record = { - success: , - failure: , - error: , - undetermined: , -} - interface HomeProps { activeTarget: TargetInstance | null onNavigate: (view: ViewName) => void @@ -264,7 +253,7 @@ export default function Home({ data-testid={`home-open-attack-${attack.attack_result_id}`} title={attack.last_message_preview || attack.attack_type} > - {OUTCOME_ICONS[attack.outcome ?? 'undetermined'] ?? OUTCOME_ICONS.undetermined} + {attack.last_message_preview || attack.attack_type} diff --git a/frontend/src/components/OutcomeBadge.styles.ts b/frontend/src/components/OutcomeBadge.styles.ts new file mode 100644 index 0000000000..361965fdb5 --- /dev/null +++ b/frontend/src/components/OutcomeBadge.styles.ts @@ -0,0 +1,47 @@ +import { makeStyles, shorthands, tokens } from '@fluentui/react-components' + +import { OUTCOME_PALETTE } from '@/styles/outcomePalette' + +export const useOutcomeBadgeStyles = makeStyles({ + success: { + '--pyrit-outcome-color': OUTCOME_PALETTE.success.color, + '--pyrit-outcome-tint': OUTCOME_PALETTE.success.tint, + }, + failure: { + '--pyrit-outcome-color': OUTCOME_PALETTE.failure.color, + '--pyrit-outcome-tint': OUTCOME_PALETTE.failure.tint, + }, + error: { + '--pyrit-outcome-color': OUTCOME_PALETTE.error.color, + '--pyrit-outcome-tint': OUTCOME_PALETTE.error.tint, + }, + undetermined: { + '--pyrit-outcome-color': OUTCOME_PALETTE.undetermined.color, + '--pyrit-outcome-tint': OUTCOME_PALETTE.undetermined.tint, + }, + filled: { + backgroundColor: 'var(--pyrit-outcome-color)', + ...shorthands.borderColor('var(--pyrit-outcome-color)'), + color: tokens.colorNeutralBackground1, + }, + tint: { + backgroundColor: 'var(--pyrit-outcome-tint)', + ...shorthands.borderColor('transparent'), + color: 'var(--pyrit-outcome-color)', + }, + outline: { + backgroundColor: 'transparent', + ...shorthands.borderColor('var(--pyrit-outcome-color)'), + color: 'var(--pyrit-outcome-color)', + }, + ghost: { + backgroundColor: 'transparent', + ...shorthands.borderColor('transparent'), + color: 'var(--pyrit-outcome-color)', + }, + icon: { + display: 'inline-flex', + flexShrink: 0, + color: 'var(--pyrit-outcome-color)', + }, +}) diff --git a/frontend/src/components/OutcomeBadge.test.tsx b/frontend/src/components/OutcomeBadge.test.tsx new file mode 100644 index 0000000000..7d35ef8ca1 --- /dev/null +++ b/frontend/src/components/OutcomeBadge.test.tsx @@ -0,0 +1,49 @@ +import type { ReactNode } from 'react' + +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { render, screen } from '@testing-library/react' + +import type { AttackOutcome } from '@/types' + +import OutcomeBadge, { OutcomeIcon } from './OutcomeBadge' + +function TestWrapper({ children }: { readonly children: ReactNode }) { + return {children} +} + +describe('OutcomeBadge', () => { + it.each(['success', 'failure', 'error', 'undetermined'])( + 'keeps the %s label and lets its icon inherit the badge foreground', + (outcome: AttackOutcome) => { + render() + const badge = screen.getByText(outcome, { exact: true }) + expect(badge).toBeVisible() + expect(badge.querySelector('svg')).not.toHaveAttribute('style') + }, + ) + + it.each([undefined, null])('treats a missing outcome as undetermined', (outcome: null | undefined) => { + render() + expect(screen.getByText('undetermined')).toBeVisible() + }) + + it('preserves custom labels, size, appearance, class names, and test IDs', () => { + render( + + + , + ) + expect(screen.getByTestId('verdict')).toHaveTextContent('No') + expect(screen.getByTestId('verdict')).toHaveClass('custom') + }) +}) + +describe('OutcomeIcon', () => { + it.each(['success', 'failure', 'error', 'undetermined'])( + 'provides a non-color label for the %s icon', + (outcome: AttackOutcome) => { + render() + expect(screen.getByRole('img', { name: outcome })).toBeVisible() + }, + ) +}) diff --git a/frontend/src/components/OutcomeBadge.tsx b/frontend/src/components/OutcomeBadge.tsx index 1da27b20b9..33199b74e2 100644 --- a/frontend/src/components/OutcomeBadge.tsx +++ b/frontend/src/components/OutcomeBadge.tsx @@ -1,4 +1,6 @@ -import { Badge, tokens } from '@fluentui/react-components' +import type { ReactElement } from 'react' + +import { Badge, mergeClasses } from '@fluentui/react-components' import type { BadgeProps } from '@fluentui/react-components' import { CheckmarkCircleRegular, @@ -9,26 +11,36 @@ import { import type { AttackOutcome } from '@/types' -const OUTCOME_ICONS: Record = { - success: , - failure: , - error: , - undetermined: , -} +import { useOutcomeBadgeStyles } from './OutcomeBadge.styles' -const OUTCOME_COLORS: Record = { - success: 'success', - failure: 'danger', - error: 'warning', - undetermined: 'informative', +const OUTCOME_ICONS: Record = { + success: , + failure: , + error: , + undetermined: , } interface OutcomeBadgeProps { - outcome?: AttackOutcome | null - testId?: string - className?: string - appearance?: BadgeProps['appearance'] - size?: BadgeProps['size'] + readonly outcome?: AttackOutcome | null + readonly testId?: string + readonly className?: string + readonly appearance?: BadgeProps['appearance'] + readonly size?: BadgeProps['size'] + readonly label?: string +} + +interface OutcomeIconProps { + readonly outcome?: AttackOutcome | null +} + +export function OutcomeIcon({ outcome }: OutcomeIconProps) { + const styles = useOutcomeBadgeStyles() + const normalizedOutcome = outcome ?? 'undetermined' + return ( + + {OUTCOME_ICONS[normalizedOutcome]} + + ) } export default function OutcomeBadge({ @@ -37,19 +49,20 @@ export default function OutcomeBadge({ className, appearance = 'filled', size, + label, }: OutcomeBadgeProps) { + const styles = useOutcomeBadgeStyles() const normalizedOutcome = outcome ?? 'undetermined' return ( - {normalizedOutcome} + {label ?? normalizedOutcome} ) } diff --git a/frontend/src/components/Scenarios/AttackExecutionTable.tsx b/frontend/src/components/Scenarios/AttackExecutionTable.tsx index 7924cfbd66..ab6287db19 100644 --- a/frontend/src/components/Scenarios/AttackExecutionTable.tsx +++ b/frontend/src/components/Scenarios/AttackExecutionTable.tsx @@ -1,5 +1,6 @@ -import { Badge, Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow, Text } from '@fluentui/react-components' +import { Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow, Text } from '@fluentui/react-components' +import OutcomeBadge from '@/components/OutcomeBadge' import { formatAttackSuccess, formatDuration, @@ -10,16 +11,6 @@ import type { ScenarioProgressResult } from '@/types' import { useScenarioRunPageStyles } from './ScenarioRunPage.styles' -const OUTCOME_BADGE_COLORS: Record< - ScenarioProgressResult['outcome'], - 'success' | 'danger' | 'warning' | 'informative' -> = { - success: 'success', - failure: 'danger', - error: 'warning', - undetermined: 'informative', -} - interface AttackExecutionTableProps { readonly attempts: ScenarioProgressResult[] readonly seedObjectives: Map @@ -81,9 +72,7 @@ export default function AttackExecutionTable({ {attempt.atomic_attack_name} - - {formatAttackSuccess(attempt.outcome)} - + diff --git a/frontend/src/components/Sidebar/Navigation.test.tsx b/frontend/src/components/Sidebar/Navigation.test.tsx index 6ebaa0943a..61ce8cd9d4 100644 --- a/frontend/src/components/Sidebar/Navigation.test.tsx +++ b/frontend/src/components/Sidebar/Navigation.test.tsx @@ -144,6 +144,7 @@ describe("Navigation", () => { "Home", "Technique.execute", "History", + "Analytics", "Scanner", "Datasets", "Registry", @@ -168,6 +169,16 @@ describe("Navigation", () => { expect(onNavigate).toHaveBeenCalledWith("history"); }); + it("marks Analytics current and opens the explorer", async () => { + const user = userEvent.setup(); + const onNavigate = jest.fn(); + renderWithProvider(); + const button = screen.getByRole("button", { name: "Analytics" }); + expect(button).toHaveAttribute("aria-current", "page"); + await user.click(button); + expect(onNavigate).toHaveBeenCalledWith("analytics"); + }); + it("calls onNavigate with 'scenarios' when the scenarios button is clicked", async () => { const user = userEvent.setup(); const onNavigate = jest.fn(); diff --git a/frontend/src/components/Sidebar/Navigation.tsx b/frontend/src/components/Sidebar/Navigation.tsx index ca70d163c3..eac0e0ebbe 100644 --- a/frontend/src/components/Sidebar/Navigation.tsx +++ b/frontend/src/components/Sidebar/Navigation.tsx @@ -14,6 +14,7 @@ import { HomeRegular, SettingsRegular, HistoryRegular, + DataBarVerticalRegular, PersonFeedbackRegular, ScriptRegular, TargetRegular, @@ -30,6 +31,7 @@ export type ViewName = | 'home' | 'chat' | 'history' + | 'analytics' | 'registry' | 'configuration' | 'scenarios' @@ -103,6 +105,17 @@ export default function Navigation({ onClick={() => onNavigate('history')} /> +