diff --git a/src/__tests__/CompareResults/ProfileCompare.test.tsx b/src/__tests__/CompareResults/ProfileCompare.test.tsx new file mode 100644 index 000000000..8a521321f --- /dev/null +++ b/src/__tests__/CompareResults/ProfileCompare.test.tsx @@ -0,0 +1,165 @@ +import fetchMock from '@fetch-mock/jest'; +import userEvent from '@testing-library/user-event'; + +import { ProfileCompareButton } from '../../components/CompareResults/ProfileCompare/ProfileCompareButton'; +import type { CompareResultsItem } from '../../types/state'; +import getTestData from '../utils/fixtures'; +import { render, screen, waitFor, within } from '../utils/test-utils'; + +// A minimal speedometer3 row derived from an existing fixture. We only need +// the fields that ProfileCompareButton / Dialog reads. +function makeSpeedometer3Row( + overrides: Partial = {}, +): CompareResultsItem { + const base = getTestData().testCompareData[0]; + return { + ...base, + suite: 'speedometer3', + header_name: 'browsertime speedometer3 opt', + base_repository_name: 'try', + new_repository_name: 'try', + base_rev: 'b45e818c8db40353dae549cd7235c8210c58802b', + new_rev: 'f00ba7f00ba7f00ba7f00ba7f00ba7f00ba7f00b', + base_retriggerable_job_ids: [111, 222], + new_retriggerable_job_ids: [333, 444], + base_runs: [412.3, 415.8], + new_runs: [425.5, 431.7], + ...overrides, + }; +} + +function mockJobInfo(repo: string, jobId: number, taskId: string, retryId = 0) { + fetchMock.get( + `begin:https://treeherder.mozilla.org/api/project/${repo}/jobs/${jobId}/`, + { taskcluster_metadata: { task_id: taskId, retry_id: retryId } }, + ); +} + +function mockArtifacts(taskId: string, runId: number, names: string[]) { + fetchMock.get( + `https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/${taskId}/runs/${runId}/artifacts`, + { artifacts: names.map((name) => ({ name })) }, + ); +} + +function mockTaskStatus(taskId: string, workerId: string, runId = 0) { + fetchMock.get( + `https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/${taskId}/status`, + // Wrap in `body` so fetch-mock doesn't interpret the response's top-level + // `status` key as an HTTP status code. + { body: { status: { runs: [{ runId, state: 'completed', workerId }] } } }, + ); +} + +describe('ProfileCompareButton', () => { + afterEach(() => { + fetchMock.mockReset(); + }); + + it('shows runs with worker IDs, sorted by score, with the median preselected', async () => { + // Base has three profiled runs (out-of-order scores) so we can verify + // both sort-by-score and median preselection. New has two profiled runs + // (one job with no profile is filtered out). + const row = makeSpeedometer3Row({ + base_retriggerable_job_ids: [111, 222, 555], + new_retriggerable_job_ids: [333, 444], + base_runs: [415.8, 412.3, 418.2], + new_runs: [425.5, 431.7], + }); + + mockJobInfo('try', 111, 'TASKBASE1'); + mockJobInfo('try', 222, 'TASKBASE2'); + mockJobInfo('try', 555, 'TASKBASE3'); + mockJobInfo('try', 333, 'TASKNEW1'); + mockJobInfo('try', 444, 'TASKNEW2'); + + const profile = 'public/test_info/profile_speedometer3_compact.jslb.gz'; + mockArtifacts('TASKBASE1', 0, [profile, 'public/logs/live.log']); + mockArtifacts('TASKBASE2', 0, [profile]); + mockArtifacts('TASKBASE3', 0, [profile]); + mockArtifacts('TASKNEW1', 0, [profile]); + // No profile on this run — should be filtered out. + mockArtifacts('TASKNEW2', 0, ['public/logs/live.log']); + + mockTaskStatus('TASKBASE1', 'worker-b1'); + mockTaskStatus('TASKBASE2', 'worker-b2'); + mockTaskStatus('TASKBASE3', 'worker-b3'); + mockTaskStatus('TASKNEW1', 'worker-n1'); + mockTaskStatus('TASKNEW2', 'worker-n2'); + + render(); + + const openButton = await screen.findByTitle( + 'open profile comparison for this result', + ); + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }); + await user.click(openButton); + + const dialog = await screen.findByRole('dialog'); + // Wait for the dialog to be populated. + await waitFor(() => + expect( + within(dialog).getByText(/Run 1.*score.*412\.3/), + ).toBeInTheDocument(), + ); + + // Base runs should appear sorted by score ascending: + // 412.3 (worker-b2), 415.8 (worker-b1), 418.2 (worker-b3). + // New runs should appear sorted: 425.5 (worker-n1), 431.7 — actually + // the only profiled new runs are TASKNEW1 (425.5). TASKNEW2 has no + // profile artifact so isn't shown. + const allLabels = within(dialog).getAllByText(/Run \d.*score/); + // 3 base + 1 new = 4 + expect(allLabels).toHaveLength(4); + + // Worker IDs should be visible. + expect(within(dialog).getByText(/worker-b2/)).toBeInTheDocument(); + expect(within(dialog).getByText(/worker-n1/)).toBeInTheDocument(); + + // Median of base is index 1 (415.8, worker-b1). Only one new run so + // that's preselected. The compare button should be enabled immediately. + const activeCompareBtn = within(dialog).getByRole('link', { + name: 'Open profile comparison', + }); + expect(activeCompareBtn).toHaveAttribute( + 'href', + expect.stringContaining('TASKBASE1'), + ); + expect(activeCompareBtn).toHaveAttribute( + 'href', + expect.stringContaining('TASKNEW1'), + ); + }); + + it('shows an empty-state message when no runs have profile artifacts', async () => { + mockJobInfo('try', 111, 'TASKBASE1'); + mockJobInfo('try', 222, 'TASKBASE2'); + mockJobInfo('try', 333, 'TASKNEW1'); + mockJobInfo('try', 444, 'TASKNEW2'); + + mockArtifacts('TASKBASE1', 0, ['public/logs/live.log']); + mockArtifacts('TASKBASE2', 0, ['public/logs/live.log']); + mockArtifacts('TASKNEW1', 0, ['public/logs/live.log']); + mockArtifacts('TASKNEW2', 0, ['public/logs/live.log']); + + mockTaskStatus('TASKBASE1', 'worker-b1'); + mockTaskStatus('TASKBASE2', 'worker-b2'); + mockTaskStatus('TASKNEW1', 'worker-n1'); + mockTaskStatus('TASKNEW2', 'worker-n2'); + + render(); + + const openButton = await screen.findByTitle( + 'open profile comparison for this result', + ); + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }); + await user.click(openButton); + + const dialog = await screen.findByRole('dialog'); + const emptyMessages = await within(dialog).findAllByText( + 'No profile artifacts found.', + ); + // Both sides show the empty message. + expect(emptyMessages).toHaveLength(2); + }); +}); diff --git a/src/__tests__/CompareResults/ProfileCompareUrls.test.ts b/src/__tests__/CompareResults/ProfileCompareUrls.test.ts new file mode 100644 index 000000000..10ad724e2 --- /dev/null +++ b/src/__tests__/CompareResults/ProfileCompareUrls.test.ts @@ -0,0 +1,41 @@ +import { + buildCompareBenchmarkUrl, + buildSingleProfileUrl, + buildTaskArtifactUrl, + SPEEDOMETER3_PROFILE_ARTIFACT, +} from '../../components/CompareResults/ProfileCompare/urls'; + +describe('ProfileCompare url helpers', () => { + it('builds a Taskcluster artifact URL', () => { + expect( + buildTaskArtifactUrl( + 'eSFQ0OC9R665QYfdgtWgKA', + 0, + SPEEDOMETER3_PROFILE_ARTIFACT, + ), + ).toBe( + 'https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/eSFQ0OC9R665QYfdgtWgKA/runs/0/artifacts/public/test_info/profile_speedometer3_compact.jslb.gz', + ); + }); + + it('builds a single-profile URL that decodes to the raw artifact URL', () => { + const url = buildSingleProfileUrl('eSFQ0OC9R665QYfdgtWgKA', 0); + expect(url).toBe( + 'https://profiler.firefox.com/from-url/https%3A%2F%2Ffirefox-ci-tc.services.mozilla.com%2Fapi%2Fqueue%2Fv1%2Ftask%2FeSFQ0OC9R665QYfdgtWgKA%2Fruns%2F0%2Fartifacts%2Fpublic%2Ftest_info%2Fprofile_speedometer3_compact.jslb.gz', + ); + }); + + it('builds a compare-benchmark URL with both profiles', () => { + const base = buildSingleProfileUrl('CVdpkviXTEyAJH37VBMTGQ', 0); + const cmp = buildSingleProfileUrl('IUYZmFShTXSjbSgQqRQ0JQ', 0); + const url = buildCompareBenchmarkUrl(base, cmp); + // The `profiles[]` parameter should appear twice and the inner URLs + // should be double-encoded (the outer URLSearchParams encoding wrapping + // the from-url URL, which itself contains an encoded task artifact URL). + expect(url).toBe( + 'https://deploy-preview-6012--perf-html.netlify.app/compare-benchmark/?' + + 'profiles%5B%5D=https%3A%2F%2Fprofiler.firefox.com%2Ffrom-url%2Fhttps%253A%252F%252Ffirefox-ci-tc.services.mozilla.com%252Fapi%252Fqueue%252Fv1%252Ftask%252FCVdpkviXTEyAJH37VBMTGQ%252Fruns%252F0%252Fartifacts%252Fpublic%252Ftest_info%252Fprofile_speedometer3_compact.jslb.gz' + + '&profiles%5B%5D=https%3A%2F%2Fprofiler.firefox.com%2Ffrom-url%2Fhttps%253A%252F%252Ffirefox-ci-tc.services.mozilla.com%252Fapi%252Fqueue%252Fv1%252Ftask%252FIUYZmFShTXSjbSgQqRQ0JQ%252Fruns%252F0%252Fartifacts%252Fpublic%252Ftest_info%252Fprofile_speedometer3_compact.jslb.gz', + ); + }); +}); diff --git a/src/__tests__/CompareResults/__snapshots__/OverTimeResultsView.test.tsx.snap b/src/__tests__/CompareResults/__snapshots__/OverTimeResultsView.test.tsx.snap index dbd059e14..2a4e41b60 100644 --- a/src/__tests__/CompareResults/__snapshots__/OverTimeResultsView.test.tsx.snap +++ b/src/__tests__/CompareResults/__snapshots__/OverTimeResultsView.test.tsx.snap @@ -346,7 +346,7 @@ exports[`Results View The table should match snapshot and other elements should
@@ -742,7 +742,7 @@ exports[`Results View The table should match snapshot and other elements should class="revision-block fw0pvlu" >
@@ -1715,7 +1715,7 @@ exports[`Results Table Should match snapshot 1`] = ` class="revision-block fw0pvlu" >
@@ -4598,7 +4598,7 @@ exports[`Results Table for MannWhitneyResultsItem for mann-whitney-u testVersion class="revision-block fw0pvlu" >
@@ -1897,7 +1897,7 @@ exports[`Results View The table should match snapshot and other elements should class="revision-block fw0pvlu" >
@@ -1112,7 +1112,7 @@ exports[`SubtestsResultsView Component Tests should render the subtests results />
@@ -3023,7 +3023,7 @@ exports[`SubtestsViewCompareOverTime Component Tests in mann-whitney-u testVersi />
@@ -4625,7 +4625,7 @@ exports[`SubtestsViewCompareOverTime Component Tests in mann-whitney-u testVersi />
@@ -6227,7 +6227,7 @@ exports[`SubtestsViewCompareOverTime Component Tests renders over-time view when />
@@ -7829,7 +7829,7 @@ exports[`SubtestsViewCompareOverTime Component Tests should render the subtests />
+ setDialogOpen(true)} + > + + + setDialogOpen(false)} + result={result} + /> + + ); +} diff --git a/src/components/CompareResults/ProfileCompare/ProfileCompareDialog.tsx b/src/components/CompareResults/ProfileCompare/ProfileCompareDialog.tsx new file mode 100644 index 000000000..0011aaabf --- /dev/null +++ b/src/components/CompareResults/ProfileCompare/ProfileCompareDialog.tsx @@ -0,0 +1,299 @@ +import { useEffect, useState } from 'react'; + +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import { + Alert, + Box, + Button, + CircularProgress, + FormControlLabel, + IconButton, + Link, + Radio, + RadioGroup, + Typography, +} from '@mui/material'; + +import { + buildCompareBenchmarkUrl, + buildSingleProfileUrl, + SPEEDOMETER3_PROFILE_ARTIFACT, +} from './urls'; +import { + fetchTaskArtifacts, + fetchTaskStatus, +} from '../../../logic/taskcluster'; +import { fetchJobInformationFromJobId } from '../../../logic/treeherder'; +import { CompareResultsItem } from '../../../types/state'; +import { formatNumber } from '../../../utils/format'; +import { CenteredModal } from '../Retrigger/CenteredModal'; + +type RunInfo = { + jobId: number; + taskId: string; + runId: number; + value: number; + workerId?: string; +}; + +type LoadState = + | { kind: 'loading' } + | { kind: 'loaded'; base: RunInfo[]; newRuns: RunInfo[] } + | { kind: 'error'; message: string }; + +const TC_ROOT_URL = 'https://firefox-ci-tc.services.mozilla.com'; + +async function loadRunsWithProfiles( + repo: string, + jobIds: number[], + values: number[], +): Promise { + // Ordering guarantee: values[i] is the score from the job with id jobIds[i]. + // See treeherder/webapp/api/performance_data.py:_get_grouped_perf_data. + const jobInfos = await Promise.all( + jobIds.map((jobId) => fetchJobInformationFromJobId(repo, jobId)), + ); + const details = await Promise.all( + jobInfos.map(async (jobInfo) => { + const { task_id: taskId, retry_id: runId } = jobInfo.taskcluster_metadata; + const [artifacts, status] = await Promise.all([ + fetchTaskArtifacts(TC_ROOT_URL, taskId, runId).catch(() => []), + fetchTaskStatus(TC_ROOT_URL, taskId).catch(() => null), + ]); + return { artifacts, status }; + }), + ); + + const runs: RunInfo[] = []; + jobInfos.forEach((jobInfo, i) => { + const { artifacts, status } = details[i]; + const hasProfile = artifacts.some( + (artifact) => artifact.name === SPEEDOMETER3_PROFILE_ARTIFACT, + ); + if (!hasProfile) return; + const { task_id: taskId, retry_id: runId } = jobInfo.taskcluster_metadata; + const workerId = status?.runs.find((r) => r.runId === runId)?.workerId; + runs.push({ + jobId: jobIds[i], + taskId, + runId, + value: values[i], + workerId, + }); + }); + runs.sort((a, b) => a.value - b.value); + return runs; +} + +function RunList({ + side, + runs, + selectedIndex, + onSelectedIndexChange, +}: { + side: 'base' | 'new'; + runs: RunInfo[]; + selectedIndex: number | null; + onSelectedIndexChange: (index: number) => void; +}) { + if (runs.length === 0) { + return ( + + No profile artifacts found. + + ); + } + return ( + onSelectedIndexChange(Number(e.target.value))} + > + {runs.map((run, index) => ( + + } + label={ + + Run {index + 1} — score {formatNumber(run.value)} + {run.workerId && ( + + ({run.workerId}) + + )} + + } + sx={{ flexGrow: 1, m: 0 }} + /> + + + + + ))} + + ); +} + +type ProfileCompareDialogProps = { + open: boolean; + onClose: () => void; + result: CompareResultsItem; +}; + +export function ProfileCompareDialog({ + open, + onClose, + result, +}: ProfileCompareDialogProps) { + const [state, setState] = useState({ kind: 'loading' }); + const [selectedBase, setSelectedBase] = useState(null); + const [selectedNew, setSelectedNew] = useState(null); + + useEffect(() => { + if (!open) return; + let cancelled = false; + setState({ kind: 'loading' }); + setSelectedBase(null); + setSelectedNew(null); + + Promise.all([ + loadRunsWithProfiles( + result.base_repository_name, + result.base_retriggerable_job_ids, + result.base_runs, + ), + loadRunsWithProfiles( + result.new_repository_name, + result.new_retriggerable_job_ids, + result.new_runs, + ), + ]).then( + ([base, newRuns]) => { + if (cancelled) return; + setState({ kind: 'loaded', base, newRuns }); + if (base.length > 0) setSelectedBase(Math.floor(base.length / 2)); + if (newRuns.length > 0) setSelectedNew(Math.floor(newRuns.length / 2)); + }, + (error: unknown) => { + if (cancelled) return; + const message = error instanceof Error ? error.message : String(error); + setState({ kind: 'error', message }); + }, + ); + + return () => { + cancelled = true; + }; + }, [open, result]); + + const canCompare = + state.kind === 'loaded' && + selectedBase !== null && + selectedNew !== null && + state.base[selectedBase] !== undefined && + state.newRuns[selectedNew] !== undefined; + + const compareUrl = canCompare + ? buildCompareBenchmarkUrl( + buildSingleProfileUrl( + state.base[selectedBase].taskId, + state.base[selectedBase].runId, + ), + buildSingleProfileUrl( + state.newRuns[selectedNew].taskId, + state.newRuns[selectedNew].runId, + ), + ) + : undefined; + + return ( + + + Profile comparison + + + {result.header_name} — {result.platform} + + + {state.kind === 'loading' && ( + + + + )} + + {state.kind === 'error' && ( + Failed to load runs: {state.message} + )} + + {state.kind === 'loaded' && ( + + + + Base ({result.base_rev.slice(0, 12)}) + + + + + + New ({result.new_rev.slice(0, 12)}) + + + + + )} + + + {canCompare ? ( + + ) : ( + + )} + + + ); +} diff --git a/src/components/CompareResults/ProfileCompare/urls.ts b/src/components/CompareResults/ProfileCompare/urls.ts new file mode 100644 index 000000000..63cbfe86f --- /dev/null +++ b/src/components/CompareResults/ProfileCompare/urls.ts @@ -0,0 +1,46 @@ +// TODO: Point this at https://profiler.firefox.com/compare-benchmark/ once +// the feature lands in production (see the PR at +// https://github.com/firefox-devtools/profiler/pull/6012). +const COMPARE_BENCHMARK_BASE_URL = + 'https://deploy-preview-6012--perf-html.netlify.app'; + +const PROFILER_BASE_URL = 'https://profiler.firefox.com'; +const TC_ARTIFACT_BASE_URL = 'https://firefox-ci-tc.services.mozilla.com'; + +// The name of the compact-profile artifact that the browsertime speedometer3 +// task uploads. We match on this exact name to decide whether a run is +// eligible for profile comparison. +export const SPEEDOMETER3_PROFILE_ARTIFACT = + 'public/test_info/profile_speedometer3_compact.jslb.gz'; + +export function buildTaskArtifactUrl( + taskId: string, + runId: number, + artifactPath: string, +): string { + return `${TC_ARTIFACT_BASE_URL}/api/queue/v1/task/${taskId}/runs/${runId}/artifacts/${artifactPath}`; +} + +// Builds a profiler.firefox.com URL that loads a single profile from a +// Taskcluster artifact. The resulting URL is safe to use as an `href`. +export function buildSingleProfileUrl( + taskId: string, + runId: number, + artifactPath: string = SPEEDOMETER3_PROFILE_ARTIFACT, +): string { + const artifactUrl = buildTaskArtifactUrl(taskId, runId, artifactPath); + return `${PROFILER_BASE_URL}/from-url/${encodeURIComponent(artifactUrl)}`; +} + +// Builds a URL for the profiler's benchmark-comparison view, comparing the +// two given profiler-URLs (each already a from-url URL). URLSearchParams is +// deliberately used so the encoding matches profiler.firefox.com's parser. +export function buildCompareBenchmarkUrl( + baseProfileUrl: string, + newProfileUrl: string, +): string { + const params = new URLSearchParams(); + params.append('profiles[]', baseProfileUrl); + params.append('profiles[]', newProfileUrl); + return `${COMPARE_BENCHMARK_BASE_URL}/compare-benchmark/?${params.toString()}`; +} diff --git a/src/components/CompareResults/RevisionRow.tsx b/src/components/CompareResults/RevisionRow.tsx index 5a4a18f22..763e64dba 100644 --- a/src/components/CompareResults/RevisionRow.tsx +++ b/src/components/CompareResults/RevisionRow.tsx @@ -8,6 +8,10 @@ import { IconButton, Box } from '@mui/material'; import Tooltip from '@mui/material/Tooltip'; import { style } from 'typestyle'; +import { + ProfileCompareButton, + supportsProfileCompare, +} from './ProfileCompare/ProfileCompareButton'; import { RetriggerButton } from './Retrigger/RetriggerButton'; import RevisionRowExpandable from './RevisionRowExpandable'; import { compareView, compareOverTimeView } from '../../common/constants'; @@ -343,6 +347,13 @@ function RevisionRow(props: RevisionRowProps) {
)} + {supportsProfileCompare(result.suite) && ( +
+
+ +
+
+ )}
diff --git a/src/components/CompareResults/rowButtonSlots.ts b/src/components/CompareResults/rowButtonSlots.ts new file mode 100644 index 000000000..cae0f18c6 --- /dev/null +++ b/src/components/CompareResults/rowButtonSlots.ts @@ -0,0 +1,27 @@ +// Geometry for the `.row-buttons` grid cell in RevisionRow and +// SubtestsRevisionRow. +// +// Every row is its own CSS grid, and TableHeader builds a matching one, so the +// cell has to be sized identically for all of them: wide enough for the most +// buttons any single row can show, even though most rows show fewer. Buttons +// are right-aligned inside the cell, so the surplus shows up as blank space to +// their left instead of as a ragged right edge. +// +// Keep the slot counts below in sync with the buttons rendered in the +// `.row-buttons` cell of each row component. +const BUTTON_SLOT_WIDTH_PX = 34; + +// RevisionRow: subtests link (only when the result has subtests), profile +// comparison (only for suites supportsProfileCompare() accepts), graph link, +// and retrigger. +const REVISION_ROW_BUTTON_SLOTS = 4; + +// SubtestsRevisionRow: graph link only. +const SUBTESTS_ROW_BUTTON_SLOTS = 1; + +export function rowButtonsGridWidth(isSubtestTable: boolean): string { + const slots = isSubtestTable + ? SUBTESTS_ROW_BUTTON_SLOTS + : REVISION_ROW_BUTTON_SLOTS; + return `${slots * BUTTON_SLOT_WIDTH_PX}px`; +} diff --git a/src/logic/taskcluster.ts b/src/logic/taskcluster.ts index 8085116aa..087ee5f89 100644 --- a/src/logic/taskcluster.ts +++ b/src/logic/taskcluster.ts @@ -172,6 +172,47 @@ type Action = { name: string; }; +type TaskArtifact = { + name: string; + storageType: string; + contentType: string; +}; + +// Lists the artifacts produced by a given run of a Taskcluster task. Used by +// the profile-comparison dialog to check which runs have a profile artifact. +export async function fetchTaskArtifacts( + rootUrl: string, + taskId: string, + runId: number, +): Promise { + const url = `${rootUrl}/api/queue/v1/task/${taskId}/runs/${runId}/artifacts`; + const response = await fetch(url); + await checkTaskclusterResponse(response); + const json = (await response.json()) as { artifacts: TaskArtifact[] }; + return json.artifacts; +} + +export type TaskRunStatus = { + runId: number; + state: string; + workerGroup?: string; + workerId?: string; +}; + +// Fetches Taskcluster task status, which includes per-run worker information. +export async function fetchTaskStatus( + rootUrl: string, + taskId: string, +): Promise<{ runs: TaskRunStatus[] }> { + const url = `${rootUrl}/api/queue/v1/task/${taskId}/status`; + const response = await fetch(url); + await checkTaskclusterResponse(response); + const json = (await response.json()) as { + status: { runs: TaskRunStatus[] }; + }; + return json.status; +} + export async function fetchActionsFromDecisionTask( rootUrl: string, decisionTaskId: string,