Skip to content

feat(trace): split API requests into a separate network stream #42073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dcrousso wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(trace): split API requests into a separate network stream #42073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/playwright-core/src/server/har/harRecorder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import path from 'path';

import { SerializedFS } from '@utils/serializedFS';
import { Artifact } from '../artifact';
import { APIRequestContext } from '../fetch';
import { HarTracer } from './harTracer';

import type { APIRequestContext } from '../fetch';
import type { BrowserContext } from '../browserContext';
import type { HarTracerDelegate } from './harTracer';
import type { Page } from '../page';
Expand Down Expand Up @@ -52,6 +52,7 @@ export class HarRecorder implements HarTracerDelegate {
const content = options.content || 'embed';
this._tracer = new HarTracer(context, page, this, {
content,
includeAPIRequests: context instanceof APIRequestContext,
slimMode: options.mode === 'minimal',
includeTraceInfo: false,
recordRequestOverrides: true,
Expand Down
14 changes: 9 additions & 5 deletions packages/playwright-core/src/server/har/harTracer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ export interface HarTracerDelegate {

type HarTracerOptions = {
content: 'omit' | 'attach' | 'embed';
includeAPIRequests?: boolean;
includeTraceInfo: boolean;
recordRequestOverrides: boolean;
waitForContentOnStop: boolean;
Expand Down Expand Up @@ -105,11 +106,14 @@ export class HarTracer {
return;
this._options.omitScripts = options.omitScripts;
this._started = true;
const apiRequest = this._context instanceof APIRequestContext ? this._context : this._context.fetchRequest;
this._eventListeners = [
eventsHelper.addEventListener(apiRequest, APIRequestContext.Events.Request, (event: APIRequestEvent) => this._onAPIRequest(event)),
eventsHelper.addEventListener(apiRequest, APIRequestContext.Events.RequestFinished, (event: APIRequestFinishedEvent) => this._onAPIRequestFinished(event)),
];
this._eventListeners = [];
if (this._options.includeAPIRequests !== false) {
const apiRequest = this._context instanceof APIRequestContext ? this._context : this._context.fetchRequest;
this._eventListeners.push(
eventsHelper.addEventListener(apiRequest, APIRequestContext.Events.Request, (event: APIRequestEvent) => this._onAPIRequest(event)),
eventsHelper.addEventListener(apiRequest, APIRequestContext.Events.RequestFinished, (event: APIRequestFinishedEvent) => this._onAPIRequestFinished(event)),
);
}
if (this._context instanceof BrowserContext) {
this._eventListeners.push(
eventsHelper.addEventListener(this._context, BrowserContext.Events.Page, (page: Page) => this._createPageEntryIfNeeded(page)),
Expand Down
115 changes: 12 additions & 103 deletions tests/library/har.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,7 +642,6 @@ it('should have security details', async ({ contextFactory, httpsServer, browser

const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.goto(httpsServer.EMPTY_PAGE);
await page.request.get(httpsServer.EMPTY_PAGE);
const log = await getLog();
const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0];
expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
Expand All @@ -651,8 +650,6 @@ it('should have security details', async ({ contextFactory, httpsServer, browser
expect(securityDetails).toEqual({ protocol: 'TLS 1.3', subjectName: 'playwright-test', validFrom: 1691708270, validTo: 2007068270 });
else
expect(securityDetails).toEqual({ issuer: 'playwright-test', protocol: 'TLS 1.3', subjectName: 'playwright-test', validFrom: 1691708270, validTo: 2007068270 });

expect(log.entries[1]._securityDetails).toEqual({ issuer: 'playwright-test', protocol: 'TLSv1.3', subjectName: 'playwright-test', validFrom: 1691708270, validTo: 2007068270 });
});

it('should have connection details for redirects', async ({ contextFactory, server, browserName, mode }, testInfo) => {
Expand Down Expand Up @@ -811,103 +808,12 @@ it('should have different hars for concurrent contexts', async ({ contextFactory
}
});

it('should include API request', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
const url = server.PREFIX + '/simple.json';
const response = await page.request.post(url, {
headers: { cookie: 'a=b; c=d' },
data: { foo: 'bar' }
});
const responseBody = await response.body();
const log = await getLog();
expect(log.entries.length).toBe(1);
const entry = log.entries[0];
expect(entry.request.url).toBe(url);
expect(entry.request.method).toBe('POST');
expect(entry.request.httpVersion).toBe('HTTP/1.1');
expect(entry.request.cookies).toEqual([
{
'name': 'a',
'value': 'b'
},
{
'name': 'c',
'value': 'd'
}
]);
expect(entry.request.headers.length).toBeGreaterThan(1);
expect(entry.request.headers.find(h => h.name.toLowerCase() === 'user-agent')).toBeTruthy();
expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toBe('application/json');
expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-length')?.value).toBe('13');
expect(entry.request.bodySize).toBe(13);

expect(entry.response.status).toBe(200);
expect(entry.response.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toContain('application/json');
expect(entry.response.content.size).toBe(15);
expect(entry.response.content.text).toBe(responseBody.toString());
expect(entry.response.bodySize).toBe(15);

expect(entry.time).toBeGreaterThan(0);
expect(entry.timings).toEqual(expect.objectContaining({
blocked: -1,
connect: expect.any(Number),
dns: expect.any(Number),
receive: expect.any(Number),
send: expect.any(Number),
ssl: expect.any(Number),
wait: expect.any(Number),
}));

expect(entry.serverIPAddress).toBeDefined();
expect(entry._serverPort).toEqual(server.PORT);
});

it('should correctly record API request cookies with equals sign in value', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
const url = server.PREFIX + '/simple.json';
await page.request.get(url, {
headers: { cookie: 'token=abc=xyz; other=val' },
});
const log = await getLog();
expect(log.entries[0].request.cookies).toEqual([
{ name: 'token', value: 'abc=xyz' },
{ name: 'other', value: 'val' },
]);
});

it('should respect minimal mode for API Requests', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo, { mode: 'minimal' });
const url = server.PREFIX + '/simple.json';
await page.request.post(url, {
headers: { cookie: 'a=b; c=d' },
data: { foo: 'bar' }
});
const { entries } = await getLog();
expect(entries).toHaveLength(1);
const [entry] = entries;
expect(entry.timings).toEqual({ receive: -1, send: -1, wait: -1 });
expect(entry.serverIPAddress).toBeUndefined();
expect(entry._serverPort).toBeUndefined();
expect(entry.request.cookies).toEqual([]);
expect(entry.request.bodySize).toBe(-1);
expect(entry.response.bodySize).toBe(-1);
});

it('should include redirects from API request', async ({ contextFactory, server }, testInfo) => {
server.setRedirect('/redirect-me', '/simple.json');
it('should exclude API requests', async ({ contextFactory, server }, testInfo) => {
const { page, getLog } = await pageWithHar(contextFactory, testInfo);
await page.request.post(server.PREFIX + '/redirect-me', {
headers: { cookie: 'a=b; c=d' },
data: { foo: 'bar' }
});
await page.goto(server.EMPTY_PAGE);
await page.request.get(server.PREFIX + '/simple.json');
const log = await getLog();
expect(log.entries.length).toBe(2);
const [redirect, json] = log.entries;
expect(redirect.request.url).toBe(server.PREFIX + '/redirect-me');
expect(json.request.url).toBe(server.PREFIX + '/simple.json');

expect(redirect.timings).toBeDefined();
expect(json.timings).toBeDefined();
expect(log.entries.map(entry => entry.request.url)).toEqual([server.EMPTY_PAGE]);
});

it('should not hang on resources served from cache', async ({ contextFactory, server, browserName, isBidi }, testInfo) => {
Expand Down Expand Up @@ -965,13 +871,14 @@ it('should not hang on slow chunked response', async ({ browserName, browser, co
expect(log.browser!.version).toBe(browser.version());
});

it('should support HAR larger than 512MB', async ({ contextFactory, server, browserName }, testInfo) => {
it('should support HAR larger than 512MB', async ({ playwright, server, browserName }, testInfo) => {
it.skip(browserName !== 'chromium', 'serializer is browser-agnostic; one browser is enough');
it.slow();
it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/36707' });

const harPath = testInfo.outputPath('test.har');
const context = await contextFactory({ recordHar: { path: harPath } });
const request = await playwright.request.newContext();
await request.tracing.startHar(harPath);

// 30 x 20MB textual responses push the HAR JSON past V8's ~512MB max
// string length. Each body still fits in a single string; only the
Expand All @@ -982,8 +889,9 @@ it('should support HAR larger than 512MB', async ({ contextFactory, server, brow
res.end(body);
});
for (let i = 0; i < 30; i++)
await context.request.get(`${server.PREFIX}/large`);
await context.close();
await request.get(`${server.PREFIX}/large`);
await request.tracing.stopHar();
await request.dispose();

const stats = fs.statSync(harPath);
expect(stats.size).toBeGreaterThan(512 * 1024 * 1024);
Expand Down Expand Up @@ -1057,12 +965,13 @@ it('should record resource type', async ({ contextFactory, server, asset }, test
});

it.describe('tracing.startHar', () => {
it('should record a HAR with options', async ({ contextFactory, server }, testInfo) => {
it('should record a HAR with options and exclude API requests', async ({ contextFactory, server }, testInfo) => {
const context = await contextFactory();
const harPath = testInfo.outputPath('tracing.har');
await context.tracing.startHar(harPath, { mode: 'minimal', urlFilter: '**/one-style.css' });
const page = await context.newPage();
await page.goto(server.PREFIX + '/one-style.html');
await context.request.get(server.PREFIX + '/one-style.css?FAIL_IF_CAPTURED');
await context.tracing.stopHar();
await context.close();

Expand Down
67 changes: 67 additions & 0 deletions tests/library/trace-viewer.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,73 @@ test('should have network requests', async ({ showTraceViewer }) => {
await expect(traceViewer.networkRequests.filter({ hasText: '404GET404text' })).toHaveCSS('background-color', 'rgb(242, 222, 222)');
});

test('should show context API request details', async ({ page, runAndTrace, server }) => {
server.setRoute('/api-details', (_, response) => {
setTimeout(() => {
response.writeHead(200, {
'Content-Type': 'application/json',
'X-Response': 'response-value',
});
response.end(JSON.stringify({ result: 'ok' }));
}, 50);
});
const traceViewer = await runAndTrace(async () => {
await page.request.post(server.PREFIX + '/api-details', {
headers: {
'cookie': 'token=abc=xyz; other=val',
'x-request': 'request-value',
},
data: { foo: 'bar' },
});
});
await traceViewer.selectAction('POST "/api-details"');
await traceViewer.showNetworkTab();
const request = traceViewer.networkRequests.filter({ hasText: 'api-details' });
await expect(request).toHaveCount(1);
await expect(request.locator('.grid-view-column-method')).toHaveText('POST');
await expect(request.locator('.grid-view-column-status')).toHaveText('200');
await expect(request.locator('.grid-view-column-route')).toHaveText('api');
expect(parseMillis(await request.locator('.grid-view-column-duration').innerText())).toBeGreaterThan(0);

await request.click();
const headersPanel = traceViewer.networkTab.getByRole('tabpanel', { name: 'Headers' });
await expect(headersPanel.getByRole('region', { name: 'General' })).toContainText(server.PREFIX + '/api-details');
await expect(headersPanel.getByRole('region', { name: 'General' })).toContainText(/MethodPOST/);
await expect(headersPanel.getByRole('region', { name: 'General' })).toContainText(/Status Code\s*200 OK/);
await expect(headersPanel.getByRole('region', { name: 'General' })).toContainText(/Duration\d+ms/);
const requestHeaders = headersPanel.getByRole('region', { name: /Request Headers/ });
await expect(requestHeaders.getByText('token=abc=xyz; other=val')).toBeVisible();
await expect(requestHeaders.getByText('request-value')).toBeVisible();
await expect(requestHeaders.getByText('application/json', { exact: true })).toBeVisible();
await expect(headersPanel.getByRole('region', { name: /Response Headers/ }).getByText('response-value')).toBeVisible();

await traceViewer.networkTab.getByRole('tab', { name: 'Payload' }).click();
const payloadPanel = traceViewer.networkTab.getByRole('tabpanel', { name: 'Payload' });
await expect(payloadPanel).toContainText('foo');
await expect(payloadPanel).toContainText('bar');

await traceViewer.networkTab.getByRole('tab', { name: 'Response' }).click();
const responsePanel = traceViewer.networkTab.getByRole('tabpanel', { name: 'Response' });
await expect(responsePanel).toContainText('result');
await expect(responsePanel).toContainText('ok');
});

test('should show context API request redirects', async ({ page, runAndTrace, server }) => {
server.setRedirect('/api-redirect', '/simple.json');
const traceViewer = await runAndTrace(async () => {
await page.request.get(server.PREFIX + '/api-redirect');
});
await traceViewer.selectAction('GET "/api-redirect"');
await traceViewer.showNetworkTab();
await expect(traceViewer.networkRequests).toHaveCount(2);
const redirect = traceViewer.networkRequests.filter({ hasText: 'api-redirect' });
await expect(redirect.locator('.grid-view-column-status')).toHaveText('302');
await expect(redirect.locator('.grid-view-column-route')).toHaveText('api');
const response = traceViewer.networkRequests.filter({ hasText: 'simple.json' });
await expect(response.locator('.grid-view-column-status')).toHaveText('200');
await expect(response.locator('.grid-view-column-route')).toHaveText('api');
});

test('should highlight network request on timeline on hover', async ({ showTraceViewer }) => {
const traceViewer = await showTraceViewer(traceFile);
await traceViewer.selectAction('Navigate');
Expand Down
Loading