From 0c26ba155b5f716e85d477697e9d04c6e4dae9d4 Mon Sep 17 00:00:00 2001 From: Rushabh Shroff Date: Tue, 13 Oct 2026 17:29:50 +0530 Subject: [PATCH] Enhance visual regression testing setup - Added a new script to approve Percy baseline builds in the package.json. - Updated the visual regression workflow to change the PERCY_BRANCH from 'master' to 'main'. - Included a new entry in .gitignore to exclude the .percy-baseline-build-id file. - Refactored visual regression tests to save the baseline build ID for CI approval. --- .github/workflows/test-visual-regression.yml | 9 +- .gitignore | 1 + web-tests/package.json | 3 +- web-tests/scripts/approve-percy-build.js | 147 +++++++++++++++++++ web-tests/tests/visual-regression.test.js | 146 ++++++------------ 5 files changed, 199 insertions(+), 107 deletions(-) create mode 100644 web-tests/scripts/approve-percy-build.js diff --git a/.github/workflows/test-visual-regression.yml b/.github/workflows/test-visual-regression.yml index b878513..1c1690e 100644 --- a/.github/workflows/test-visual-regression.yml +++ b/.github/workflows/test-visual-regression.yml @@ -48,8 +48,13 @@ jobs: BUILD_NUMBER: ${{ github.run_number }}-baseline BASE_URL: ${{ inputs.localTesting && 'http://localhost:3000' || 'https://browserstackce.github.io/features-demo-web' }} BROWSERSTACK_LOCAL: ${{ inputs.localTesting }} - PERCY_BRANCH: master + PERCY_BRANCH: main VISUAL_DIFF: "0" + - name: Approve Percy baseline build + run: npm run approve:percy-build + env: + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.accessKey }} - name: Percy with visual diffs (diff on) run: npm run test:visual-regression env: @@ -59,4 +64,4 @@ jobs: BASE_URL: ${{ inputs.localTesting && 'http://localhost:3000' || 'https://browserstackce.github.io/features-demo-web' }} BROWSERSTACK_LOCAL: ${{ inputs.localTesting }} VISUAL_DIFF: "1" - PERCY_BRANCH: diff-on + PERCY_BRANCH: main diff --git a/.gitignore b/.gitignore index 2e4335d..483ef96 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ web-tests/.env web-tests/node_modules/ web-tests/.env web-tests/log +web-tests/.percy-baseline-build-id browserstack.err local.log .cursor \ No newline at end of file diff --git a/web-tests/package.json b/web-tests/package.json index f438b44..91ee56a 100644 --- a/web-tests/package.json +++ b/web-tests/package.json @@ -16,7 +16,8 @@ "test:iframe":"BROWSERSTACK_CONFIG_FILE='config/iframe.yml' browserstack-node-sdk jest --testPathPattern=tests/iframe", "test:permissions":"BROWSERSTACK_CONFIG_FILE='config/permissions.yml' browserstack-node-sdk jest --testPathPattern=tests/permissions", "test:accessibility":"BROWSERSTACK_CONFIG_FILE='config/accessibility.yml' browserstack-node-sdk jest --testPathPattern=tests/accessibility", - "test:visual-regression": "BROWSERSTACK_CONFIG_FILE='config/visual-regression.yml' browserstack-node-sdk jest --testPathPattern=tests/visual-regression" + "test:visual-regression": "BROWSERSTACK_CONFIG_FILE='config/visual-regression.yml' browserstack-node-sdk jest --testPathPattern=tests/visual-regression", + "approve:percy-build": "node scripts/approve-percy-build.js" }, "devDependencies": { "@percy/selenium-webdriver": "^2.2.7", diff --git a/web-tests/scripts/approve-percy-build.js b/web-tests/scripts/approve-percy-build.js new file mode 100644 index 0000000..7a4aef2 --- /dev/null +++ b/web-tests/scripts/approve-percy-build.js @@ -0,0 +1,147 @@ +#!/usr/bin/env node +/** + * Wait for the baseline Percy build to finish, then approve it. + * Auth: BrowserStack username/access key only (Basic) — no PERCY_TOKEN. + * Build id: PERCY_BUILD_ID env, or web-tests/.percy-baseline-build-id (from healthcheck). + * + * @see https://www.browserstack.com/docs/percy/api-reference/builds#approve-build + */ +const fs = require('fs'); +const path = require('path'); + +const BUILD_ID_FILE = path.join(__dirname, '..', '.percy-baseline-build-id'); +const PERCY_API = 'https://percy.io/api/v1'; +const POLL_MS = 10_000; +const TIMEOUT_MS = 10 * 60 * 1000; + +function basicAuthHeader(username, accessKey) { + const token = Buffer.from(`${username}:${accessKey}`, 'utf8').toString('base64'); + return `Basic ${token}`; +} + +function readBuildId() { + if (process.env.PERCY_BUILD_ID) { + return String(process.env.PERCY_BUILD_ID).trim(); + } + if (!fs.existsSync(BUILD_ID_FILE)) { + throw new Error( + `Missing Percy build id. Set PERCY_BUILD_ID or ensure baseline test wrote ${BUILD_ID_FILE}` + ); + } + const id = fs.readFileSync(BUILD_ID_FILE, 'utf8').trim(); + if (!id) throw new Error(`${BUILD_ID_FILE} is empty`); + return id; +} + +async function percyFetch(pathname, { method = 'GET', body, auth } = {}) { + const res = await fetch(`${PERCY_API}${pathname}`, { + method, + headers: { + Authorization: auth, + Accept: 'application/json', + ...(body ? { 'Content-Type': 'application/json' } : {}), + }, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + let json = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = { raw: text }; + } + return { ok: res.ok, status: res.status, json }; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitUntilFinished(buildId, auth) { + const started = Date.now(); + while (Date.now() - started < TIMEOUT_MS) { + const { ok, status, json } = await percyFetch(`/builds/${buildId}`, { auth }); + if (!ok) { + throw new Error(`GET /builds/${buildId} failed (${status}): ${JSON.stringify(json)}`); + } + const attrs = (json && json.data && json.data.attributes) || {}; + const state = attrs.state; + const reviewState = attrs['review-state']; + const webUrl = attrs['web-url']; + const total = attrs['total-comparisons']; + const finishedComparisons = attrs['total-comparisons-finished']; + const comparisonsDone = + total == null || finishedComparisons == null || Number(finishedComparisons) >= Number(total); + + console.log( + `[percy] build ${buildId} state=${state} review-state=${reviewState}` + + (total != null ? ` comparisons=${finishedComparisons}/${total}` : '') + + (webUrl ? ` ${webUrl}` : '') + ); + + if (state === 'failed') { + throw new Error(`Percy build ${buildId} failed: ${attrs['failed-reason'] || attrs['failure-details'] || 'unknown'}`); + } + + if (state === 'finished' && comparisonsDone) { + return { attrs, webUrl }; + } + + await sleep(POLL_MS); + } + throw new Error(`Timed out waiting for Percy build ${buildId} to finish (${TIMEOUT_MS}ms)`); +} + +async function approveBuild(buildId, auth) { + const body = { + data: { + attributes: { action: 'approve' }, + relationships: { + build: { + data: { type: 'builds', id: String(buildId) }, + }, + }, + }, + }; + const { ok, status, json } = await percyFetch('/reviews', { method: 'POST', body, auth }); + if (!ok) { + throw new Error(`POST /reviews approve failed (${status}): ${JSON.stringify(json)}`); + } + return json; +} + +async function main() { + const username = process.env.BROWSERSTACK_USERNAME; + const accessKey = process.env.BROWSERSTACK_ACCESS_KEY; + if (!username || !accessKey) { + throw new Error('BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY are required'); + } + + const buildId = readBuildId(); + const auth = basicAuthHeader(username, accessKey); + console.log(`[percy] approving build id=${buildId}`); + + const { attrs, webUrl } = await waitUntilFinished(buildId, auth); + + if (attrs['review-state'] === 'approved') { + console.log(`[percy] build ${buildId} already approved${webUrl ? ` — ${webUrl}` : ''}`); + return; + } + + const review = await approveBuild(buildId, auth); + const reviewState = + review && + review.data && + review.data.attributes && + review.data.attributes['review-state']; + console.log( + `[percy] approved build ${buildId}` + + (reviewState ? ` review-state=${reviewState}` : '') + + (webUrl ? ` — ${webUrl}` : '') + ); +} + +main().catch((err) => { + console.error(`[percy] ${err.message || err}`); + process.exit(1); +}); diff --git a/web-tests/tests/visual-regression.test.js b/web-tests/tests/visual-regression.test.js index ad09927..54f9800 100644 --- a/web-tests/tests/visual-regression.test.js +++ b/web-tests/tests/visual-regression.test.js @@ -1,107 +1,58 @@ -const dns = require('dns').promises; +const fs = require('fs'); const http = require('http'); -const net = require('net'); +const path = require('path'); const { Builder, By, until } = require('selenium-webdriver'); const { percyScreenshot, createRegion } = require('@percy/selenium-webdriver'); const { pageUrl } = require('../helpers/baseUrl'); -function agentLog(hypothesisId, location, message, data) { - // #region agent log - fetch('http://127.0.0.1:7917/ingest/e831e398-393d-4109-bd15-dfafb42d91c1',{method:'POST',headers:{'Content-Type':'application/json','X-Debug-Session-Id':'38d3cb'},body:JSON.stringify({sessionId:'38d3cb',runId:'pre-fix',hypothesisId,location,message,data,timestamp:Date.now()})}).catch(()=>{}); - // #endregion -} - -function sanitizeUrl(value) { - if (!value || typeof value !== 'string') return null; - try { - const u = new URL(value); - return { - protocol: u.protocol, - hostname: u.hostname, - port: u.port || (u.protocol === 'https:' ? '443' : '80'), - hasAuth: Boolean(u.username), - }; - } catch { - return { unparseable: true }; - } -} +const BUILD_ID_FILE = path.join(__dirname, '..', '.percy-baseline-build-id'); -function tcpProbe(host, port, timeoutMs = 4000) { - return new Promise((resolve) => { - const socket = net.connect({ host, port: Number(port) }); - const timer = setTimeout(() => { - socket.destroy(); - resolve({ ok: false, error: 'timeout' }); - }, timeoutMs); - socket.on('connect', () => { - clearTimeout(timer); - socket.end(); - resolve({ ok: true }); - }); - socket.on('error', (err) => { - clearTimeout(timer); - resolve({ ok: false, error: err.code || err.message }); - }); - }); -} +// VISUAL_DIFF=0 (default) → baseline; VISUAL_DIFF=1 → intentional UI diffs +const DIFF_ON = process.env.VISUAL_DIFF === '1' || process.env.VISUAL_DIFF === 'on'; +const PAGE_URL = `${pageUrl('visual-regression.html')}?diff=${DIFF_ON ? 'on' : 'off'}`; -function percyHealth() { - return new Promise((resolve) => { - const req = http.get('http://127.0.0.1:5338/percy/healthcheck', { timeout: 3000 }, (res) => { +/** Full JSON from local Percy CLI healthcheck (includes build.id while Percy is running). */ +function fetchPercyHealthcheck() { + return new Promise((resolve, reject) => { + const req = http.get('http://127.0.0.1:5338/percy/healthcheck', { timeout: 5000 }, (res) => { let body = ''; res.on('data', (chunk) => { body += chunk; }); - res.on('end', () => resolve({ status: res.statusCode, body: body.slice(0, 180) })); + res.on('end', () => { + try { + resolve({ status: res.statusCode, data: JSON.parse(body) }); + } catch (err) { + reject(new Error(`Invalid healthcheck JSON: ${err.message}`)); + } + }); }); - req.on('error', (err) => resolve({ error: err.code || err.message })); + req.on('error', reject); req.on('timeout', () => { req.destroy(); - resolve({ error: 'timeout' }); + reject(new Error('Percy healthcheck timeout')); }); }); } -// VISUAL_DIFF=0 (default) → baseline; VISUAL_DIFF=1 → intentional UI diffs -const DIFF_ON = process.env.VISUAL_DIFF === '1' || process.env.VISUAL_DIFF === 'on'; -const PAGE_URL = `${pageUrl('visual-regression.html')}?diff=${DIFF_ON ? 'on' : 'off'}`; +/** Persist build.id from healthcheck for the CI approve step (baseline run only). */ +async function saveBaselineBuildId() { + if (DIFF_ON) return; + const { status, data } = await fetchPercyHealthcheck(); + const buildId = data && data.build && data.build.id; + if (status !== 200 || !buildId) { + throw new Error( + `Percy healthcheck did not return build.id (status=${status}, build=${JSON.stringify(data && data.build)})` + ); + } + fs.writeFileSync(BUILD_ID_FILE, String(buildId), 'utf8'); + // eslint-disable-next-line no-console + console.log(`[percy] baseline build id=${buildId} url=${(data.build && data.build.url) || 'n/a'} → ${BUILD_ID_FILE}`); +} describe('Visual Regression / Percy', () => { let driver; beforeAll(async () => { driver = await new Builder().build(); - const session = await driver.getSession(); - const caps = Object.fromEntries(session.getCapabilities().map_); - const urlCaps = {}; - for (const [key, value] of Object.entries(caps)) { - if (typeof value === 'string' && /https?:|wss?:/i.test(value)) { - urlCaps[key] = sanitizeUrl(value); - } - } - const hub = urlCaps['se:cdp'] || urlCaps.webSocketUrl || null; - let dnsResult = null; - let directTcp = null; - if (hub && hub.hostname) { - try { - dnsResult = await dns.lookup(hub.hostname, { all: true }); - } catch (err) { - dnsResult = { error: err.code || err.message }; - } - directTcp = await tcpProbe(hub.hostname, hub.port); - } - const health = await percyHealth(); - agentLog('H1', 'visual-regression.test.js:beforeAll', 'proxy env, hub url, direct tcp', { - node: process.version, - httpProxy: sanitizeUrl(process.env.HTTP_PROXY || process.env.http_proxy), - httpsProxy: sanitizeUrl(process.env.HTTPS_PROXY || process.env.https_proxy), - noProxy: process.env.NO_PROXY || process.env.no_proxy || null, - percyTokenSet: Boolean(process.env.PERCY_TOKEN), - browserstackLocal: process.env.BROWSERSTACK_LOCAL || null, - baseUrl: process.env.BASE_URL || null, - urlCaps, - dnsResult, - directTcp, - health, - }); }, 180000); afterAll(async () => { @@ -117,24 +68,8 @@ describe('Visual Regression / Percy', () => { const diffState = await driver.executeScript('return document.body.dataset.diff'); expect(diffState).toBe(DIFF_ON ? 'on' : 'off'); - process.env.PERCY_RAISE_ERROR = 'true'; - const shoot = async (name, options) => { - try { - const data = await percyScreenshot(driver, name, options); - agentLog('H5', 'visual-regression.test.js:shoot', 'screenshot returned', { - name, - hasData: Boolean(data), - }); - } catch (error) { - agentLog('H1', 'visual-regression.test.js:shoot', 'screenshot failed', { - name, - error: String(error && error.message || error).slice(0, 400), - }); - } - }; - // 1) Full-page capture — freeze animations, responsive-style widths metadata via options - await shoot('VR — Full page', { + await percyScreenshot(driver, 'VR — Full page', { fullPage: true, freezeAnimatedImage: true, percyCSS: '.hide-in-percy { visibility: hidden !important; }', @@ -145,18 +80,21 @@ describe('Visual Regression / Percy', () => { testCase: 'percy-feature-showcase', }); + // Capture build.id while Percy CLI is still running (needed for CI approve) + await saveBaselineBuildId(); + // 2) Scoped element — product card only await driver.findElement(By.id('scoped-card')).then((el) => driver.executeScript('arguments[0].scrollIntoView({block:"center"})', el) ); - await shoot('VR — Scoped card', { + await percyScreenshot(driver, 'VR — Scoped card', { scope: '#scoped-card', fullPage: false, labels: 'visual-regression,scope', }); // 3) Ignore dynamic clock + ad banner; hide noise with percyCSS - await shoot('VR — Ignore dynamic', { + await percyScreenshot(driver, 'VR — Ignore dynamic', { fullPage: true, ignoreRegionSelectors: ['#dynamic-clock', '#ad-banner'], percyCSS: ` @@ -167,7 +105,7 @@ describe('Visual Regression / Percy', () => { }); // 4) Region algorithms — layout on grid, ignore on ad, intelliignore on clock - await shoot('VR — Layout region', { + await percyScreenshot(driver, 'VR — Layout region', { fullPage: true, regions: [ createRegion({ @@ -195,7 +133,7 @@ describe('Visual Regression / Percy', () => { await driver.findElement(By.id('animated-banner')).then((el) => driver.executeScript('arguments[0].scrollIntoView({block:"center"})', el) ); - await shoot('VR — JS animation', { + await percyScreenshot(driver, 'VR — JS animation', { scope: '#animated-banner', enableJavaScript: true, freezeAnimatedImage: true, @@ -208,5 +146,5 @@ describe('Visual Regression / Percy', () => { `, labels: 'visual-regression,enable-javascript', }); - }, 580000); + }, 180000); });