From b1890953222e97dc170102a52a586544c8681b4a Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Mon, 17 Aug 2026 14:35:52 +0200 Subject: [PATCH] fix: fall back to fetch when prerender returns 4xx page.content() treats WAF/challenge HTML as success and cancelled a parallel 2xx fetch. Co-authored-by: Cursor --- src/index.js | 9 +++++++++ test/mode.js | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index fed8470..7cce780 100644 --- a/src/index.js +++ b/src/index.js @@ -185,6 +185,15 @@ const prerender = PCancelable.fn( ) const payload = await getPayload(url, opts) + // page.content() succeeds on 4xx/5xx challenge pages; only cancel + // fetch when prerender is actually usable, otherwise keep a 2xx/3xx fetch. + if (payload.statusCode >= 400) { + const { isRejected, ...dataProps } = await fetchRes + if (!isRejected && dataProps.statusCode < 400) { + debug('prerender', { url, state: 'fallback', statusCode: payload.statusCode }) + return dataProps + } + } await fetchRes.cancel() debug('prerender', { url, state: 'success' }) return payload diff --git a/test/mode.js b/test/mode.js index 24c8261..094c3de 100644 --- a/test/mode.js +++ b/test/mode.js @@ -1,6 +1,6 @@ 'use strict' -const { getBrowserContext, test } = require('./helpers') +const { getBrowserContext, runServer, test } = require('./helpers') const getHTML = require('../src') @@ -30,6 +30,61 @@ test("`{ prerender: 'auto' }`", async t => { t.is(stats.mode, 'fetch') }) +test('prerender 4xx falls back to a successful fetch', async t => { + const url = await runServer(t, (_, res) => { + res.setHeader('content-type', 'text/html') + res.end('About Us') + }) + + const blockedBrowserless = () => ({ + evaluate: () => async () => ({ + headers: { 'content-type': 'text/html' }, + html: 'ERROR: The request could not be satisfied', + mode: 'prerender', + url: String(url), + statusCode: 403, + redirects: [] + }) + }) + + const { stats, html, statusCode } = await getHTML(String(url), { + prerender: true, + getBrowserless: blockedBrowserless + }) + + t.is(stats.mode, 'fetch') + t.is(statusCode, 200) + t.true(html.includes('About Us')) +}) + +test('prerender 4xx is kept when fetch is also unsuccessful', async t => { + const url = await runServer(t, (_, res) => { + res.statusCode = 403 + res.setHeader('content-type', 'text/html') + res.end('blocked') + }) + + const blockedBrowserless = () => ({ + evaluate: () => async () => ({ + headers: { 'content-type': 'text/html' }, + html: 'ERROR: The request could not be satisfied', + mode: 'prerender', + url: String(url), + statusCode: 403, + redirects: [] + }) + }) + + const { stats, html, statusCode } = await getHTML(String(url), { + prerender: true, + getBrowserless: blockedBrowserless + }) + + t.is(stats.mode, 'prerender') + t.is(statusCode, 403) + t.true(html.includes('The request could not be satisfied')) +}) + test.skip('prerender error fallback into fetch mode', async t => { const url = 'https://www.sportsnet.ca/hockey/nhl/leafs-john-tavares-return-new-york-hope-positive/'