From b8bd78798c8ad75fcd5ad8fb4efa70f2b034b6b4 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 10 Aug 2026 19:37:29 +0530 Subject: [PATCH 1/2] fix: stop CSP allow-lists triggering challenge detection (#264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isChallengeResponse matched challenge markers against every response header, so any site whose CSP names cdnjs.cloudflare.com or google.com/recaptcha (HN, among many) was classified as a challenge: two wasted impit fetches and a misleading FETCH_BLOCKED for a page that was never blocked. Header evidence is now limited to headers that describe this response (server, cf-mitigated, cf-chl-*, x-datadome*, set-cookie), and a 200 needs body evidence — headers alone never prove a challenge on a served page. Co-Authored-By: Claude Opus 5 --- src/fetch/classify.test.ts | 10 ++++++++++ src/fetch/classify.ts | 10 ++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/fetch/classify.test.ts b/src/fetch/classify.test.ts index 7d4f42f7..d118ea2f 100644 --- a/src/fetch/classify.test.ts +++ b/src/fetch/classify.test.ts @@ -5,6 +5,16 @@ describe('fetch classification', () => { it('recognizes explicit challenges but not bare forbidden responses', () => { expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); expect(isChallengeResponse(403, {}, 'forbidden')).toBe(false); + expect(isChallengeResponse(403, { server: 'cloudflare' }, 'forbidden')).toBe(true); + }); + it('ignores third-party allow-lists in CSP and friends', () => { + const csp = "default-src 'self'; script-src https://www.google.com/recaptcha/ https://cdnjs.cloudflare.com/"; + expect(isChallengeResponse(200, { 'content-security-policy': csp }, 'Hacker News')).toBe(false); + expect(isChallengeResponse(403, { 'content-security-policy': csp }, 'forbidden')).toBe(false); + }); + it('does not treat a served 200 as a challenge on headers alone', () => { + expect(isChallengeResponse(200, { server: 'cloudflare' }, 'real page')).toBe(false); + expect(isChallengeResponse(200, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); }); it('recognizes script-heavy app shells', () => expect(isJavaScriptShell('
')).toBe(true)); }); diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts index 4051c13c..c8474fdb 100644 --- a/src/fetch/classify.ts +++ b/src/fetch/classify.ts @@ -1,8 +1,14 @@ const challengeMarkers = /cloudflare|cf-chl|datadome|perimeterx|px-captcha|akamai|captcha|just a moment|verify you are human/i; +// Headers that say something about *this* response. CSP/report-to/link are allow-lists of third +// parties (cdnjs.cloudflare.com, google.com/recaptcha) and are evidence of nothing. +const signalHeaders = /^(?:server|cf-mitigated|cf-chl-[\w-]+|x-datadome[\w-]*|set-cookie)$/i; export function isChallengeResponse(status: number, headers: Record, body: string): boolean { - const evidence = `${Object.entries(headers).map(([key, value]) => `${key}:${value}`).join('\n')}\n${body.slice(0, 20_000)}`; - return challengeMarkers.test(evidence) && (status === 403 || status === 429 || status === 503 || status === 200); + if (status !== 403 && status !== 429 && status !== 503 && status !== 200) return false; + if (challengeMarkers.test(body.slice(0, 20_000))) return true; + // A 200 with a real body is a served page; headers alone (server: cloudflare) never prove otherwise. + if (status === 200) return false; + return Object.entries(headers).some(([key, value]) => signalHeaders.test(key) && challengeMarkers.test(value)); } export function isJavaScriptShell(body: string): boolean { From 34ffd0094e4bcecb15dc046822048dcb72874455 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Wed, 12 Aug 2026 00:53:15 +0530 Subject: [PATCH 2/2] fix: keep decisive challenge headers while ignoring CSP allow-lists cf-chl-* and x-datadome* carry their evidence in the header name, so match on the normalized name rather than only the value, and treat Cloudflare's documented cf-mitigated: challenge as decisive even on a 200. server and set-cookie stay weak: they can support a 403/429/503 but never turn a served 200 into a challenge. CSP/report-to/link remain ignored. Co-Authored-By: Claude Opus 5 --- src/fetch/classify.test.ts | 9 +++++++++ src/fetch/classify.ts | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/fetch/classify.test.ts b/src/fetch/classify.test.ts index d118ea2f..0af98645 100644 --- a/src/fetch/classify.test.ts +++ b/src/fetch/classify.test.ts @@ -16,5 +16,14 @@ describe('fetch classification', () => { expect(isChallengeResponse(200, { server: 'cloudflare' }, 'real page')).toBe(false); expect(isChallengeResponse(200, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); }); + it('treats cf-mitigated: challenge as decisive, even on a 200', () => { + expect(isChallengeResponse(200, { 'cf-mitigated': 'challenge' }, 'looks fine')).toBe(true); + expect(isChallengeResponse(403, { 'cf-mitigated': 'challenge' }, 'forbidden')).toBe(true); + }); + it('reads decisive evidence from the header name, not just its value', () => { + expect(isChallengeResponse(403, { 'cf-chl-out': 'AAAA1111' }, 'forbidden')).toBe(true); + expect(isChallengeResponse(403, { 'x-datadome': 'protected' }, 'forbidden')).toBe(true); + expect(isChallengeResponse(403, { 'x-datadome-cid': 'abc123' }, 'forbidden')).toBe(true); + }); it('recognizes script-heavy app shells', () => expect(isJavaScriptShell('
')).toBe(true)); }); diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts index c8474fdb..d874c767 100644 --- a/src/fetch/classify.ts +++ b/src/fetch/classify.ts @@ -1,14 +1,22 @@ const challengeMarkers = /cloudflare|cf-chl|datadome|perimeterx|px-captcha|akamai|captcha|just a moment|verify you are human/i; -// Headers that say something about *this* response. CSP/report-to/link are allow-lists of third -// parties (cdnjs.cloudflare.com, google.com/recaptcha) and are evidence of nothing. -const signalHeaders = /^(?:server|cf-mitigated|cf-chl-[\w-]+|x-datadome[\w-]*|set-cookie)$/i; +// Headers a bot-mitigation product sets *because it challenged this request*. The evidence is in +// the name — values are opaque tokens — so these are decisive on their own, whatever the status. +// https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/detect-response/ +const decisiveHeaders = /^(?:cf-chl-[\w-]+|x-datadome[\w-]*)$/i; +// Provider branding: present on every response the provider proxies, challenge or not. It can +// support an already-blocked status but must never turn a served 200 into a challenge. +const weakHeaders = /^(?:server|set-cookie)$/i; +// CSP/report-to/link are allow-lists of third parties (cdnjs.cloudflare.com, google.com/recaptcha) +// and are evidence of nothing — they are in neither list. export function isChallengeResponse(status: number, headers: Record, body: string): boolean { if (status !== 403 && status !== 429 && status !== 503 && status !== 200) return false; if (challengeMarkers.test(body.slice(0, 20_000))) return true; - // A 200 with a real body is a served page; headers alone (server: cloudflare) never prove otherwise. + const entries = Object.entries(headers).map(([key, value]) => [key.trim().toLowerCase(), value] as const); + if (entries.some(([key, value]) => decisiveHeaders.test(key) || (key === 'cf-mitigated' && /challenge/i.test(value)))) return true; + // A 200 with a real body is a served page; branding alone (server: cloudflare) never proves otherwise. if (status === 200) return false; - return Object.entries(headers).some(([key, value]) => signalHeaders.test(key) && challengeMarkers.test(value)); + return entries.some(([key, value]) => weakHeaders.test(key) && challengeMarkers.test(value)); } export function isJavaScriptShell(body: string): boolean {