From 4138ddd641be7cf8361c154e95803cca41de0e31 Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 16:09:35 +0200 Subject: [PATCH 1/8] Apply header redactions when the body is not screened A redaction that reads response headers only is now applied to responses whose body is not screened (over the cap, binary, a live stream, or content-encoded), on both the fetch and Node paths. A rule's prefilter anchors are looked for in header values as well as the body. Co-Authored-By: Claude Opus 5.5 --- src/protect/response-hardening.js | 29 ++- src/protect/runtime.js | 113 ++++++-- .../header-redaction-without-a-body.test.ts | 245 ++++++++++++++++++ 3 files changed, 359 insertions(+), 28 deletions(-) create mode 100644 tests/protect/header-redaction-without-a-body.test.ts diff --git a/src/protect/response-hardening.js b/src/protect/response-hardening.js index 1d34aa0b..2c8a550e 100644 --- a/src/protect/response-hardening.js +++ b/src/protect/response-hardening.js @@ -45,7 +45,30 @@ export function hardensWithoutBody(rule) { // A parameterless match — `cross_origin`, `off_origin`, `cors_reflected` — reads the request's origin // and the response's own headers, so it carries no parameter and needs no body. - return parameters.every( - (parameter) => BODY_INDEPENDENT.includes(parameter) || parameter.startsWith(BODY_INDEPENDENT_PREFIX), - ); + return parameters.every(bodyIndependent); +} + +/** + * Does this rule read response headers and nothing else? + * + * The decision half of `hardensWithoutBody`, for a rule whose action is not a header action — a + * `redact` keyed on `response.header.*` masks the header value it matched, which needs no body either. + * A header must be among what it reads: a rule keyed on the status alone is aimed at the body, and + * masking header values on its behalf would carry out a different rule from the one written. + */ +export function readsOnlyResponseHeaders(rule) { + if (!rule) return false; + + const { parameters, complete } = readRuleParameters(rule); + if (!complete || !parameters.some(isHeaderParameter)) return false; + + return parameters.every(bodyIndependent); +} + +function isHeaderParameter(parameter) { + return parameter === 'response.headers' || parameter.startsWith(BODY_INDEPENDENT_PREFIX); +} + +function bodyIndependent(parameter) { + return BODY_INDEPENDENT.includes(parameter) || parameter.startsWith(BODY_INDEPENDENT_PREFIX); } diff --git a/src/protect/runtime.js b/src/protect/runtime.js index e05e772a..f7923d87 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -36,7 +36,7 @@ import { resolveRules } from './rules/source.js'; import { startRefresh, startRecovery, makeRefreshHandler, serialise } from './rules/refresh.js'; import { createDetectionReporter } from './detections.js'; import { reportingState } from './reporting-state.js'; -import { hardensWithoutBody } from './response-hardening.js'; +import { hardensWithoutBody, readsOnlyResponseHeaders } from './response-hardening.js'; import { notify } from './notify.js'; import { SOURCE_REQUEST } from './supabase-guard.js'; import { createFirewallLogReporter, resolveApiBase, telemetryEnabled } from './firewall-log.js'; @@ -397,22 +397,36 @@ export async function createProtection(options = {}) { }); // One engine per response rule so we can find ALL matches (to redact each). `action: // "redact"` masks the offending span(s); anything else withholds the whole response. - responseRuleSet = responseRules.map((rule) => ({ - rule, - engine: new RuleEngine({ firewall: [rule], onError }), - redactors: rule.action === 'redact' || rule.action === 'encode' ? extractRedactors(rule) : null, + responseRuleSet = responseRules.map((rule) => { + const redactors = rule.action === 'redact' || rule.action === 'encode' ? extractRedactors(rule) : null; // A redact/encode condition that carries body-transforming mutations (base64_decode, urldecode, // json_decode, …) detects on the DECODED body but the span redactors run on the RAW body — so // they mask nothing and the secret is served while the log says "redacted". Flag it so screenText // fails such a rule CLOSED (block) instead of serving a no-op redaction. - mutatedSpan: (rule.action === 'redact' || rule.action === 'encode') && hasSpanMutations(rule), - // Optional cheap pre-filter: literal anchor(s) that MUST appear for the (expensive) regex to - // have any chance of matching. Lets screenText skip the full scan on bodies with no candidate — - // the common case — cutting CPU/latency and shrinking the regex/ReDoS surface. Case-insensitive. - prefilter: Array.isArray(rule.prefilter) && rule.prefilter.length - ? rule.prefilter.map((s) => String(s).toLowerCase()) - : null, - })); + const mutatedSpan = (rule.action === 'redact' || rule.action === 'encode') && hasSpanMutations(rule); + + return { + rule, + engine: new RuleEngine({ firewall: [rule], onError }), + redactors, + mutatedSpan, + // A redaction that reads response headers only, and masks spans in them, is decided and carried + // out without the body — so it still applies to a response whose body was not screened. + redactsHeaders: + rule.action === 'redact' && + !mutatedSpan && + (redactors ?? []).some((r) => !r.jsonPath) && + readsOnlyResponseHeaders(rule), + // Optional cheap pre-filter: literal anchor(s) that MUST appear for the (expensive) regex to + // have any chance of matching. Lets screenText skip the full scan on responses with no candidate + // — the common case — cutting CPU/latency and shrinking the regex/ReDoS surface. + // Case-insensitive, and checked against the header values as well as the body, since a rule may + // read either. + prefilter: Array.isArray(rule.prefilter) && rule.prefilter.length + ? rule.prefilter.map((s) => String(s).toLowerCase()) + : null, + }; + }); // One engine per rule, as the response phase does. It preserves the identity of every rule that // matches: each is evaluated on its own, and each match is attributable to the rule that made it. egressRuleSet = egressRules.map((rule) => ({ @@ -532,6 +546,11 @@ export async function createProtection(options = {}) { return effectiveMode === 'block' ? block() : allow(); }; + // The rules a response whose body was not read can still be screened against: header hardening, and + // redactions of header values. Neither reads the body, and neither needs one to act on. + const decidedWithoutBody = (rule, entry) => hardensWithoutBody(rule) || Boolean(entry?.redactsHeaders); + const redactsHeaders = (_rule, entry) => Boolean(entry?.redactsHeaders); + // Response phase core: screen a text body → { verdict: 'pass'|'block'|'redact', body? }. // redact masks matched spans; block withholds; block wins over redact. Enforcement only in // block mode (dry-run records via onDetect but returns 'pass'). @@ -539,18 +558,19 @@ export async function createProtection(options = {}) { let blockRule = null; const redactions = []; const headerMutations = []; - let lowerText = null; // lazily lowercased body, only if a rule uses a prefilter + let lowerText = null; // lazily lowercased body and header values, only if a rule uses a prefilter const responseSkips = new Set(); // each inspection limit counted once per response - for (const { rule, engine: re, redactors, prefilter, mutatedSpan } of responseRuleSet) { + for (const entry of responseRuleSet) { + const { rule, engine: re, redactors, prefilter, mutatedSpan } = entry; // `only` narrows the set to the rules a caller is entitled to run. The no-body path uses it to // exclude every rule that reads the body, rather than evaluating one against an empty string — // `not_contains` matches everything when there is nothing there, so that is not an undecided rule // but a wrongly decided one. - if (only && !only(rule)) continue; - // Cheap pre-filter: if none of the rule's literal anchors is in the body, its regex can't + if (only && !only(rule, entry)) continue; + // Cheap pre-filter: if none of the rule's literal anchors is in the response, its regex can't // match — skip the full scan (the common no-secret case) before touching the engine. if (prefilter) { - if (lowerText === null) lowerText = text.toLowerCase(); + if (lowerText === null) lowerText = prefilterText(text, meta.headers); if (!prefilter.some((p) => lowerText.includes(p))) continue; } let result; @@ -979,8 +999,9 @@ export async function createProtection(options = {}) { const hardenHeadersOnly = (response, reqCtx) => { try { const meta = { status: response.status, headers: headerObject(response.headers) }; - const r = screenText('', meta, reqCtx, hardensWithoutBody); - // `block` cannot arise: only header actions were eligible. + const r = screenText('', meta, reqCtx, decidedWithoutBody); + // `block` cannot arise: only header actions, and redactions that have a header span to mask, + // were eligible. if (r.verdict !== 'redact' || !r.headers) return response; // A matched rule is not a changed header. `harden-cookie` on a response that sets no cookie, @@ -1247,6 +1268,37 @@ export async function createProtection(options = {}) { }; }; + /** + * Redactions of header values, for a response whose body will not be screened. + * + * Such a rule reads headers only, so it is decided the same way whether or not the body is read — it + * is run here instead of at `end`, never as well, so it reports once. Returns the changes for + * `sendHead`, or undefined when there are none. A head that has already gone cannot take them, which + * is recorded, as it is for a screened body. + */ + const redactUnreadHeaders = () => { + try { + const head = effectiveHead(); + const r = screenText('', head, reqCtx, redactsHeaders); + if (r.verdict !== 'redact' || !r.headers) return undefined; + const changed = new Map(); + for (const [name, value] of Object.entries(r.headers)) { + if (headerValueChanged(head.headers[name], value)) changed.set(name.toLowerCase(), value); + } + if (changed.size && reallySent()) { + recordSkip('response', 'headers-sent', { headers: [...changed.keys()] }); + + return undefined; + } + + return changed.size ? { headers: changed } : undefined; + } catch (err) { + notify(onError, err, 'onError'); + + return undefined; + } + }; + /** * Send the head, if the application asked for one explicitly. `changes` is what the screen at `end` * decided: a new status, and header values keyed by lower-cased name (null removes one). Applied to @@ -1300,7 +1352,7 @@ export async function createProtection(options = {}) { // Too big to screen — abandon buffering, but FLUSH what we already captured (the head) plus // this chunk before switching to pass-through, so the client gets a complete body (not a // truncated one missing everything before the cap was hit). - sendHead(); + sendHead(redactUnreadHeaders()); for (const c of chunks) origWrite(c); chunks.length = 0; origWrite(buf); @@ -1325,8 +1377,8 @@ export async function createProtection(options = {}) { if (overflow) return origEnd(cb); // collect just flushed head + final chunk on overflow hardenOnce(); - const passThrough = () => { - sendHead(); + const passThrough = (changes) => { + sendHead(changes); for (const c of chunks) origWrite(c); return origEnd(cb); @@ -1341,7 +1393,7 @@ export async function createProtection(options = {}) { if (kind === 'skip' || (kind === 'sniff' && looksBinary(buffer))) { recordSkip('response', kind === 'skip' ? (baseContentType(ct) === 'text/event-stream' ? 'live-stream' : 'non-text-content-type') : 'binary-body'); - return passThrough(); + return passThrough(redactUnreadHeaders()); } // Encoded bytes cannot be screened as text: sent exactly as they are, under their own coding. The // coding is read from the head that will be sent, which includes a held `writeHead`'s own headers. @@ -1349,7 +1401,7 @@ export async function createProtection(options = {}) { if (codings.length > 0 && stillEncoded(buffer)) { recordSkip('response', 'encoded-body', { encoding: codings.join(', ') }); - return passThrough(); + return passThrough(redactUnreadHeaders()); } const text = buffer.toString('utf8'); let r; @@ -1932,6 +1984,17 @@ function screenableContentType(ct) { if (base === 'application/octet-stream') return 'sniff'; // maybe a text/JSON export mislabeled return 'skip'; // image/video/audio/font/pdf/zip/wasm/… — don't buffer binary } +/** What a response rule's prefilter looks for its anchors in, lower-cased: the body and every header value. */ +function prefilterText(body, headers) { + const values = []; + for (const value of Object.values(headers ?? {})) { + if (Array.isArray(value)) values.push(...value.map(String)); + else if (value !== undefined && value !== null) values.push(String(value)); + } + + return [body, ...values].join('\n').toLowerCase(); +} + // Cheap binary sniff over a byte prefix: a NUL byte, or many control chars, means "don't treat as text". function looksBinary(bytes) { const n = Math.min(bytes.length, 512); diff --git a/tests/protect/header-redaction-without-a-body.test.ts b/tests/protect/header-redaction-without-a-body.test.ts new file mode 100644 index 00000000..f86ac728 --- /dev/null +++ b/tests/protect/header-redaction-without-a-body.test.ts @@ -0,0 +1,245 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { gzipSync } from 'node:zlib'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A redaction keyed on a response header reads the header alone, so it is decided — and carried out — + * whether or not the body was screened: over the cap, binary, a live stream, or content-encoded. A rule + * that reads the body is still left out when there is no body to read. + * + * A rule's `prefilter` anchors are looked for in the header values as well as the body, since a rule + * may read either. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const BINARY = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0x02, 0x03]); + +const headerRule = (extra: Record = {}) => ({ + id: 'header-value', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [{ parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }], + ...extra, +}); + +async function guard(rules: object[], mode = 'block') { + const detections: any[] = []; + const skips: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode, + responseRules: rules, + onDetect: (event: any) => detections.push(event), + onSkip: (skip: any) => skips.push(skip), + onError: () => {}, + }); + + return { protection, detections, skips }; +} + +describe('a header redaction when the body is not screened (fetch)', () => { + it.each([ + ['a binary body', () => new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } }), 'non-text-content-type'], + ['a body over the cap', () => new Response('x'.repeat(600 * 1024), { headers: { 'content-type': 'text/plain', 'x-sample': SAMPLE } }), 'body-cap'], + ['a live stream', () => new Response('data: 1\n\n', { headers: { 'content-type': 'text/event-stream', 'x-sample': SAMPLE } }), 'live-stream'], + ['an encoded body', () => new Response(gzipSync('{"a":1}'), { headers: { 'content-type': 'application/json', 'content-encoding': 'gzip', 'x-sample': SAMPLE } }), 'encoded-body'], + ])('masks the header on %s and leaves the body alone', async (_label, make, reason) => { + const { protection, detections, skips } = await guard([headerRule()]); + const original = make(); + const expected = new Uint8Array(await original.clone().arrayBuffer()); + const out = await protection.screenResponse(original); + + expect(out.headers.get('x-sample')).toBe('[REDACTED]'); + expect(new Uint8Array(await out.arrayBuffer())).toEqual(expected); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).toContain(reason); + }); + + it('records the match in dry-run without changing the header', async () => { + const { protection, detections } = await guard([headerRule()], 'dry-run'); + const out = await protection.screenResponse(new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } })); + + expect(out.headers.get('x-sample')).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it.each([ + ['one that reads the body', { rule_v2: [{ parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }] }], + ['one that reads the status alone', { rule_v2: [{ parameter: 'response.status', match: { type: 'contains', value: '200' } }] }], + ['one that reads the body beside a header', { rule_v2: [ + { parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + { parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + ] }], + ['an encoding rule', { action: 'encode' }], + ['one whose match is decoded first', { rule_v2: [{ parameter: 'response.header.x-sample', mutations: ['urldecode'], match: { type: 'contains', value: SAMPLE } }] }], + ['one with no span to mask', { rule_v2: [{ parameter: 'response.header.x-sample', match: { type: 'isset' } }] }], + ])('leaves out %s', async (_label, shape) => { + const { protection, detections } = await guard([headerRule(shape)]); + const original = new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } }); + const out = await protection.screenResponse(original); + + expect(out).toBe(original); + expect(detections).toHaveLength(0); + }); +}); + +describe('a prefilter on a header redaction (fetch)', () => { + it.each([ + ['a screened body', () => new Response('{"ok":true}', { headers: { 'content-type': 'application/json', 'x-sample': SAMPLE } })], + ['an unscreened body', () => new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } })], + ])('finds its anchor in the header value with %s', async (_label, make) => { + const { protection, detections } = await guard([headerRule({ prefilter: ['sample-token'] })]); + const out = await protection.screenResponse(make()); + + expect(out.headers.get('x-sample')).toBe('[REDACTED]'); + expect(detections).toHaveLength(1); + }); + + it('still skips the rule when the anchor is nowhere in the response', async () => { + const { protection, detections } = await guard([headerRule({ prefilter: ['absent-anchor'] })]); + const out = await protection.screenResponse(new Response('{"ok":true}', { headers: { 'content-type': 'application/json', 'x-sample': SAMPLE } })); + + expect(out.headers.get('x-sample')).toBe(SAMPLE); + expect(detections).toHaveLength(0); + }); +}); + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + + return `http://127.0.0.1:${(server.address() as { port: number }).port}/`; +} + +async function rawGet(url: string): Promise<{ headers: Record; body: Buffer }> { + const { request } = await import('node:http'); + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ headers: res.headers, body: Buffer.concat(chunks) })); + }).on('error', reject).end(); + }); +} + +describe('a header redaction when the body is not screened (Node)', () => { + const bodies: Array<[string, string, Buffer, Record, string]> = [ + ['a binary body', 'image/png', Buffer.from(BINARY), {}, 'non-text-content-type'], + ['a sniffed binary body', 'application/octet-stream', Buffer.from(BINARY), {}, 'binary-body'], + ['an encoded body', 'application/json', gzipSync('{"a":1}'), { 'content-encoding': 'gzip' }, 'encoded-body'], + ['a body over the cap', 'text/plain', Buffer.from('x'.repeat(600 * 1024)), {}, 'body-cap'], + ]; + + it.each(bodies)('masks the header on %s, set before the body', async (_label, type, body, extra, reason) => { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(extra)) res.setHeader(name, value); + res.setHeader('x-sample', SAMPLE); + res.end(body); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.equals(body)).toBe(true); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).toContain(reason); + }); + + it.each(bodies)('masks the header on %s, supplied through writeHead', async (_label, type, body, extra) => { + const { protection, detections } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.writeHead(200, { 'content-type': type, 'x-sample': SAMPLE, ...extra }); + res.end(body); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.equals(body)).toBe(true); + expect(detections).toHaveLength(1); + }); + + it('reports a match whose header had already gone, and records that it could not be masked', async () => { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + res.end(Buffer.from(BINARY)); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe(SAMPLE); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips).toContainEqual(expect.objectContaining({ reason: 'headers-sent', detail: { headers: ['x-sample'] } })); + }); + + it('reports a screened text response once', async () => { + const { protection, detections } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('x-sample', SAMPLE); + res.end('{"ok":true}'); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.toString()).toBe('{"ok":true}'); + expect(detections).toHaveLength(1); + }); + + it('leaves the header alone in dry-run', async () => { + const { protection, detections } = await guard([headerRule()], 'dry-run'); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.end(Buffer.from(BINARY)); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it('finds a prefilter anchor in the header value', async () => { + const { protection, detections } = await guard([headerRule({ prefilter: ['sample-token'] })]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('x-sample', SAMPLE); + res.end('{"ok":true}'); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(detections).toHaveLength(1); + }); +}); From 39987953a8e32a89bfec82ec8bbf2071be9bac18 Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 16:42:11 +0200 Subject: [PATCH 2/8] Enforce header-only block rules when the body is not screened An explicit block rule that reads response headers only is now enforced on responses whose body is not screened, on both paths, while the head can still change. If it has already been sent, the match is reported and recorded as a headers-sent skip. The Node withheld response carries only its own framing headers. Co-Authored-By: Claude Opus 5.5 --- src/protect/runtime.js | 101 +++++-- .../header-block-without-a-body.test.ts | 260 ++++++++++++++++++ 2 files changed, 332 insertions(+), 29 deletions(-) create mode 100644 tests/protect/header-block-without-a-body.test.ts diff --git a/src/protect/runtime.js b/src/protect/runtime.js index f7923d87..fcfe65db 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -410,13 +410,15 @@ export async function createProtection(options = {}) { engine: new RuleEngine({ firewall: [rule], onError }), redactors, mutatedSpan, - // A redaction that reads response headers only, and masks spans in them, is decided and carried - // out without the body — so it still applies to a response whose body was not screened. + // A redaction that reads response headers only, and masks spans in them, and an explicit block + // that reads response headers only, are decided and carried out without the body — so they still + // apply to a response whose body was not screened. redactsHeaders: rule.action === 'redact' && !mutatedSpan && (redactors ?? []).some((r) => !r.jsonPath) && readsOnlyResponseHeaders(rule), + blocksOnHeaders: rule.action === 'block' && readsOnlyResponseHeaders(rule), // Optional cheap pre-filter: literal anchor(s) that MUST appear for the (expensive) regex to // have any chance of matching. Lets screenText skip the full scan on responses with no candidate // — the common case — cutting CPU/latency and shrinking the regex/ReDoS surface. @@ -546,10 +548,11 @@ export async function createProtection(options = {}) { return effectiveMode === 'block' ? block() : allow(); }; - // The rules a response whose body was not read can still be screened against: header hardening, and - // redactions of header values. Neither reads the body, and neither needs one to act on. - const decidedWithoutBody = (rule, entry) => hardensWithoutBody(rule) || Boolean(entry?.redactsHeaders); - const redactsHeaders = (_rule, entry) => Boolean(entry?.redactsHeaders); + // The rules a response whose body was not read can still be screened against: header hardening, + // redactions of header values, and blocks decided on headers. None reads the body, and none needs one + // to act on. + const decidedByHeaders = (_rule, entry) => Boolean(entry?.redactsHeaders || entry?.blocksOnHeaders); + const decidedWithoutBody = (rule, entry) => hardensWithoutBody(rule) || decidedByHeaders(rule, entry); // Response phase core: screen a text body → { verdict: 'pass'|'block'|'redact', body? }. // redact masks matched spans; block withholds; block wins over redact. Enforcement only in @@ -1000,8 +1003,7 @@ export async function createProtection(options = {}) { try { const meta = { status: response.status, headers: headerObject(response.headers) }; const r = screenText('', meta, reqCtx, decidedWithoutBody); - // `block` cannot arise: only header actions, and redactions that have a header span to mask, - // were eligible. + if (r.verdict === 'block') return leakResponse(); if (r.verdict !== 'redact' || !r.headers) return response; // A matched rule is not a changed header. `harden-cookie` on a response that sets no cookie, @@ -1040,6 +1042,7 @@ export async function createProtection(options = {}) { const chunks = []; let size = 0; let overflow = false; + let withheld = false; const MAX = screenCap; /** @@ -1269,17 +1272,24 @@ export async function createProtection(options = {}) { }; /** - * Redactions of header values, for a response whose body will not be screened. + * The rules decided on headers alone — redactions of header values, and blocks — for a response whose + * body will not be screened. * - * Such a rule reads headers only, so it is decided the same way whether or not the body is read — it - * is run here instead of at `end`, never as well, so it reports once. Returns the changes for - * `sendHead`, or undefined when there are none. A head that has already gone cannot take them, which - * is recorded, as it is for a screened body. + * Such a rule is decided the same way whether or not the body is read, so it is run here instead of at + * `end`, never as well, and reports once. Returns `{ withhold: true }` for a block, the header changes + * for `sendHead`, or undefined when there are none. A head that has already gone can take neither, + * which is recorded, as it is for a screened body. */ - const redactUnreadHeaders = () => { + const screenUnreadHead = () => { try { const head = effectiveHead(); - const r = screenText('', head, reqCtx, redactsHeaders); + const r = screenText('', head, reqCtx, decidedByHeaders); + if (r.verdict === 'block') { + if (!reallySent()) return { withhold: true }; + recordSkip('response', 'headers-sent', { action: 'block' }); + + return undefined; + } if (r.verdict !== 'redact' || !r.headers) return undefined; const changed = new Map(); for (const [name, value] of Object.entries(r.headers)) { @@ -1331,6 +1341,26 @@ export async function createProtection(options = {}) { origWriteHead(...args); }; + /** + * Send the generic withheld response in place of the application's, before its head has gone. + * + * Only its own framing goes out. Every header the application set is dropped: they described a + * response that is not the one sent, and a header can be what the verdict was about. Anything the + * application writes afterwards is discarded. + */ + const sendWithheld = (cb) => { + const body = JSON.stringify({ error: 'Response withheld by Patchstack (sensitive data detected)' }); + const headers = new Map(); + for (const name of Object.keys(effectiveHead().headers)) headers.set(name.toLowerCase(), null); + headers.set('content-type', 'application/json'); + headers.set('content-length', String(Buffer.byteLength(body))); + headers.set('transfer-encoding', null); + withheld = true; + sendHead({ status: 500, headers }); + + return origEnd(body, cb); + }; + /** Header changes onto the response's own header state, for the head Node writes implicitly. */ const applyToResponse = (changes) => { if (changes.status !== undefined) res.statusCode = changes.status; @@ -1352,7 +1382,13 @@ export async function createProtection(options = {}) { // Too big to screen — abandon buffering, but FLUSH what we already captured (the head) plus // this chunk before switching to pass-through, so the client gets a complete body (not a // truncated one missing everything before the cap was hit). - sendHead(redactUnreadHeaders()); + const unread = screenUnreadHead(); + if (unread?.withhold) { + sendWithheld(); + + return; + } + sendHead(unread); for (const c of chunks) origWrite(c); chunks.length = 0; origWrite(buf); @@ -1363,6 +1399,12 @@ export async function createProtection(options = {}) { chunks.push(buf); }; res.write = function (chunk, enc, cb) { + if (withheld) { + if (typeof enc === 'function') enc(); + else if (typeof cb === 'function') cb(); + + return true; + } if (overflow) return origWrite(chunk, enc, cb); collect(chunk, enc); if (typeof enc === 'function') enc(); @@ -1372,12 +1414,23 @@ export async function createProtection(options = {}) { res.end = function (chunk, enc, cb) { if (typeof chunk === 'function') { cb = chunk; chunk = undefined; enc = undefined; } else if (typeof enc === 'function') { cb = enc; enc = undefined; } + if (withheld) { + if (typeof cb === 'function') cb(); + + return res; + } if (overflow) { if (chunk != null) origWrite(chunk, enc); return origEnd(cb); } collect(chunk, enc); + if (withheld) { + if (typeof cb === 'function') cb(); + + return res; + } if (overflow) return origEnd(cb); // collect just flushed head + final chunk on overflow hardenOnce(); const passThrough = (changes) => { + if (changes?.withhold) return sendWithheld(cb); sendHead(changes); for (const c of chunks) origWrite(c); @@ -1393,7 +1446,7 @@ export async function createProtection(options = {}) { if (kind === 'skip' || (kind === 'sniff' && looksBinary(buffer))) { recordSkip('response', kind === 'skip' ? (baseContentType(ct) === 'text/event-stream' ? 'live-stream' : 'non-text-content-type') : 'binary-body'); - return passThrough(redactUnreadHeaders()); + return passThrough(screenUnreadHead()); } // Encoded bytes cannot be screened as text: sent exactly as they are, under their own coding. The // coding is read from the head that will be sent, which includes a held `writeHead`'s own headers. @@ -1401,7 +1454,7 @@ export async function createProtection(options = {}) { if (codings.length > 0 && stillEncoded(buffer)) { recordSkip('response', 'encoded-body', { encoding: codings.join(', ') }); - return passThrough(redactUnreadHeaders()); + return passThrough(screenUnreadHead()); } const text = buffer.toString('utf8'); let r; @@ -1426,7 +1479,6 @@ export async function createProtection(options = {}) { } if (r.verdict === 'block') { - const body = JSON.stringify({ error: 'Response withheld by Patchstack (sensitive data detected)' }); if (reallySent()) { // Chunked, so the body can still change, but the status already went out as the application's. res.destroy?.(); @@ -1435,16 +1487,7 @@ export async function createProtection(options = {}) { } // One framing: the length describes this body, so any transfer coding the application chose // for its own goes with it. - sendHead({ - status: 500, - headers: new Map([ - ['content-type', 'application/json'], - ['content-length', String(Buffer.byteLength(body))], - ['transfer-encoding', null], - ]), - }); - - return origEnd(body, cb); + return sendWithheld(cb); } // Redact: the rule's header values, and a length for the rewritten body. A length is always diff --git a/tests/protect/header-block-without-a-body.test.ts b/tests/protect/header-block-without-a-body.test.ts new file mode 100644 index 00000000..65ef6392 --- /dev/null +++ b/tests/protect/header-block-without-a-body.test.ts @@ -0,0 +1,260 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { gzipSync } from 'node:zlib'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A `block` rule that reads response headers alone is decided without the body, so it is enforced on a + * response whose body was not screened — over the cap, binary, a live stream, or content-encoded — as + * long as the head has not gone out. When it has, the match is reported and the limitation is recorded + * as a `headers-sent` skip. A rule that reads the body, or has no explicit `block` action, is left out + * when there is no body. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const BINARY = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0x02, 0x03]); +const WITHHELD = { error: 'Response withheld by Patchstack (sensitive data detected)' }; + +const blockRule = (extra: Record = {}) => ({ + id: 'header-block', + phase: 'response', + category: 'secret-exposure', + action: 'block', + rule_v2: [{ parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }], + ...extra, +}); + +async function guard(rules: object[], mode = 'block') { + const detections: any[] = []; + const skips: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode, + responseRules: rules, + onDetect: (event: any) => detections.push(event), + onSkip: (skip: any) => skips.push(skip), + onError: () => {}, + }); + + return { protection, detections, skips }; +} + +const unscreened: Array<[string, () => Response]> = [ + ['a binary body', () => new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } })], + ['a body over the cap', () => new Response('x'.repeat(600 * 1024), { headers: { 'content-type': 'text/plain', 'x-sample': SAMPLE } })], + ['a live stream', () => new Response('data: 1\n\n', { headers: { 'content-type': 'text/event-stream', 'x-sample': SAMPLE } })], + ['an encoded body', () => new Response(gzipSync('{"a":1}'), { headers: { 'content-type': 'application/json', 'content-encoding': 'gzip', 'x-sample': SAMPLE } })], +]; + +describe('a header block when the body is not screened (fetch)', () => { + it.each(unscreened)('withholds %s', async (_label, make) => { + const { protection, detections } = await guard([blockRule()]); + const out = await protection.screenResponse(make()); + + expect(out.status).toBe(500); + expect(await out.json()).toEqual(WITHHELD); + expect(out.headers.get('x-sample')).toBeNull(); + expect(detections).toHaveLength(1); + }); + + it('wins over a header redaction on the same response', async () => { + const redact = { ...blockRule(), id: 'header-redact', action: 'redact' }; + const { protection, detections } = await guard([redact, blockRule()]); + const out = await protection.screenResponse(unscreened[0][1]()); + + expect(out.status).toBe(500); + expect(detections).toHaveLength(2); + }); + + it('records the match in dry-run and sends the response', async () => { + const { protection, detections } = await guard([blockRule()], 'dry-run'); + const original = unscreened[0][1](); + + expect(await protection.screenResponse(original)).toBe(original); + expect(detections).toHaveLength(1); + }); + + it.each([ + ['one that reads the body', { rule_v2: [{ parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }] }], + ['one that reads the body beside a header', { rule_v2: [ + { parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + { parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + ] }], + ['one with no explicit action', { action: undefined }], + ])('leaves out %s', async (_label, shape) => { + const { protection, detections } = await guard([blockRule(shape)]); + const original = unscreened[0][1](); + + expect(await protection.screenResponse(original)).toBe(original); + expect(detections).toHaveLength(0); + }); +}); + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + + return `http://127.0.0.1:${(server.address() as { port: number }).port}/`; +} + +async function rawGet(url: string): Promise<{ status: number; headers: Record; body: Buffer }> { + const { request } = await import('node:http'); + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: Buffer.concat(chunks) })); + }).on('error', reject).end(); + }); +} + +describe('a header block when the body is not screened (Node)', () => { + const bodies: Array<[string, string, Buffer, Record]> = [ + ['a binary body', 'image/png', Buffer.from(BINARY), {}], + ['a sniffed binary body', 'application/octet-stream', Buffer.from(BINARY), {}], + ['an encoded body', 'application/json', gzipSync('{"a":1}'), { 'content-encoding': 'gzip' }], + ['a body over the cap', 'text/plain', Buffer.from('x'.repeat(600 * 1024)), {}], + ]; + + function expectWithheld(got: { status: number; headers: Record; body: Buffer }) { + expect(got.status).toBe(500); + expect(Number(got.headers['content-length'])).toBe(got.body.length); + expect(JSON.parse(got.body.toString())).toEqual(WITHHELD); + expect(got.headers['x-sample']).toBeUndefined(); + expect(got.headers['content-encoding']).toBeUndefined(); + } + + it.each(bodies)('withholds %s, headers set before the body', async (_label, type, body, extra) => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(extra)) res.setHeader(name, value); + res.setHeader('x-sample', SAMPLE); + res.end(body); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); + + it.each(bodies)('withholds %s, headers supplied through writeHead', async (_label, type, body, extra) => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.writeHead(200, 'OK', { 'content-type': type, 'x-sample': SAMPLE, ...extra }); + res.end(body); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); + + it('withholds a body that crosses the cap across writes, and drops what follows', async () => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const errors: unknown[] = []; + const written: boolean[] = []; + const ended = new Promise((resolve) => { + void listen((req, res) => + node(req, res, () => { + res.on('error', (error) => errors.push(error)); + res.setHeader('content-type', 'text/plain'); + res.setHeader('x-sample', SAMPLE); + for (let i = 0; i < 4; i++) written.push(res.write('x'.repeat(200 * 1024))); + res.end('tail', () => resolve()); + }), + ).then((url) => rawGet(url).then((got) => { + expectWithheld(got); + })); + }); + + await ended; + expect(errors).toEqual([]); + expect(written.every(Boolean)).toBe(true); + expect(detections).toHaveLength(1); + }); + + it('completes the application\'s end once when the cap is crossed by its final chunk', async () => { + const { protection } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const calls: unknown[] = []; + const errors: unknown[] = []; + const url = await listen((req, res) => + node(req, res, () => { + res.on('error', (error) => errors.push(error)); + res.setHeader('content-type', 'text/plain'); + res.setHeader('x-sample', SAMPLE); + res.end('x'.repeat(600 * 1024), (error?: unknown) => calls.push(error ?? null)); + }), + ); + + expectWithheld(await rawGet(url)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(calls).toEqual([null]); + expect(errors).toEqual([]); + }); + + it('reports a match whose head had already gone, and records that it could not be enforced', async () => { + const { protection, detections, skips } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + res.end(Buffer.from(BINARY)); + }), + ); + + const got = await rawGet(url); + expect(got.status).toBe(200); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips).toContainEqual(expect.objectContaining({ reason: 'headers-sent', detail: { action: 'block' } })); + }); + + it('leaves the response alone in dry-run', async () => { + const { protection, detections } = await guard([blockRule()], 'dry-run'); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.end(Buffer.from(BINARY)); + }), + ); + + const got = await rawGet(url); + expect(got.status).toBe(200); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + }); + + it('withholds a screened text response once', async () => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('x-sample', SAMPLE); + res.end('{"ok":true}'); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); +}); From abeba4e534c357fcb214a93854cbc8c9f2196fa1 Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 17:00:13 +0200 Subject: [PATCH 3/8] Screen header-only rules when the head is flushed An explicit flushHeaders() now runs the rules decided on headers alone before the head goes out: a block withholds the response and a redaction masks the header. They run once per response, so the screen at end does not report them again. Co-Authored-By: Claude Opus 5.5 --- src/protect/runtime.js | 32 ++++++++--- .../header-block-without-a-body.test.ts | 56 ++++++++++++++++--- .../header-redaction-without-a-body.test.ts | 56 +++++++++++++++++-- 3 files changed, 125 insertions(+), 19 deletions(-) diff --git a/src/protect/runtime.js b/src/protect/runtime.js index fcfe65db..7e889a1e 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -1043,6 +1043,8 @@ export async function createProtection(options = {}) { let size = 0; let overflow = false; let withheld = false; + // Whether the rules decided on headers alone have already been run for this response. + let headScreened = false; const MAX = screenCap; /** @@ -1110,6 +1112,10 @@ export async function createProtection(options = {}) { // The rules the pass above did not answer: the ones that read the body, which is why a // body-reading hardening rule is still honoured at `end`. const stillToAnswer = (rule) => !hardensWithoutBody(rule); + // What is left for the screen at `end`: neither the hardening answered before the head, nor the rules + // decided on headers alone once they have run. + const notYetAnswered = (rule, entry) => + (!answeredWithoutBody || stillToAnswer(rule)) && !(headScreened && decidedByHeaders(rule, entry)); /** * `writeHead`, held until the first byte actually leaves. @@ -1217,12 +1223,19 @@ export async function createProtection(options = {}) { return original(...args); }; } - // An explicit flush asks for the head now, so it goes now, hardened. The body that follows can then - // only change where no length was promised, which `end` already accounts for. + // An explicit flush asks for the head now, so it goes now — hardened, and screened by the rules + // decided on headers alone, which is the last point at which they can still change it. The body that + // follows can then only change where no length was promised, which `end` already accounts for. if (typeof res.flushHeaders === 'function') { const origFlushHeaders = res.flushHeaders.bind(res); res.flushHeaders = function () { - sendHead(); + const unread = screenUnreadHead(); + if (unread?.withhold) { + sendWithheld(); + + return undefined; + } + sendHead(unread); return origFlushHeaders(); }; @@ -1275,12 +1288,15 @@ export async function createProtection(options = {}) { * The rules decided on headers alone — redactions of header values, and blocks — for a response whose * body will not be screened. * - * Such a rule is decided the same way whether or not the body is read, so it is run here instead of at - * `end`, never as well, and reports once. Returns `{ withhold: true }` for a block, the header changes - * for `sendHead`, or undefined when there are none. A head that has already gone can take neither, - * which is recorded, as it is for a screened body. + * Such a rule is decided the same way whether or not the body is read, so it runs once per response — + * at an explicit flush, or on a branch that does not screen the body — and is left out of the screen at + * `end`. Returns `{ withhold: true }` for a block, the header changes for `sendHead`, or undefined when + * there are none. A head that had already gone can take neither, which is recorded, as it is for a + * screened body. */ const screenUnreadHead = () => { + if (headScreened) return undefined; + headScreened = true; try { const head = effectiveHead(); const r = screenText('', head, reqCtx, decidedByHeaders); @@ -1459,7 +1475,7 @@ export async function createProtection(options = {}) { const text = buffer.toString('utf8'); let r; try { - r = screenText(text, head, reqCtx, answeredWithoutBody ? stillToAnswer : undefined); + r = screenText(text, head, reqCtx, notYetAnswered); } catch (err) { notify(onError, err, 'onError'); diff --git a/tests/protect/header-block-without-a-body.test.ts b/tests/protect/header-block-without-a-body.test.ts index 65ef6392..5ebd7d36 100644 --- a/tests/protect/header-block-without-a-body.test.ts +++ b/tests/protect/header-block-without-a-body.test.ts @@ -207,23 +207,65 @@ describe('a header block when the body is not screened (Node)', () => { expect(errors).toEqual([]); }); - it('reports a match whose head had already gone, and records that it could not be enforced', async () => { + it('reports a match whose head went out before the guard saw it, and records that it could not be enforced', async () => { const { protection, detections, skips } = await guard([blockRule()]); const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + node(req, res, () => res.end(Buffer.from(BINARY))); + }); + + const got = await rawGet(url); + expect(got.status).toBe(200); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips).toContainEqual(expect.objectContaining({ reason: 'headers-sent', detail: { action: 'block' } })); + }); + + it.each([ + ['a streamed text body', 'text/plain', ['first ', 'second ', 'third']], + ['a streamed binary body', 'application/octet-stream', [Buffer.from(BINARY), Buffer.from(BINARY)]], + ])('withholds on flushHeaders before %s', async (_label, type, parts) => { + for (const viaWriteHead of [false, true]) { + const { protection, detections, skips } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + if (viaWriteHead) res.writeHead(200, { 'content-type': type, 'x-sample': SAMPLE }); + else { + res.setHeader('content-type', type); + res.setHeader('x-sample', SAMPLE); + } + res.flushHeaders(); + for (const part of parts.slice(0, -1)) res.write(part); + res.end(parts[parts.length - 1]); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).not.toContain('headers-sent'); + await close?.(); + close = null; + } + }); + + it('withholds a body written before the application ends it, with no explicit head', async () => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); const url = await listen((req, res) => node(req, res, () => { - res.setHeader('content-type', 'image/png'); + res.setHeader('content-type', 'application/octet-stream'); res.setHeader('x-sample', SAMPLE); - res.flushHeaders(); + res.write(Buffer.from(BINARY)); res.end(Buffer.from(BINARY)); }), ); - const got = await rawGet(url); - expect(got.status).toBe(200); - expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expectWithheld(await rawGet(url)); expect(detections).toHaveLength(1); - expect(skips).toContainEqual(expect.objectContaining({ reason: 'headers-sent', detail: { action: 'block' } })); }); it('leaves the response alone in dry-run', async () => { diff --git a/tests/protect/header-redaction-without-a-body.test.ts b/tests/protect/header-redaction-without-a-body.test.ts index f86ac728..3eab87fa 100644 --- a/tests/protect/header-redaction-without-a-body.test.ts +++ b/tests/protect/header-redaction-without-a-body.test.ts @@ -175,18 +175,66 @@ describe('a header redaction when the body is not screened (Node)', () => { expect(detections).toHaveLength(1); }); - it('reports a match whose header had already gone, and records that it could not be masked', async () => { - const { protection, detections, skips } = await guard([headerRule()]); + it.each([ + ['a streamed text body', 'text/plain', ['first ', 'second ', 'third']], + ['a streamed binary body', 'image/png', [Buffer.from(BINARY), Buffer.from(BINARY)]], + ])('masks the header on flushHeaders before %s', async (_label, type, parts) => { + for (const viaWriteHead of [false, true]) { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + if (viaWriteHead) res.writeHead(200, { 'content-type': type, 'x-sample': SAMPLE }); + else { + res.setHeader('content-type', type); + res.setHeader('x-sample', SAMPLE); + } + res.flushHeaders(); + for (const part of parts.slice(0, -1)) res.write(part); + res.end(parts[parts.length - 1]); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.equals(Buffer.concat(parts.map((p) => Buffer.from(p))))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).not.toContain('headers-sent'); + await close?.(); + close = null; + } + }); + + it.each([ + ['a text body', 'text/plain', 'plain text'], + ['a binary body', 'image/png', Buffer.from(BINARY)], + ])('reports a match once in dry-run when flushHeaders comes before %s', async (_label, type, body) => { + const { protection, detections } = await guard([headerRule()], 'dry-run'); const node = protection.node({ screenResponses: true }); const url = await listen((req, res) => node(req, res, () => { - res.setHeader('content-type', 'image/png'); + res.setHeader('content-type', type); res.setHeader('x-sample', SAMPLE); res.flushHeaders(); - res.end(Buffer.from(BINARY)); + res.end(body); }), ); + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it('reports a match whose header went out before the guard saw it, and records that it could not be masked', async () => { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + node(req, res, () => res.end(Buffer.from(BINARY))); + }); + const got = await rawGet(url); expect(got.headers['x-sample']).toBe(SAMPLE); expect(got.body.equals(Buffer.from(BINARY))).toBe(true); From f4ba55b6fb988eaeed29a74b0d4ff638b972e724 Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 17:25:43 +0200 Subject: [PATCH 4/8] Document how header-only response rules apply Co-Authored-By: Claude Opus 5.5 --- src/protect/protect.d.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/protect/protect.d.ts b/src/protect/protect.d.ts index f58d14e7..a0fa3680 100644 --- a/src/protect/protect.d.ts +++ b/src/protect/protect.d.ts @@ -29,6 +29,11 @@ export interface Protection { * * The client address is the one resolved when this guard screened that request, if it did; otherwise it * is resolved here, and any further arguments are passed to `peerAddress` as the host's handler arguments. + * + * A rule that reads only response headers (`response.header.*`, `response.headers`) redacts or blocks on + * the headers alone, so it is enforced even when the body cannot be screened. A redaction masks only the + * header value it matched; protecting the body takes a rule on `response.body`. A withheld response + * carries only its own `content-type` and `content-length`. */ screenResponse(response: Response, request?: Request, ...hostArgs: unknown[]): Promise; express(options?: { screenResponses?: boolean }): (req: unknown, res: unknown, next: () => void) => void; @@ -344,7 +349,10 @@ export interface CreateProtectionOptions { header?: string; isTrusted?: (ip: string) => boolean; }; - /** Override the default response-phase (secret-leak) rule set. */ + /** + * Override the default response-phase (secret-leak) rule set. A rule that reads only response headers is + * enforced from the headers, and masks only the header it matched — see `screenResponse`. + */ responseRules?: unknown[]; /** Override the default egress-phase (SSRF) rule set. */ egressRules?: unknown[]; From f0495a2e42802510167e0978dd3dee23edeb557a Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 17:52:47 +0200 Subject: [PATCH 5/8] Mask each redaction where its condition read A span redaction from a response.header. condition now masks that header only, and one from response.headers masks the headers only; neither rewrites the body. A body condition masks the body and the same text in the headers, as before. Co-Authored-By: Claude Opus 5.5 --- src/protect/protect.d.ts | 5 +- src/protect/runtime.js | 45 +++- .../protect/response-redaction-scope.test.ts | 244 ++++++++++++++++++ 3 files changed, 282 insertions(+), 12 deletions(-) create mode 100644 tests/protect/response-redaction-scope.test.ts diff --git a/src/protect/protect.d.ts b/src/protect/protect.d.ts index a0fa3680..aeb0e247 100644 --- a/src/protect/protect.d.ts +++ b/src/protect/protect.d.ts @@ -32,8 +32,9 @@ export interface Protection { * * A rule that reads only response headers (`response.header.*`, `response.headers`) redacts or blocks on * the headers alone, so it is enforced even when the body cannot be screened. A redaction masks only the - * header value it matched; protecting the body takes a rule on `response.body`. A withheld response - * carries only its own `content-type` and `content-length`. + * header value it matched; protecting the body takes a rule on `response.body`, which masks the body and + * the same text wherever it appears in a header. A withheld response carries only its own `content-type` + * and `content-length`. */ screenResponse(response: Response, request?: Request, ...hostArgs: unknown[]): Promise; express(options?: { screenResponses?: boolean }): (req: unknown, res: unknown, next: () => void) => void; diff --git a/src/protect/runtime.js b/src/protect/runtime.js index 7e889a1e..d5353027 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -626,9 +626,9 @@ export async function createProtection(options = {}) { if (mode !== 'block' || (!blockRule && !redactions.length && !headerMutations.length)) return { verdict: 'pass' }; if (blockRule) return { verdict: 'block' }; let body = text; - // Redact the offending spans in the body AND in every (string) header value — so a secret - // that leaks in a header (Set-Cookie, an echoed X-Api-Key, …) is masked too, and a rule that - // targets `response.header.*` actually strips the header rather than just detecting it. + // Each span redactor masks where its condition read (`redactorScope`): a body condition the body and + // every (string) header value — so a secret that leaks in a header (Set-Cookie, an echoed X-Api-Key, + // …) is masked too — and a header condition only the header it read. const headers = { ...(meta.headers || {}) }; // Whether the response is a JSON document, asked only once a span rewrite needs it, and the lexed // structure of the body as it stands — reused by the next span rewrite while the body is unchanged. @@ -638,14 +638,17 @@ export async function createProtection(options = {}) { const mask = maskFn(rule.category); // action `encode` HTML-escapes the matched value in place (neutralize stored XSS at output); // `redact` masks it. jsonPath redactors act on a structural JSON location, span redactors on - // text spans in the body AND header values. Apply structural first (on clean JSON), then spans. + // text spans in their scope. Apply structural first (on clean JSON), then spans. const transform = rule.action === 'encode' ? htmlEscape : null; const pathRedactors = redactors.filter((r) => r.jsonPath); const spanRedactors = redactors.filter((r) => !r.jsonPath); if (pathRedactors.length) body = applyPathRedactors(body, pathRedactors, mask, screenCap, transform); if (!spanRedactors.length) continue; const beforeSpan = body; - body = applyRedactors(body, spanRedactors, mask, transform); + // A redactor masks the body only when its condition read the body; one that read only a header + // leaves the body unchanged, and so never reaches the structure check below. + const bodySpans = spanRedactors.filter((r) => r.scope.body === true); + if (bodySpans.length) body = applyRedactors(body, bodySpans, mask, transform); // Span rewrites may change JSON string values, but not keys, containers or other values, and a // rewrite that would produce an invalid document is withheld rather than sent. Each result is // checked before it becomes input to another transformation. A response that was a JSON document @@ -663,14 +666,15 @@ export async function createProtection(options = {}) { if (transform) continue; // encoding is a body/output concern — headers aren't HTML for (const name of Object.keys(headers)) { const value = headers[name]; + const headerSpans = spanRedactors.filter((r) => masksHeader(r, name)); if (typeof value === 'string') { - setOwn(headers, name, applyRedactors(value, spanRedactors, mask)); + setOwn(headers, name, applyRedactors(value, headerSpans, mask)); } else if (Array.isArray(value)) { // Multi-valued headers (Set-Cookie) — redact each entry. setOwn( headers, name, - value.map((item) => (typeof item === 'string' ? applyRedactors(item, spanRedactors, mask) : item)), + value.map((item) => (typeof item === 'string' ? applyRedactors(item, headerSpans, mask) : item)), ); } } @@ -2208,6 +2212,27 @@ function hasSpanMutations(rule) { return found; } +/** + * Where a span redactor masks, from the parameter its condition read. + * + * A single response header — `response.header.` — masks that header only, and `response.headers` + * masks the headers only: a rule that read no body does not rewrite it. Any other parameter, the body + * among them, masks the body and the same text in every header, so a secret found in the body is not left + * behind where it was echoed into a header. + */ +function redactorScope(parameter) { + const name = typeof parameter === 'string' ? parameter : ''; + if (name.startsWith('response.header.')) return { header: name.slice('response.header.'.length).toLowerCase() }; + if (name === 'response.headers') return { headers: true }; + + return { body: true, headers: true }; +} + +/** Does a span redactor apply to this header? `name` is lower-cased, as every screened header name is. */ +function masksHeader(redactor, name) { + return redactor.scope.headers === true || redactor.scope.header === name; +} + // Derive redaction targets from a rule's own conditions: regex → mask every match; // contains/stripos → mask the literal. (Other match types can't identify a span → the // rule falls back to block.) @@ -2225,19 +2250,19 @@ function extractRedactors(rule) { if (safe) { const flags = safe.flags.includes('g') ? safe.flags : safe.flags + 'g'; try { - out.push({ re: new RegExp(safe.source, flags) }); + out.push({ re: new RegExp(safe.source, flags), scope: redactorScope(c.parameter) }); } catch { /* skip invalid */ } } } else if ((m.type === 'contains' || m.type === 'stripos') && m.value != null) { - out.push({ literal: String(m.value) }); + out.push({ literal: String(m.value), scope: redactorScope(c.parameter) }); } else if (m.type === 'jwt_claim_equals' && typeof m.claim === 'string') { // A span-producing target, not a predicate. A boolean-only matcher would leave `redact` with // no span to mask, so the rule would fall back to withholding the WHOLE response — turning a // one-token leak into an outage. The spans come from the same `jwtClaimSpans` the matcher // used, so what is reported and what is masked cannot diverge. - out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') } }); + out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') }, scope: redactorScope(c.parameter) }); } else if (m.type === 'array_key_value' && m.match && isBodyParam(c.parameter)) { // Structural redaction: mask the value at a JSON path (fanning out over arrays) rather than // a text span — e.g. key "orders.customers.email" masks that field in every array element. diff --git a/tests/protect/response-redaction-scope.test.ts b/tests/protect/response-redaction-scope.test.ts new file mode 100644 index 00000000..57b47610 --- /dev/null +++ b/tests/protect/response-redaction-scope.test.ts @@ -0,0 +1,244 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A redaction masks where its condition read. A `response.header.` condition masks that header + * only — each entry of a multi-valued one — and `response.headers` masks the headers only; neither + * touches the body. A `response.body` condition masks the body, and the same text in the headers. A rule + * with conditions on both masks each where it reads. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const PATTERN = { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' }; +const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url'); +const TOKEN = `${encode({ alg: 'none', typ: 'JWT' })}.${encode({ role: 'sample' })}.c2lnbmF0dXJl`; + +const rule = (...parameters: string[]) => ({ + id: 'scoped', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: parameters.map((parameter) => ({ parameter, match: PATTERN })), +}); + +async function guard(rules: object[], mode = 'block') { + const detections: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode, + responseRules: rules, + onDetect: (event: any) => detections.push(event), + onError: () => {}, + }); + + return { protection, detections }; +} + +type Case = { + label: string; + rules: object[]; + body: string; + headers: Record; + status: number; + expectBody: string; + expectHeaders: Record; +}; + +const cases: Case[] = [ + { + label: 'a header rule leaves the body alone', + rules: [rule('response.header.x-sample')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE }, + status: 200, + expectBody: `{"note":"${SAMPLE}"}`, + expectHeaders: { 'x-sample': '[REDACTED]' }, + }, + { + label: 'a header rule leaves an unrelated header alone', + rules: [rule('response.header.x-sample')], + body: '{"ok":true}', + headers: { 'x-sample': SAMPLE, 'x-other': SAMPLE }, + status: 200, + expectBody: '{"ok":true}', + expectHeaders: { 'x-sample': '[REDACTED]', 'x-other': SAMPLE }, + }, + { + label: 'a header rule leaves a JSON key alone', + rules: [rule('response.header.x-sample')], + body: `{"${SAMPLE}":1}`, + headers: { 'x-sample': SAMPLE }, + status: 200, + expectBody: `{"${SAMPLE}":1}`, + expectHeaders: { 'x-sample': '[REDACTED]' }, + }, + { + label: 'an all-headers rule masks every header and not the body', + rules: [rule('response.headers')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE, 'x-other': SAMPLE }, + status: 200, + expectBody: `{"note":"${SAMPLE}"}`, + expectHeaders: { 'x-sample': '[REDACTED]', 'x-other': '[REDACTED]' }, + }, + { + label: 'a body rule masks the body, and the same text in the headers', + rules: [rule('response.body')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE }, + status: 200, + expectBody: '{"note":"[REDACTED]"}', + expectHeaders: { 'x-sample': '[REDACTED]' }, + }, + { + label: 'a rule on a header and the body masks both', + rules: [rule('response.header.x-sample', 'response.body')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE, 'x-other': SAMPLE }, + status: 200, + expectBody: '{"note":"[REDACTED]"}', + expectHeaders: { 'x-sample': '[REDACTED]', 'x-other': '[REDACTED]' }, + }, +]; + +describe('redaction scope (fetch)', () => { + it.each(cases)('$label', async ({ rules, body, headers, status, expectBody, expectHeaders }) => { + const { protection } = await guard(rules); + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json', ...headers } })); + + expect(out.status).toBe(status); + expect(await out.text()).toBe(expectBody); + for (const [name, value] of Object.entries(expectHeaders)) expect(out.headers.get(name)).toBe(value); + }); + + it('masks a header rule entry by entry on a multi-valued header', async () => { + const { protection } = await guard([rule('response.header.set-cookie')]); + const headers = new Headers({ 'content-type': 'application/json' }); + headers.append('set-cookie', `a=${SAMPLE}`); + headers.append('set-cookie', 'b=plain'); + headers.append('x-other', SAMPLE); + const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}"}`, { headers })); + + expect(out.headers.getSetCookie()).toEqual(['a=[REDACTED]', 'b=plain']); + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(await out.text()).toBe(`{"note":"${SAMPLE}"}`); + }); + + it('encodes the body only for a rule on the body', async () => { + const encode = (parameter: string) => ({ ...rule(parameter), action: 'encode', rule_v2: [{ parameter, match: { type: 'contains', value: '' } }] }); + const body = JSON.stringify({ note: 'x' }); + for (const [parameter, expected] of [['response.header.x-sample', body], ['response.body', JSON.stringify({ note: '<b>x' })]]) { + const { protection } = await guard([encode(parameter)]); + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json', 'x-sample': '' } })); + + expect(await out.text()).toBe(expected); + expect(out.headers.get('x-sample')).toBe(''); + } + }); + + it.each([ + ['a mixed-case header name', { parameter: 'response.header.X-Sample', match: PATTERN }, SAMPLE], + ['a literal match', { parameter: 'response.header.x-sample', match: { type: 'contains', value: SAMPLE } }, SAMPLE], + ['a token claim', { parameter: 'response.header.x-sample', match: { type: 'jwt_claim_equals', claim: 'role', value: 'sample' } }, TOKEN], + ])('keeps %s to its header', async (_label, condition, value) => { + const { protection } = await guard([{ ...rule(), rule_v2: [condition] }]); + const out = await protection.screenResponse(new Response(`{"note":"${value}"}`, { + headers: { 'content-type': 'application/json', 'x-sample': value, 'x-other': value }, + })); + + expect(out.headers.get('x-sample')).not.toContain(value); + expect(out.headers.get('x-other')).toBe(value); + expect(await out.text()).toBe(`{"note":"${value}"}`); + }); + + it('keeps the scope when the body is not screened', async () => { + const { protection } = await guard([rule('response.header.x-sample')]); + const out = await protection.screenResponse(new Response(new Uint8Array([0x89, 0x50, 0x00, 0x01]), { + headers: { 'content-type': 'image/png', 'x-sample': SAMPLE, 'x-other': SAMPLE }, + })); + + expect(out.headers.get('x-sample')).toBe('[REDACTED]'); + expect(out.headers.get('x-other')).toBe(SAMPLE); + }); +}); + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + + return `http://127.0.0.1:${(server.address() as { port: number }).port}/`; +} + +async function rawGet(url: string): Promise<{ status: number; headers: Record; body: Buffer }> { + const { request } = await import('node:http'); + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: Buffer.concat(chunks) })); + }).on('error', reject).end(); + }); +} + +describe('redaction scope (Node)', () => { + async function serve(rules: object[], type: string, headers: Record, body: string | Buffer, flush: boolean) { + const g = await guard(rules); + const node = g.protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(headers)) res.setHeader(name, value); + if (flush) res.flushHeaders(); + res.end(body); + }), + ); + + return { ...g, got: await rawGet(url) }; + } + + it.each(cases)('$label', async ({ rules, body, headers, status, expectBody, expectHeaders }) => { + const { got } = await serve(rules, 'application/json', headers, body, false); + + expect(got.status).toBe(status); + expect(got.body.toString()).toBe(expectBody); + for (const [name, value] of Object.entries(expectHeaders)) expect(got.headers[name]).toBe(value); + }); + + it.each([ + ['an early flush', 'text/plain', 'plain text', true], + ['a body that is not screened', 'image/png', Buffer.from([0x89, 0x50, 0x00, 0x01]), false], + ])('keeps a header rule to its header on %s', async (_label, type, body, flush) => { + const { got, detections } = await serve([rule('response.header.x-sample')], type, { 'x-sample': SAMPLE, 'x-other': SAMPLE }, body, flush); + + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.headers['x-other']).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it('masks a header rule entry by entry on a multi-valued header', async () => { + const g = await guard([rule('response.header.set-cookie')]); + const node = g.protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('set-cookie', [`a=${SAMPLE}`, 'b=plain']); + res.setHeader('x-other', SAMPLE); + res.end(`{"note":"${SAMPLE}"}`); + }), + ); + const got = await rawGet(url); + + expect(got.headers['set-cookie']).toEqual(['a=[REDACTED]', 'b=plain']); + expect(got.headers['x-other']).toBe(SAMPLE); + expect(got.body.toString()).toBe(`{"note":"${SAMPLE}"}`); + }); +}); From e2cea45ccc485526218899bedd1c59c22cf7a0af Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 19:07:03 +0200 Subject: [PATCH 6/8] Scope redactions read through a parameter list A condition that names a list of parameters now masks the union of its members' scopes, so a list of response headers masks those headers only. Structural masking also applies when the body is read through a list. Co-Authored-By: Claude Opus 5.5 --- src/protect/runtime.js | 49 +++-- .../redaction-scope-parameter-forms.test.ts | 170 ++++++++++++++++++ 2 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 tests/protect/redaction-scope-parameter-forms.test.ts diff --git a/src/protect/runtime.js b/src/protect/runtime.js index d5353027..8f59b683 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -2213,24 +2213,46 @@ function hasSpanMutations(rule) { } /** - * Where a span redactor masks, from the parameter its condition read. + * Where a span redactor masks, from the parameter its condition read: `{ body, allHeaders, headers }`, + * `headers` being a set of lower-cased names. * - * A single response header — `response.header.` — masks that header only, and `response.headers` - * masks the headers only: a rule that read no body does not rewrite it. Any other parameter, the body - * among them, masks the body and the same text in every header, so a secret found in the body is not left - * behind where it was echoed into a header. + * - `response.header.` masks that header; `response.headers` masks every header. Neither is the + * body, and a rule that read no body does not rewrite it. + * - `response.body` masks the body and the same text in every header, so a secret found in the body is not + * left behind where it was echoed into a header. + * - A list of parameters — the engine reads each member — masks the union of its members' scopes. A + * member that is not a string resolves to nothing in the engine, and adds nothing here. + * - Anything else names no place in the response to mask: the status, a request source, a missing + * parameter, or one outside the contract. Such a condition keeps the widest scope, the body and every + * header, so its redaction still acts on the response rather than reporting a mask that changed nothing. + * + * A group (`parameter: "rules"`) is not a scope of its own: the engine evaluates each of its conditions + * with that condition's parameter, and each yields its own redactor with its own scope. */ function redactorScope(parameter) { - const name = typeof parameter === 'string' ? parameter : ''; - if (name.startsWith('response.header.')) return { header: name.slice('response.header.'.length).toLowerCase() }; - if (name === 'response.headers') return { headers: true }; + const scope = { body: false, allHeaders: false, headers: new Set() }; + const add = (name) => { + if (typeof name !== 'string') return; + if (name.startsWith('response.header.')) scope.headers.add(name.slice('response.header.'.length).toLowerCase()); + else if (name === 'response.headers') scope.allHeaders = true; + else { + scope.body = true; + scope.allHeaders = true; + } + }; + if (Array.isArray(parameter)) for (const member of parameter) add(member); + else if (typeof parameter === 'string') add(parameter); + else { + scope.body = true; + scope.allHeaders = true; + } - return { body: true, headers: true }; + return scope; } /** Does a span redactor apply to this header? `name` is lower-cased, as every screened header name is. */ function masksHeader(redactor, name) { - return redactor.scope.headers === true || redactor.scope.header === name; + return redactor.scope.allHeaders || redactor.scope.headers.has(name); } // Derive redaction targets from a rule's own conditions: regex → mask every match; @@ -2263,7 +2285,7 @@ function extractRedactors(rule) { // one-token leak into an outage. The spans come from the same `jwtClaimSpans` the matcher // used, so what is reported and what is masked cannot diverge. out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') }, scope: redactorScope(c.parameter) }); - } else if (m.type === 'array_key_value' && m.match && isBodyParam(c.parameter)) { + } else if (m.type === 'array_key_value' && m.match && readsBody(c.parameter)) { // Structural redaction: mask the value at a JSON path (fanning out over arrays) rather than // a text span — e.g. key "orders.customers.email" masks that field in every array element. const keys = Array.isArray(m.key) ? m.key : [m.key]; @@ -2436,6 +2458,11 @@ function applyRedactors(body, redactors, mask, transform) { // A response-body redaction target (array_key_value masks the JSON body). A bare condition with no // parameter also defaults to the body. +/** Does a condition read the body, alone or as a member of its parameter list? */ +function readsBody(parameter) { + return Array.isArray(parameter) ? parameter.some((member) => typeof member === 'string' && isBodyParam(member)) : isBodyParam(parameter); +} + function isBodyParam(parameter) { return parameter == null || parameter === 'response.body' || parameter === 'raw' || parameter === 'response.raw'; } diff --git a/tests/protect/redaction-scope-parameter-forms.test.ts b/tests/protect/redaction-scope-parameter-forms.test.ts new file mode 100644 index 00000000..ffb11dff --- /dev/null +++ b/tests/protect/redaction-scope-parameter-forms.test.ts @@ -0,0 +1,170 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A redaction's scope does not depend on how its parameter is written. A list masks the union of its + * members' scopes, a single-item list masks what the string does, and a condition inside a group masks + * where it reads. The same holds on every path a response can take: screened on fetch or Node, with the + * head flushed early, or with a body that is not screened. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const PATTERN = { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' }; +const HEADERS = { 'x-sample': SAMPLE, 'x-extra': SAMPLE, 'x-other': SAMPLE }; +const BINARY = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01]); + +type Form = { label: string; condition: Record; masked: string[]; readsBody: boolean }; + +const leaf = (parameter: unknown) => ({ parameter, match: PATTERN }); +const forms: Form[] = [ + { label: 'a header, as a string', condition: leaf('response.header.x-sample'), masked: ['x-sample'], readsBody: false }, + { label: 'a header, as a one-item list', condition: leaf(['response.header.x-sample']), masked: ['x-sample'], readsBody: false }, + { label: 'two headers, as a list', condition: leaf(['response.header.x-sample', 'response.header.X-Extra']), masked: ['x-sample', 'x-extra'], readsBody: false }, + { label: 'all headers, as a string', condition: leaf('response.headers'), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: false }, + { label: 'all headers, as a list', condition: leaf(['response.headers']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: false }, + { label: 'a header beside a nested list, which reads nothing', condition: leaf(['response.header.x-sample', ['response.body']]), masked: ['x-sample'], readsBody: false }, + { label: 'a header list in a group', condition: { parameter: 'rules', rules: [leaf(['response.header.x-sample'])] }, masked: ['x-sample'], readsBody: false }, + { label: 'the body, as a one-item list', condition: leaf(['response.body']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: true }, + { label: 'the body and a header, as a list', condition: leaf(['response.body', 'response.header.x-sample']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: true }, +]; + +const ruleOf = (condition: Record) => ({ + id: 'scoped', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [condition], +}); + +async function guard(condition: Record) { + const detections: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + responseRules: [ruleOf(condition)], + onDetect: (event: any) => detections.push(event), + onError: () => {}, + }); + + return { protection, detections }; +} + +type Seen = { status: number; body: Buffer; headers: Record }; + +const seenHeaders = (get: (name: string) => string | null | undefined) => + Object.fromEntries(Object.keys(HEADERS).map((name) => [name, get(name) ?? null])); + +async function viaFetch(condition: Record, body: string | Buffer, type: string): Promise { + const { protection } = await guard(condition); + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': type, ...HEADERS } })); + + return { status: out.status, body: Buffer.from(await out.arrayBuffer()), headers: seenHeaders((n) => out.headers.get(n)) }; +} + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function viaNode(condition: Record, body: string | Buffer, type: string, flush: boolean): Promise { + const { protection } = await guard(condition); + const node = protection.node({ screenResponses: true }); + const server = createServer((req: IncomingMessage, res: ServerResponse) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(HEADERS)) res.setHeader(name, value); + if (flush) res.flushHeaders(); + res.end(body); + }), + ); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + const { request } = await import('node:http'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}/`; + + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const got = res.headers; + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks), headers: seenHeaders((n) => got[n] as string | undefined) }); + }); + }).on('error', reject).end(); + }); +} + +const expectedHeaders = (masked: string[]) => + Object.fromEntries(Object.keys(HEADERS).map((name) => [name, masked.includes(name) ? '[REDACTED]' : SAMPLE])); + +describe.each([ + ['fetch', (c: Record, b: string | Buffer, t: string) => viaFetch(c, b, t)], + ['Node', (c: Record, b: string | Buffer, t: string) => viaNode(c, b, t, false)], +])('parameter forms on a screened body (%s)', (_path, run) => { + it.each(forms)('$label', async ({ condition, masked, readsBody }) => { + const body = `{"note":"${SAMPLE}"}`; + const got = await run(condition, body, 'application/json'); + + expect(got.status).toBe(200); + expect(got.body.toString()).toBe(readsBody ? '{"note":"[REDACTED]"}' : body); + expect(got.headers).toEqual(expectedHeaders(masked)); + }); + + it.each(forms.filter((f) => !f.readsBody))('$label leaves a JSON key alone', async ({ condition, masked }) => { + const body = `{"${SAMPLE}":1}`; + const got = await run(condition, body, 'application/json'); + + expect(got.status).toBe(200); + expect(got.body.toString()).toBe(body); + expect(got.headers).toEqual(expectedHeaders(masked)); + }); +}); + +describe.each([ + ['fetch, a body that is not screened', (c: Record) => viaFetch(c, BINARY, 'image/png')], + ['Node, a body that is not screened', (c: Record) => viaNode(c, BINARY, 'image/png', false)], + ['Node, an early flush', (c: Record) => viaNode(c, 'plain text', 'text/plain', true)], +])('header-only parameter forms (%s)', (_path, run) => { + it.each(forms.filter((f) => !f.readsBody))('$label', async ({ condition, masked }) => { + const got = await run(condition); + + expect(got.status).toBe(200); + expect(got.headers).toEqual(expectedHeaders(masked)); + }); +}); + +describe('a condition that names no place in the response', () => { + it.each([ + ['no parameter', undefined], + ['the status', 'response.status'], + ['a request source', 'get.q'], + ])('keeps the widest scope with %s', async (_label, parameter) => { + const extra = { ...(parameter === undefined ? {} : { parameter }), match: { type: 'regex', value: '/EXTRA-\\d+/' } }; + const detections: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + responseRules: [{ ...ruleOf(leaf('response.body')), rule_v2: [leaf('response.body'), extra] }], + onDetect: (event: any) => detections.push(event), + }); + const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}","more":"EXTRA-1"}`, { + headers: { 'content-type': 'application/json', 'x-other': 'EXTRA-1' }, + })); + + expect(JSON.parse(await out.text())).toEqual({ note: '[REDACTED]', more: '[REDACTED]' }); + expect(out.headers.get('x-other')).toBe('[REDACTED]'); + }); +}); + +describe('structural masking with a parameter list', () => { + it('masks a JSON path read through a one-item list', async () => { + const condition = { parameter: ['response.body'], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } }; + const got = await viaFetch(condition, `{"note":"${SAMPLE}","other":1}`, 'application/json'); + + expect(got.status).toBe(200); + expect(JSON.parse(got.body.toString())).toEqual({ note: '[REDACTED]', other: 1 }); + }); +}); From 02f7507bf486d6a2e397d7998551173454b05c44 Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 19:10:33 +0200 Subject: [PATCH 7/8] Report redactions whose parameter names nowhere to mask The status and request sources keep their broad mask, listed explicitly. A condition whose parameter the contract refuses, or that names no place in the response, now masks nothing and is reported through onError when the rules load; a rule left with nothing to mask withholds the response. Co-Authored-By: Claude Opus 5.5 --- src/protect/runtime.js | 71 ++++++++++------ .../redaction-scope-parameter-forms.test.ts | 81 +++++++++++++++++-- 2 files changed, 121 insertions(+), 31 deletions(-) diff --git a/src/protect/runtime.js b/src/protect/runtime.js index 8f59b683..5e612d3c 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -31,6 +31,7 @@ import { DEFAULT_RESPONSE_RULES, DEFAULT_EGRESS_RULES } from './defaults.js'; import { renderBlockPage } from './block-page.js'; // Rule lifecycle (source / tiered store / refresh) lives in ./rules/ — this file stays focused on // composing the engine + guards and running the three screening phases. +import { parameterProblem } from './rules/contract.js'; import { makeStore } from './rules/store.js'; import { resolveRules } from './rules/source.js'; import { startRefresh, startRecovery, makeRefreshHandler, serialise } from './rules/refresh.js'; @@ -398,7 +399,20 @@ export async function createProtection(options = {}) { // One engine per response rule so we can find ALL matches (to redact each). `action: // "redact"` masks the offending span(s); anything else withholds the whole response. responseRuleSet = responseRules.map((rule) => { - const redactors = rule.action === 'redact' || rule.action === 'encode' ? extractRedactors(rule) : null; + const unscoped = []; + const redactors = + rule.action === 'redact' || rule.action === 'encode' + ? extractRedactors(rule, (parameter) => unscoped.push(parameter)) + : null; + if (unscoped.length) { + notify( + onError, + new Error( + `Patchstack: response rule ${JSON.stringify(rule.id ?? null)} has a condition whose parameter names no place in the response to mask (${unscoped.map((p) => JSON.stringify(p ?? null)).join(', ')}); its matches are not masked`, + ), + 'onError', + ); + } // A redact/encode condition that carries body-transforming mutations (base64_decode, urldecode, // json_decode, …) detects on the DECODED body but the span redactors run on the RAW body — so // they mask nothing and the secret is served while the log says "redacted". Flag it so screenText @@ -2214,42 +2228,41 @@ function hasSpanMutations(rule) { /** * Where a span redactor masks, from the parameter its condition read: `{ body, allHeaders, headers }`, - * `headers` being a set of lower-cased names. + * `headers` being a set of lower-cased names — or null when the parameter names nowhere to mask. * * - `response.header.` masks that header; `response.headers` masks every header. Neither is the * body, and a rule that read no body does not rewrite it. * - `response.body` masks the body and the same text in every header, so a secret found in the body is not * left behind where it was echoed into a header. - * - A list of parameters — the engine reads each member — masks the union of its members' scopes. A - * member that is not a string resolves to nothing in the engine, and adds nothing here. - * - Anything else names no place in the response to mask: the status, a request source, a missing - * parameter, or one outside the contract. Such a condition keeps the widest scope, the body and every - * header, so its redaction still acts on the response rather than reporting a mask that changed nothing. + * - `response.status` and the request sources (`BROAD_SCOPE_SOURCES`) name no place in the response. A + * response rule reading them has always masked its match in the body and every header, and still does. + * - A list masks the union of its members' scopes. + * - Anything else is null: a shape the contract refuses (an empty or nested list, an unknown source or + * key, a value that is not a parameter), a valid parameter with no response location (`egress.*`), or no + * parameter at all. Such a condition masks nothing, and the rule that carries it is reported when the + * rules load. * * A group (`parameter: "rules"`) is not a scope of its own: the engine evaluates each of its conditions * with that condition's parameter, and each yields its own redactor with its own scope. */ function redactorScope(parameter) { + if (parameter === undefined || parameter === null || parameterProblem(parameter) !== null) return null; const scope = { body: false, allHeaders: false, headers: new Set() }; - const add = (name) => { - if (typeof name !== 'string') return; + for (const name of Array.isArray(parameter) ? parameter : [parameter]) { if (name.startsWith('response.header.')) scope.headers.add(name.slice('response.header.'.length).toLowerCase()); else if (name === 'response.headers') scope.allHeaders = true; - else { + else if (name === 'response.body' || name === 'response.status' || BROAD_SCOPE_SOURCES.has(name.split('.')[0])) { scope.body = true; scope.allHeaders = true; - } - }; - if (Array.isArray(parameter)) for (const member of parameter) add(member); - else if (typeof parameter === 'string') add(parameter); - else { - scope.body = true; - scope.allHeaders = true; + } else return null; } return scope; } +// Request sources, whose conditions in a response rule mask their match in the body and every header. +const BROAD_SCOPE_SOURCES = new Set(['get', 'post', 'request', 'cookie', 'files', 'server', 'raw', 'all']); + /** Does a span redactor apply to this header? `name` is lower-cased, as every screened header name is. */ function masksHeader(redactor, name) { return redactor.scope.allHeaders || redactor.scope.headers.has(name); @@ -2258,8 +2271,15 @@ function masksHeader(redactor, name) { // Derive redaction targets from a rule's own conditions: regex → mask every match; // contains/stripos → mask the literal. (Other match types can't identify a span → the // rule falls back to block.) -function extractRedactors(rule) { +function extractRedactors(rule, onUnscoped) { const out = []; + // A span redactor needs a place to mask. One whose parameter names none is left out and reported. + const scoped = (c) => { + const scope = redactorScope(c.parameter); + if (scope === null) onUnscoped?.(c.parameter); + + return scope; + }; const walk = (conds) => { for (const c of conds ?? []) { if (Array.isArray(c.rules)) walk(c.rules); @@ -2271,20 +2291,23 @@ function extractRedactors(rule) { const safe = safeRegExp(m.value); if (safe) { const flags = safe.flags.includes('g') ? safe.flags : safe.flags + 'g'; + const scope = scoped(c); try { - out.push({ re: new RegExp(safe.source, flags), scope: redactorScope(c.parameter) }); + if (scope) out.push({ re: new RegExp(safe.source, flags), scope }); } catch { /* skip invalid */ } } } else if ((m.type === 'contains' || m.type === 'stripos') && m.value != null) { - out.push({ literal: String(m.value), scope: redactorScope(c.parameter) }); + const scope = scoped(c); + if (scope) out.push({ literal: String(m.value), scope }); } else if (m.type === 'jwt_claim_equals' && typeof m.claim === 'string') { // A span-producing target, not a predicate. A boolean-only matcher would leave `redact` with // no span to mask, so the rule would fall back to withholding the WHOLE response — turning a // one-token leak into an outage. The spans come from the same `jwtClaimSpans` the matcher // used, so what is reported and what is masked cannot diverge. - out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') }, scope: redactorScope(c.parameter) }); + const scope = scoped(c); + if (scope) out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') }, scope }); } else if (m.type === 'array_key_value' && m.match && readsBody(c.parameter)) { // Structural redaction: mask the value at a JSON path (fanning out over arrays) rather than // a text span — e.g. key "orders.customers.email" masks that field in every array element. @@ -2458,9 +2481,11 @@ function applyRedactors(body, redactors, mask, transform) { // A response-body redaction target (array_key_value masks the JSON body). A bare condition with no // parameter also defaults to the body. -/** Does a condition read the body, alone or as a member of its parameter list? */ +/** Does a condition read the body, alone or as a member of a parameter list the contract accepts? */ function readsBody(parameter) { - return Array.isArray(parameter) ? parameter.some((member) => typeof member === 'string' && isBodyParam(member)) : isBodyParam(parameter); + if (!Array.isArray(parameter)) return isBodyParam(parameter); + + return parameterProblem(parameter) === null && parameter.some((member) => isBodyParam(member)); } function isBodyParam(parameter) { diff --git a/tests/protect/redaction-scope-parameter-forms.test.ts b/tests/protect/redaction-scope-parameter-forms.test.ts index ffb11dff..50b4ce51 100644 --- a/tests/protect/redaction-scope-parameter-forms.test.ts +++ b/tests/protect/redaction-scope-parameter-forms.test.ts @@ -24,7 +24,6 @@ const forms: Form[] = [ { label: 'two headers, as a list', condition: leaf(['response.header.x-sample', 'response.header.X-Extra']), masked: ['x-sample', 'x-extra'], readsBody: false }, { label: 'all headers, as a string', condition: leaf('response.headers'), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: false }, { label: 'all headers, as a list', condition: leaf(['response.headers']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: false }, - { label: 'a header beside a nested list, which reads nothing', condition: leaf(['response.header.x-sample', ['response.body']]), masked: ['x-sample'], readsBody: false }, { label: 'a header list in a group', condition: { parameter: 'rules', rules: [leaf(['response.header.x-sample'])] }, masked: ['x-sample'], readsBody: false }, { label: 'the body, as a one-item list', condition: leaf(['response.body']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: true }, { label: 'the body and a header, as a list', condition: leaf(['response.body', 'response.header.x-sample']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: true }, @@ -136,19 +135,16 @@ describe.each([ }); }); -describe('a condition that names no place in the response', () => { +describe('a legacy parameter that names no place in the response', () => { it.each([ - ['no parameter', undefined], ['the status', 'response.status'], ['a request source', 'get.q'], - ])('keeps the widest scope with %s', async (_label, parameter) => { - const extra = { ...(parameter === undefined ? {} : { parameter }), match: { type: 'regex', value: '/EXTRA-\\d+/' } }; - const detections: any[] = []; + ['a request source in a list', ['cookie.session', 'server.HTTP_HOST']], + ])('keeps its broad mask with %s', async (_label, parameter) => { const protection: any = await createProtection({ rules: emptyBundle, mode: 'block', - responseRules: [{ ...ruleOf(leaf('response.body')), rule_v2: [leaf('response.body'), extra] }], - onDetect: (event: any) => detections.push(event), + responseRules: [{ ...ruleOf(leaf('response.body')), rule_v2: [leaf('response.body'), { parameter, match: { type: 'regex', value: '/EXTRA-\\d+/' } }] }], }); const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}","more":"EXTRA-1"}`, { headers: { 'content-type': 'application/json', 'x-other': 'EXTRA-1' }, @@ -159,6 +155,75 @@ describe('a condition that names no place in the response', () => { }); }); +describe('a parameter shape with no place to mask', () => { + const unsupported: Array<[string, unknown]> = [ + ['a nested list', ['response.header.x-extra', ['response.body']]], + ['an empty list', []], + ['a value that is not a parameter', 42], + ['an unknown source', 'nope.value'], + ['an unknown response key', 'response.nope'], + ['an outbound source', 'egress.url'], + ['a keyed source without its key', 'get'], + ['a key the source does not answer for', 'server.NOT_A_KEY'], + ['no parameter', undefined], + ]; + + async function withUnsupported(parameter: unknown) { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ + ...ruleOf(leaf('response.header.x-sample')), + rule_v2: [leaf('response.header.x-sample'), { ...(parameter === undefined ? {} : { parameter }), match: PATTERN }], + }], + }); + + return { protection, errors }; + } + + it.each(unsupported)('does not widen the mask for %s, and reports it', async (_label, parameter) => { + const { protection, errors } = await withUnsupported(parameter); + const body = `{"note":"${SAMPLE}"}`; + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json', ...HEADERS } })); + + expect(out.status).toBe(200); + expect(await out.text()).toBe(body); + expect(seenHeaders((n) => out.headers.get(n))).toEqual(expectedHeaders(['x-sample'])); + expect(errors.map((e) => e.message)).toEqual([expect.stringContaining('names no place in the response to mask')]); + }); + + it('withholds a response whose only condition cannot be masked, rather than report a mask it did not make', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [ruleOf(leaf(['response.header.x-sample', ['response.body']]))], + }); + const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}"}`, { headers: { 'content-type': 'application/json', ...HEADERS } })); + + expect(out.status).toBe(500); + expect(errors).toHaveLength(1); + }); + + it('does not mask a JSON path read through a refused list', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [ruleOf({ parameter: ['response.body', ['response.body']], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } })], + }); + const body = `{"note":"${SAMPLE}","other":1}`; + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json' } })); + + expect(out.status).toBe(500); + expect(await out.text()).not.toContain(SAMPLE); + }); +}); + describe('structural masking with a parameter list', () => { it('masks a JSON path read through a one-item list', async () => { const condition = { parameter: ['response.body'], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } }; From af79e62c6566e18029f0eac2ee3a95bac4f5bc0c Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 19:37:37 +0200 Subject: [PATCH 8/8] Load response rules whose parameters cannot be read A parameter list member that is not a non-empty string, or a condition that is not an object, no longer makes rule loading throw. Such a condition masks nothing and is reported through onError, on the first load and on a refresh, and any error while reading a rule's redactions is reported without failing the load. Co-Authored-By: Claude Opus 5.5 --- src/protect/runtime.js | 66 +++-- ...daction-scope-malformed-parameters.test.ts | 227 ++++++++++++++++++ 2 files changed, 269 insertions(+), 24 deletions(-) create mode 100644 tests/protect/redaction-scope-malformed-parameters.test.ts diff --git a/src/protect/runtime.js b/src/protect/runtime.js index 5e612d3c..85ab2d39 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -400,10 +400,35 @@ export async function createProtection(options = {}) { // "redact"` masks the offending span(s); anything else withholds the whole response. responseRuleSet = responseRules.map((rule) => { const unscoped = []; - const redactors = - rule.action === 'redact' || rule.action === 'encode' - ? extractRedactors(rule, (parameter) => unscoped.push(parameter)) - : null; + const rewrites = rule.action === 'redact' || rule.action === 'encode'; + // What the rule masks, and whether it is decided on headers alone. A rule this cannot read is loaded + // and reported rather than failing the whole load: it masks nothing, is not treated as header-only, + // and — like any rule with nothing to mask — withholds a response it matches. + let derived; + try { + const redactors = rewrites ? extractRedactors(rule, (parameter) => unscoped.push(parameter)) : null; + // A redact/encode condition that carries body-transforming mutations (base64_decode, urldecode, + // json_decode, …) detects on the DECODED body but the span redactors run on the RAW body — so + // they mask nothing and the secret is served while the log says "redacted". Flag it so screenText + // fails such a rule CLOSED (block) instead of serving a no-op redaction. + const mutatedSpan = rewrites && hasSpanMutations(rule); + derived = { + redactors, + mutatedSpan, + // A redaction that reads response headers only, and masks spans in them, and an explicit block + // that reads response headers only, are decided and carried out without the body — so they + // still apply to a response whose body was not screened. + redactsHeaders: + rule.action === 'redact' && + !mutatedSpan && + (redactors ?? []).some((r) => !r.jsonPath) && + readsOnlyResponseHeaders(rule), + blocksOnHeaders: rule.action === 'block' && readsOnlyResponseHeaders(rule), + }; + } catch (err) { + notify(onError, err, 'onError'); + derived = { redactors: null, mutatedSpan: false, redactsHeaders: false, blocksOnHeaders: false }; + } if (unscoped.length) { notify( onError, @@ -413,26 +438,10 @@ export async function createProtection(options = {}) { 'onError', ); } - // A redact/encode condition that carries body-transforming mutations (base64_decode, urldecode, - // json_decode, …) detects on the DECODED body but the span redactors run on the RAW body — so - // they mask nothing and the secret is served while the log says "redacted". Flag it so screenText - // fails such a rule CLOSED (block) instead of serving a no-op redaction. - const mutatedSpan = (rule.action === 'redact' || rule.action === 'encode') && hasSpanMutations(rule); - return { rule, engine: new RuleEngine({ firewall: [rule], onError }), - redactors, - mutatedSpan, - // A redaction that reads response headers only, and masks spans in them, and an explicit block - // that reads response headers only, are decided and carried out without the body — so they still - // apply to a response whose body was not screened. - redactsHeaders: - rule.action === 'redact' && - !mutatedSpan && - (redactors ?? []).some((r) => !r.jsonPath) && - readsOnlyResponseHeaders(rule), - blocksOnHeaders: rule.action === 'block' && readsOnlyResponseHeaders(rule), + ...derived, // Optional cheap pre-filter: literal anchor(s) that MUST appear for the (expensive) regex to // have any chance of matching. Lets screenText skip the full scan on responses with no candidate // — the common case — cutting CPU/latency and shrinking the regex/ReDoS surface. @@ -2215,8 +2224,9 @@ function headerObject(headers) { function hasSpanMutations(rule) { let found = false; const walk = (conds) => { - for (const c of conds ?? []) { + for (const c of Array.isArray(conds) ? conds : []) { if (found) return; + if (!c || typeof c !== 'object') continue; if (Array.isArray(c.rules)) walk(c.rules); const isSpan = c.match && (c.match.type === 'regex' || c.match.type === 'contains' || c.match.type === 'stripos'); if (isSpan && Array.isArray(c.mutations) && c.mutations.length) found = true; @@ -2249,6 +2259,9 @@ function redactorScope(parameter) { if (parameter === undefined || parameter === null || parameterProblem(parameter) !== null) return null; const scope = { body: false, allHeaders: false, headers: new Set() }; for (const name of Array.isArray(parameter) ? parameter : [parameter]) { + // Checked here rather than left to the contract, which answers a different question: a member that is + // not a parameter name has no place to mask, whatever the validator makes of it. + if (typeof name !== 'string' || name === '') return null; if (name.startsWith('response.header.')) scope.headers.add(name.slice('response.header.'.length).toLowerCase()); else if (name === 'response.headers') scope.allHeaders = true; else if (name === 'response.body' || name === 'response.status' || BROAD_SCOPE_SOURCES.has(name.split('.')[0])) { @@ -2281,7 +2294,8 @@ function extractRedactors(rule, onUnscoped) { return scope; }; const walk = (conds) => { - for (const c of conds ?? []) { + for (const c of Array.isArray(conds) ? conds : []) { + if (!c || typeof c !== 'object') continue; if (Array.isArray(c.rules)) walk(c.rules); const m = c.match; if (!m) continue; @@ -2308,6 +2322,8 @@ function extractRedactors(rule, onUnscoped) { // used, so what is reported and what is masked cannot diverge. const scope = scoped(c); if (scope) out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') }, scope }); + } else if (m.type === 'array_key_value' && m.match && Array.isArray(c.parameter) && !readsBody(c.parameter) && redactorScope(c.parameter) === null) { + onUnscoped?.(c.parameter); } else if (m.type === 'array_key_value' && m.match && readsBody(c.parameter)) { // Structural redaction: mask the value at a JSON path (fanning out over arrays) rather than // a text span — e.g. key "orders.customers.email" masks that field in every array element. @@ -2485,7 +2501,9 @@ function applyRedactors(body, redactors, mask, transform) { function readsBody(parameter) { if (!Array.isArray(parameter)) return isBodyParam(parameter); - return parameterProblem(parameter) === null && parameter.some((member) => isBodyParam(member)); + const names = parameter.every((member) => typeof member === 'string' && member !== ''); + + return names && parameterProblem(parameter) === null && parameter.some((member) => isBodyParam(member)); } function isBodyParam(parameter) { diff --git a/tests/protect/redaction-scope-malformed-parameters.test.ts b/tests/protect/redaction-scope-malformed-parameters.test.ts new file mode 100644 index 00000000..9f68bae1 --- /dev/null +++ b/tests/protect/redaction-scope-malformed-parameters.test.ts @@ -0,0 +1,227 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A response rule whose parameter list holds something other than a parameter name is loaded, not + * rejected: the condition masks nothing, the rule is reported through `onError`, and a response the rule + * matches follows the fallback for a match with nothing to mask. That holds for the first load and for a + * rule arriving on a refresh, which must not throw either. + */ + +afterEach(() => vi.restoreAllMocks()); + +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const PATTERN = { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' }; +const EMPTY = { firewall: [], whitelists: [], whitelist_keys: {} }; +const URL_OPT = 'https://x.test/monitor/pulse'; + +// `matches`: whether the engine can match the rule at all. A list whose readable member reads the header +// matches, and with nothing it can mask, withholds; one with no readable member never matches; and a +// member the engine cannot resolve fails the rule's evaluation, which the engine treats as no match. +const malformed: Array<[string, unknown[], boolean]> = [ + ['[null]', [null], false], + ['[undefined]', [undefined], false], + ['a header list with null', ['response.header.x-sample', null], true], + ['a body list with null', ['response.body', null], true], + ['a list with a number', ['response.header.x-sample', 42], false], + ['a list with an object', ['response.header.x-sample', { name: 'response.body' }], false], + ['a list with an empty string', ['response.header.x-sample', ''], true], +]; + +const SCOPE_REPORT = 'names no place in the response to mask'; +const scopeReports = (errors: Error[]) => errors.filter((e) => e.message.includes(SCOPE_REPORT)); + +/** Withheld, or sent without anything masked beyond what is known: never widened. */ +async function expectFallback(out: Response, matches: boolean) { + expect(out.status).toBe(matches ? 500 : 200); + if (!matches) { + expect(out.headers.get('x-sample')).toBe(SAMPLE); + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(await out.text()).toBe(`{"note":"${SAMPLE}"}`); + } +} + +const badRule = (parameter: unknown) => ({ + id: 'malformed', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [{ parameter, match: PATTERN }], +}); + +const goodRule = { + id: 'well-formed', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [{ parameter: 'response.header.x-good', match: PATTERN }], +}; + +const response = () => + new Response(`{"note":"${SAMPLE}"}`, { + headers: { 'content-type': 'application/json', 'x-sample': SAMPLE, 'x-good': SAMPLE, 'x-other': SAMPLE }, + }); + +describe('a malformed parameter list on the first load', () => { + it.each(malformed)('loads with %s, reports it, and follows the fallback', async (_label, parameter, matches) => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [badRule(parameter)], + }); + + expect(scopeReports(errors)).toHaveLength(1); + await expectFallback(await protection.screenResponse(response()), matches); + }); + + it('keeps a well-formed rule beside it working', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [badRule([null]), goodRule], + }); + const out = await protection.screenResponse(response()); + + expect(out.status).toBe(200); + expect(out.headers.get('x-good')).toBe('[REDACTED]'); + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(errors).toHaveLength(1); + }); + + it.each([ + ['a null condition', [null, { parameter: 'response.header.x-good', match: PATTERN }]], + ['a condition that is not an object', ['text', { parameter: 'response.header.x-good', match: PATTERN }]], + ])('loads a rule with %s without widening a mask', async (_label, rule_v2) => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ ...goodRule, rule_v2 }], + }); + const out = await protection.screenResponse(response()); + + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(out.headers.get('x-sample')).toBe(SAMPLE); + // Loading reads past the condition it cannot use. What is reported comes from evaluating the rule, + // once per evaluation, and not from loading it. + const fromLoading = errors.filter((e) => /reading 'rules'|reading 'match'|reading 'mutations'/.test(e.message)); + expect(fromLoading).toEqual([]); + }); + + it.each([['match'], ['mutations']])('loads a rule whose condition cannot be read (%s) and reports it', async (property) => { + const errors: Error[] = []; + const condition: Record = { parameter: 'response.header.x-good', match: PATTERN }; + Object.defineProperty(condition, property, { get() { throw new Error('unreadable condition'); }, enumerable: true }); + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ ...goodRule, rule_v2: [condition] }], + }); + // Reported by loading, before any response is screened. + expect(errors.some((e) => e.message === 'unreadable condition')).toBe(true); + + const out = await protection.screenResponse(response()); + expect(out.headers.get('x-other')).toBe(SAMPLE); + }); + + it('reports a structural condition over a list with null, and withholds a response it matches', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ + ...goodRule, + rule_v2: [{ parameter: ['response.body', null], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } }], + }], + }); + const out = await protection.screenResponse(response()); + + expect(scopeReports(errors)).toHaveLength(1); + expect(out.status).toBe(500); + }); +}); + +describe('a malformed parameter list arriving on a refresh', () => { + // The rule source validates a delivered bundle against the rule contract, and an update carrying a rule + // it refuses is rejected whole: the previous rules stay in force. A `null` member passes that check, so + // it is applied, and the scope handling reports it. + const refused = new Set(['a list with a number', 'a list with an object', 'a list with an empty string']); + const previous = { ...goodRule, id: 'previous', rule_v2: [{ parameter: 'response.header.x-other', match: PATTERN }] }; + + it.each(malformed.filter(([label]) => label !== '[undefined]'))('refreshes with %s without throwing', async (label, parameter, matches) => { + const bundle = { firewall: [badRule(parameter), goodRule], whitelists: [], whitelist_keys: {} }; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ ...EMPTY, firewall: [previous] }), { status: 200 })) + .mockResolvedValue(new Response(JSON.stringify(bundle), { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const errors: Error[] = []; + const protection: any = await createProtection({ + siteUuid: 's', + pulseRulesUrl: URL_OPT, + mode: 'block', + reportManifest: false, + onError: (error: Error) => errors.push(error), + }); + expect(scopeReports(errors)).toHaveLength(0); + + const outcome = await protection.refresh(); + const out = await protection.screenResponse(response()); + + if (refused.has(label)) { + expect(outcome).toMatchObject({ ok: false, reason: 'update rejected' }); + expect(errors.some((e) => e.message.includes('rejected the entire update'))).toBe(true); + expect(scopeReports(errors)).toHaveLength(0); + // The previous rules are still the ones in force. + expect(out.status).toBe(200); + expect(out.headers.get('x-other')).toBe('[REDACTED]'); + expect(out.headers.get('x-good')).toBe(SAMPLE); + + return; + } + + expect(outcome).toMatchObject({ ok: true }); + expect(scopeReports(errors)).toHaveLength(1); + expect(out.status).toBe(matches ? 500 : 200); + if (!matches) { + // The well-formed rule the same refresh delivered is in force beside it, and the previous one is not. + expect(out.headers.get('x-good')).toBe('[REDACTED]'); + expect(out.headers.get('x-other')).toBe(SAMPLE); + } + }); +}); + +describe('a malformed parameter list arriving through the push endpoint', () => { + it('refreshes and reports it', async () => { + const bundle = { firewall: [badRule(['response.header.x-sample', null]), goodRule], whitelists: [], whitelist_keys: {} }; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(JSON.stringify(EMPTY), { status: 200 })) + .mockResolvedValue(new Response(JSON.stringify(bundle), { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const errors: Error[] = []; + const protection: any = await createProtection({ + siteUuid: 's', + pulseRulesUrl: URL_OPT, + mode: 'block', + reportManifest: false, + refreshSecret: 'sample-secret', + onError: (error: Error) => errors.push(error), + }); + + const handled = await protection.refreshHandler()( + new Request('https://app.example.test/_ps/refresh', { headers: { 'x-patchstack-refresh': 'sample-secret' } }), + ); + + expect(handled.status).toBe(200); + expect(scopeReports(errors)).toHaveLength(1); + expect((await protection.screenResponse(response())).status).toBe(500); + }); +});