diff --git a/src/protect/protect.d.ts b/src/protect/protect.d.ts index f58d14e7..aeb0e247 100644 --- a/src/protect/protect.d.ts +++ b/src/protect/protect.d.ts @@ -29,6 +29,12 @@ export interface Protection { * * The client address is the one resolved when this guard screened that request, if it did; otherwise it * is resolved here, and any further arguments are passed to `peerAddress` as the host's handler arguments. + * + * A rule that reads only response headers (`response.header.*`, `response.headers`) redacts or blocks on + * the headers alone, so it is enforced even when the body cannot be screened. A redaction masks only the + * header value it matched; protecting the body takes a rule on `response.body`, which masks the body and + * the same text wherever it appears in a header. A withheld response carries only its own `content-type` + * and `content-length`. */ screenResponse(response: Response, request?: Request, ...hostArgs: unknown[]): Promise; express(options?: { screenResponses?: boolean }): (req: unknown, res: unknown, next: () => void) => void; @@ -344,7 +350,10 @@ export interface CreateProtectionOptions { header?: string; isTrusted?: (ip: string) => boolean; }; - /** Override the default response-phase (secret-leak) rule set. */ + /** + * Override the default response-phase (secret-leak) rule set. A rule that reads only response headers is + * enforced from the headers, and masks only the header it matched — see `screenResponse`. + */ responseRules?: unknown[]; /** Override the default egress-phase (SSRF) rule set. */ egressRules?: unknown[]; diff --git a/src/protect/response-hardening.js b/src/protect/response-hardening.js index 1d34aa0b..2c8a550e 100644 --- a/src/protect/response-hardening.js +++ b/src/protect/response-hardening.js @@ -45,7 +45,30 @@ export function hardensWithoutBody(rule) { // A parameterless match — `cross_origin`, `off_origin`, `cors_reflected` — reads the request's origin // and the response's own headers, so it carries no parameter and needs no body. - return parameters.every( - (parameter) => BODY_INDEPENDENT.includes(parameter) || parameter.startsWith(BODY_INDEPENDENT_PREFIX), - ); + return parameters.every(bodyIndependent); +} + +/** + * Does this rule read response headers and nothing else? + * + * The decision half of `hardensWithoutBody`, for a rule whose action is not a header action — a + * `redact` keyed on `response.header.*` masks the header value it matched, which needs no body either. + * A header must be among what it reads: a rule keyed on the status alone is aimed at the body, and + * masking header values on its behalf would carry out a different rule from the one written. + */ +export function readsOnlyResponseHeaders(rule) { + if (!rule) return false; + + const { parameters, complete } = readRuleParameters(rule); + if (!complete || !parameters.some(isHeaderParameter)) return false; + + return parameters.every(bodyIndependent); +} + +function isHeaderParameter(parameter) { + return parameter === 'response.headers' || parameter.startsWith(BODY_INDEPENDENT_PREFIX); +} + +function bodyIndependent(parameter) { + return BODY_INDEPENDENT.includes(parameter) || parameter.startsWith(BODY_INDEPENDENT_PREFIX); } diff --git a/src/protect/runtime.js b/src/protect/runtime.js index e05e772a..85ab2d39 100644 --- a/src/protect/runtime.js +++ b/src/protect/runtime.js @@ -31,12 +31,13 @@ import { DEFAULT_RESPONSE_RULES, DEFAULT_EGRESS_RULES } from './defaults.js'; import { renderBlockPage } from './block-page.js'; // Rule lifecycle (source / tiered store / refresh) lives in ./rules/ — this file stays focused on // composing the engine + guards and running the three screening phases. +import { parameterProblem } from './rules/contract.js'; import { makeStore } from './rules/store.js'; import { resolveRules } from './rules/source.js'; import { startRefresh, startRecovery, makeRefreshHandler, serialise } from './rules/refresh.js'; import { createDetectionReporter } from './detections.js'; import { reportingState } from './reporting-state.js'; -import { hardensWithoutBody } from './response-hardening.js'; +import { hardensWithoutBody, readsOnlyResponseHeaders } from './response-hardening.js'; import { notify } from './notify.js'; import { SOURCE_REQUEST } from './supabase-guard.js'; import { createFirewallLogReporter, resolveApiBase, telemetryEnabled } from './firewall-log.js'; @@ -397,22 +398,60 @@ export async function createProtection(options = {}) { }); // One engine per response rule so we can find ALL matches (to redact each). `action: // "redact"` masks the offending span(s); anything else withholds the whole response. - responseRuleSet = responseRules.map((rule) => ({ - rule, - engine: new RuleEngine({ firewall: [rule], onError }), - redactors: rule.action === 'redact' || rule.action === 'encode' ? extractRedactors(rule) : null, - // A redact/encode condition that carries body-transforming mutations (base64_decode, urldecode, - // json_decode, …) detects on the DECODED body but the span redactors run on the RAW body — so - // they mask nothing and the secret is served while the log says "redacted". Flag it so screenText - // fails such a rule CLOSED (block) instead of serving a no-op redaction. - mutatedSpan: (rule.action === 'redact' || rule.action === 'encode') && hasSpanMutations(rule), - // Optional cheap pre-filter: literal anchor(s) that MUST appear for the (expensive) regex to - // have any chance of matching. Lets screenText skip the full scan on bodies with no candidate — - // the common case — cutting CPU/latency and shrinking the regex/ReDoS surface. Case-insensitive. - prefilter: Array.isArray(rule.prefilter) && rule.prefilter.length - ? rule.prefilter.map((s) => String(s).toLowerCase()) - : null, - })); + responseRuleSet = responseRules.map((rule) => { + const unscoped = []; + const rewrites = rule.action === 'redact' || rule.action === 'encode'; + // What the rule masks, and whether it is decided on headers alone. A rule this cannot read is loaded + // and reported rather than failing the whole load: it masks nothing, is not treated as header-only, + // and — like any rule with nothing to mask — withholds a response it matches. + let derived; + try { + const redactors = rewrites ? extractRedactors(rule, (parameter) => unscoped.push(parameter)) : null; + // A redact/encode condition that carries body-transforming mutations (base64_decode, urldecode, + // json_decode, …) detects on the DECODED body but the span redactors run on the RAW body — so + // they mask nothing and the secret is served while the log says "redacted". Flag it so screenText + // fails such a rule CLOSED (block) instead of serving a no-op redaction. + const mutatedSpan = rewrites && hasSpanMutations(rule); + derived = { + redactors, + mutatedSpan, + // A redaction that reads response headers only, and masks spans in them, and an explicit block + // that reads response headers only, are decided and carried out without the body — so they + // still apply to a response whose body was not screened. + redactsHeaders: + rule.action === 'redact' && + !mutatedSpan && + (redactors ?? []).some((r) => !r.jsonPath) && + readsOnlyResponseHeaders(rule), + blocksOnHeaders: rule.action === 'block' && readsOnlyResponseHeaders(rule), + }; + } catch (err) { + notify(onError, err, 'onError'); + derived = { redactors: null, mutatedSpan: false, redactsHeaders: false, blocksOnHeaders: false }; + } + if (unscoped.length) { + notify( + onError, + new Error( + `Patchstack: response rule ${JSON.stringify(rule.id ?? null)} has a condition whose parameter names no place in the response to mask (${unscoped.map((p) => JSON.stringify(p ?? null)).join(', ')}); its matches are not masked`, + ), + 'onError', + ); + } + return { + rule, + engine: new RuleEngine({ firewall: [rule], onError }), + ...derived, + // Optional cheap pre-filter: literal anchor(s) that MUST appear for the (expensive) regex to + // have any chance of matching. Lets screenText skip the full scan on responses with no candidate + // — the common case — cutting CPU/latency and shrinking the regex/ReDoS surface. + // Case-insensitive, and checked against the header values as well as the body, since a rule may + // read either. + prefilter: Array.isArray(rule.prefilter) && rule.prefilter.length + ? rule.prefilter.map((s) => String(s).toLowerCase()) + : null, + }; + }); // One engine per rule, as the response phase does. It preserves the identity of every rule that // matches: each is evaluated on its own, and each match is attributable to the rule that made it. egressRuleSet = egressRules.map((rule) => ({ @@ -532,6 +571,12 @@ export async function createProtection(options = {}) { return effectiveMode === 'block' ? block() : allow(); }; + // The rules a response whose body was not read can still be screened against: header hardening, + // redactions of header values, and blocks decided on headers. None reads the body, and none needs one + // to act on. + const decidedByHeaders = (_rule, entry) => Boolean(entry?.redactsHeaders || entry?.blocksOnHeaders); + const decidedWithoutBody = (rule, entry) => hardensWithoutBody(rule) || decidedByHeaders(rule, entry); + // Response phase core: screen a text body → { verdict: 'pass'|'block'|'redact', body? }. // redact masks matched spans; block withholds; block wins over redact. Enforcement only in // block mode (dry-run records via onDetect but returns 'pass'). @@ -539,18 +584,19 @@ export async function createProtection(options = {}) { let blockRule = null; const redactions = []; const headerMutations = []; - let lowerText = null; // lazily lowercased body, only if a rule uses a prefilter + let lowerText = null; // lazily lowercased body and header values, only if a rule uses a prefilter const responseSkips = new Set(); // each inspection limit counted once per response - for (const { rule, engine: re, redactors, prefilter, mutatedSpan } of responseRuleSet) { + for (const entry of responseRuleSet) { + const { rule, engine: re, redactors, prefilter, mutatedSpan } = entry; // `only` narrows the set to the rules a caller is entitled to run. The no-body path uses it to // exclude every rule that reads the body, rather than evaluating one against an empty string — // `not_contains` matches everything when there is nothing there, so that is not an undecided rule // but a wrongly decided one. - if (only && !only(rule)) continue; - // Cheap pre-filter: if none of the rule's literal anchors is in the body, its regex can't + if (only && !only(rule, entry)) continue; + // Cheap pre-filter: if none of the rule's literal anchors is in the response, its regex can't // match — skip the full scan (the common no-secret case) before touching the engine. if (prefilter) { - if (lowerText === null) lowerText = text.toLowerCase(); + if (lowerText === null) lowerText = prefilterText(text, meta.headers); if (!prefilter.some((p) => lowerText.includes(p))) continue; } let result; @@ -603,9 +649,9 @@ export async function createProtection(options = {}) { if (mode !== 'block' || (!blockRule && !redactions.length && !headerMutations.length)) return { verdict: 'pass' }; if (blockRule) return { verdict: 'block' }; let body = text; - // Redact the offending spans in the body AND in every (string) header value — so a secret - // that leaks in a header (Set-Cookie, an echoed X-Api-Key, …) is masked too, and a rule that - // targets `response.header.*` actually strips the header rather than just detecting it. + // Each span redactor masks where its condition read (`redactorScope`): a body condition the body and + // every (string) header value — so a secret that leaks in a header (Set-Cookie, an echoed X-Api-Key, + // …) is masked too — and a header condition only the header it read. const headers = { ...(meta.headers || {}) }; // Whether the response is a JSON document, asked only once a span rewrite needs it, and the lexed // structure of the body as it stands — reused by the next span rewrite while the body is unchanged. @@ -615,14 +661,17 @@ export async function createProtection(options = {}) { const mask = maskFn(rule.category); // action `encode` HTML-escapes the matched value in place (neutralize stored XSS at output); // `redact` masks it. jsonPath redactors act on a structural JSON location, span redactors on - // text spans in the body AND header values. Apply structural first (on clean JSON), then spans. + // text spans in their scope. Apply structural first (on clean JSON), then spans. const transform = rule.action === 'encode' ? htmlEscape : null; const pathRedactors = redactors.filter((r) => r.jsonPath); const spanRedactors = redactors.filter((r) => !r.jsonPath); if (pathRedactors.length) body = applyPathRedactors(body, pathRedactors, mask, screenCap, transform); if (!spanRedactors.length) continue; const beforeSpan = body; - body = applyRedactors(body, spanRedactors, mask, transform); + // A redactor masks the body only when its condition read the body; one that read only a header + // leaves the body unchanged, and so never reaches the structure check below. + const bodySpans = spanRedactors.filter((r) => r.scope.body === true); + if (bodySpans.length) body = applyRedactors(body, bodySpans, mask, transform); // Span rewrites may change JSON string values, but not keys, containers or other values, and a // rewrite that would produce an invalid document is withheld rather than sent. Each result is // checked before it becomes input to another transformation. A response that was a JSON document @@ -640,14 +689,15 @@ export async function createProtection(options = {}) { if (transform) continue; // encoding is a body/output concern — headers aren't HTML for (const name of Object.keys(headers)) { const value = headers[name]; + const headerSpans = spanRedactors.filter((r) => masksHeader(r, name)); if (typeof value === 'string') { - setOwn(headers, name, applyRedactors(value, spanRedactors, mask)); + setOwn(headers, name, applyRedactors(value, headerSpans, mask)); } else if (Array.isArray(value)) { // Multi-valued headers (Set-Cookie) — redact each entry. setOwn( headers, name, - value.map((item) => (typeof item === 'string' ? applyRedactors(item, spanRedactors, mask) : item)), + value.map((item) => (typeof item === 'string' ? applyRedactors(item, headerSpans, mask) : item)), ); } } @@ -979,8 +1029,8 @@ export async function createProtection(options = {}) { const hardenHeadersOnly = (response, reqCtx) => { try { const meta = { status: response.status, headers: headerObject(response.headers) }; - const r = screenText('', meta, reqCtx, hardensWithoutBody); - // `block` cannot arise: only header actions were eligible. + const r = screenText('', meta, reqCtx, decidedWithoutBody); + if (r.verdict === 'block') return leakResponse(); if (r.verdict !== 'redact' || !r.headers) return response; // A matched rule is not a changed header. `harden-cookie` on a response that sets no cookie, @@ -1019,6 +1069,9 @@ export async function createProtection(options = {}) { const chunks = []; let size = 0; let overflow = false; + let withheld = false; + // Whether the rules decided on headers alone have already been run for this response. + let headScreened = false; const MAX = screenCap; /** @@ -1086,6 +1139,10 @@ export async function createProtection(options = {}) { // The rules the pass above did not answer: the ones that read the body, which is why a // body-reading hardening rule is still honoured at `end`. const stillToAnswer = (rule) => !hardensWithoutBody(rule); + // What is left for the screen at `end`: neither the hardening answered before the head, nor the rules + // decided on headers alone once they have run. + const notYetAnswered = (rule, entry) => + (!answeredWithoutBody || stillToAnswer(rule)) && !(headScreened && decidedByHeaders(rule, entry)); /** * `writeHead`, held until the first byte actually leaves. @@ -1193,12 +1250,19 @@ export async function createProtection(options = {}) { return original(...args); }; } - // An explicit flush asks for the head now, so it goes now, hardened. The body that follows can then - // only change where no length was promised, which `end` already accounts for. + // An explicit flush asks for the head now, so it goes now — hardened, and screened by the rules + // decided on headers alone, which is the last point at which they can still change it. The body that + // follows can then only change where no length was promised, which `end` already accounts for. if (typeof res.flushHeaders === 'function') { const origFlushHeaders = res.flushHeaders.bind(res); res.flushHeaders = function () { - sendHead(); + const unread = screenUnreadHead(); + if (unread?.withhold) { + sendWithheld(); + + return undefined; + } + sendHead(unread); return origFlushHeaders(); }; @@ -1247,6 +1311,47 @@ export async function createProtection(options = {}) { }; }; + /** + * The rules decided on headers alone — redactions of header values, and blocks — for a response whose + * body will not be screened. + * + * Such a rule is decided the same way whether or not the body is read, so it runs once per response — + * at an explicit flush, or on a branch that does not screen the body — and is left out of the screen at + * `end`. Returns `{ withhold: true }` for a block, the header changes for `sendHead`, or undefined when + * there are none. A head that had already gone can take neither, which is recorded, as it is for a + * screened body. + */ + const screenUnreadHead = () => { + if (headScreened) return undefined; + headScreened = true; + try { + const head = effectiveHead(); + const r = screenText('', head, reqCtx, decidedByHeaders); + if (r.verdict === 'block') { + if (!reallySent()) return { withhold: true }; + recordSkip('response', 'headers-sent', { action: 'block' }); + + return undefined; + } + if (r.verdict !== 'redact' || !r.headers) return undefined; + const changed = new Map(); + for (const [name, value] of Object.entries(r.headers)) { + if (headerValueChanged(head.headers[name], value)) changed.set(name.toLowerCase(), value); + } + if (changed.size && reallySent()) { + recordSkip('response', 'headers-sent', { headers: [...changed.keys()] }); + + return undefined; + } + + return changed.size ? { headers: changed } : undefined; + } catch (err) { + notify(onError, err, 'onError'); + + return undefined; + } + }; + /** * Send the head, if the application asked for one explicitly. `changes` is what the screen at `end` * decided: a new status, and header values keyed by lower-cased name (null removes one). Applied to @@ -1279,6 +1384,26 @@ export async function createProtection(options = {}) { origWriteHead(...args); }; + /** + * Send the generic withheld response in place of the application's, before its head has gone. + * + * Only its own framing goes out. Every header the application set is dropped: they described a + * response that is not the one sent, and a header can be what the verdict was about. Anything the + * application writes afterwards is discarded. + */ + const sendWithheld = (cb) => { + const body = JSON.stringify({ error: 'Response withheld by Patchstack (sensitive data detected)' }); + const headers = new Map(); + for (const name of Object.keys(effectiveHead().headers)) headers.set(name.toLowerCase(), null); + headers.set('content-type', 'application/json'); + headers.set('content-length', String(Buffer.byteLength(body))); + headers.set('transfer-encoding', null); + withheld = true; + sendHead({ status: 500, headers }); + + return origEnd(body, cb); + }; + /** Header changes onto the response's own header state, for the head Node writes implicitly. */ const applyToResponse = (changes) => { if (changes.status !== undefined) res.statusCode = changes.status; @@ -1300,7 +1425,13 @@ export async function createProtection(options = {}) { // Too big to screen — abandon buffering, but FLUSH what we already captured (the head) plus // this chunk before switching to pass-through, so the client gets a complete body (not a // truncated one missing everything before the cap was hit). - sendHead(); + const unread = screenUnreadHead(); + if (unread?.withhold) { + sendWithheld(); + + return; + } + sendHead(unread); for (const c of chunks) origWrite(c); chunks.length = 0; origWrite(buf); @@ -1311,6 +1442,12 @@ export async function createProtection(options = {}) { chunks.push(buf); }; res.write = function (chunk, enc, cb) { + if (withheld) { + if (typeof enc === 'function') enc(); + else if (typeof cb === 'function') cb(); + + return true; + } if (overflow) return origWrite(chunk, enc, cb); collect(chunk, enc); if (typeof enc === 'function') enc(); @@ -1320,13 +1457,24 @@ export async function createProtection(options = {}) { res.end = function (chunk, enc, cb) { if (typeof chunk === 'function') { cb = chunk; chunk = undefined; enc = undefined; } else if (typeof enc === 'function') { cb = enc; enc = undefined; } + if (withheld) { + if (typeof cb === 'function') cb(); + + return res; + } if (overflow) { if (chunk != null) origWrite(chunk, enc); return origEnd(cb); } collect(chunk, enc); + if (withheld) { + if (typeof cb === 'function') cb(); + + return res; + } if (overflow) return origEnd(cb); // collect just flushed head + final chunk on overflow hardenOnce(); - const passThrough = () => { - sendHead(); + const passThrough = (changes) => { + if (changes?.withhold) return sendWithheld(cb); + sendHead(changes); for (const c of chunks) origWrite(c); return origEnd(cb); @@ -1341,7 +1489,7 @@ export async function createProtection(options = {}) { if (kind === 'skip' || (kind === 'sniff' && looksBinary(buffer))) { recordSkip('response', kind === 'skip' ? (baseContentType(ct) === 'text/event-stream' ? 'live-stream' : 'non-text-content-type') : 'binary-body'); - return passThrough(); + return passThrough(screenUnreadHead()); } // Encoded bytes cannot be screened as text: sent exactly as they are, under their own coding. The // coding is read from the head that will be sent, which includes a held `writeHead`'s own headers. @@ -1349,12 +1497,12 @@ export async function createProtection(options = {}) { if (codings.length > 0 && stillEncoded(buffer)) { recordSkip('response', 'encoded-body', { encoding: codings.join(', ') }); - return passThrough(); + return passThrough(screenUnreadHead()); } const text = buffer.toString('utf8'); let r; try { - r = screenText(text, head, reqCtx, answeredWithoutBody ? stillToAnswer : undefined); + r = screenText(text, head, reqCtx, notYetAnswered); } catch (err) { notify(onError, err, 'onError'); @@ -1374,7 +1522,6 @@ export async function createProtection(options = {}) { } if (r.verdict === 'block') { - const body = JSON.stringify({ error: 'Response withheld by Patchstack (sensitive data detected)' }); if (reallySent()) { // Chunked, so the body can still change, but the status already went out as the application's. res.destroy?.(); @@ -1383,16 +1530,7 @@ export async function createProtection(options = {}) { } // One framing: the length describes this body, so any transfer coding the application chose // for its own goes with it. - sendHead({ - status: 500, - headers: new Map([ - ['content-type', 'application/json'], - ['content-length', String(Buffer.byteLength(body))], - ['transfer-encoding', null], - ]), - }); - - return origEnd(body, cb); + return sendWithheld(cb); } // Redact: the rule's header values, and a length for the rewritten body. A length is always @@ -1932,6 +2070,17 @@ function screenableContentType(ct) { if (base === 'application/octet-stream') return 'sniff'; // maybe a text/JSON export mislabeled return 'skip'; // image/video/audio/font/pdf/zip/wasm/… — don't buffer binary } +/** What a response rule's prefilter looks for its anchors in, lower-cased: the body and every header value. */ +function prefilterText(body, headers) { + const values = []; + for (const value of Object.values(headers ?? {})) { + if (Array.isArray(value)) values.push(...value.map(String)); + else if (value !== undefined && value !== null) values.push(String(value)); + } + + return [body, ...values].join('\n').toLowerCase(); +} + // Cheap binary sniff over a byte prefix: a NUL byte, or many control chars, means "don't treat as text". function looksBinary(bytes) { const n = Math.min(bytes.length, 512); @@ -2075,8 +2224,9 @@ function headerObject(headers) { function hasSpanMutations(rule) { let found = false; const walk = (conds) => { - for (const c of conds ?? []) { + for (const c of Array.isArray(conds) ? conds : []) { if (found) return; + if (!c || typeof c !== 'object') continue; if (Array.isArray(c.rules)) walk(c.rules); const isSpan = c.match && (c.match.type === 'regex' || c.match.type === 'contains' || c.match.type === 'stripos'); if (isSpan && Array.isArray(c.mutations) && c.mutations.length) found = true; @@ -2086,13 +2236,66 @@ function hasSpanMutations(rule) { return found; } +/** + * Where a span redactor masks, from the parameter its condition read: `{ body, allHeaders, headers }`, + * `headers` being a set of lower-cased names — or null when the parameter names nowhere to mask. + * + * - `response.header.` masks that header; `response.headers` masks every header. Neither is the + * body, and a rule that read no body does not rewrite it. + * - `response.body` masks the body and the same text in every header, so a secret found in the body is not + * left behind where it was echoed into a header. + * - `response.status` and the request sources (`BROAD_SCOPE_SOURCES`) name no place in the response. A + * response rule reading them has always masked its match in the body and every header, and still does. + * - A list masks the union of its members' scopes. + * - Anything else is null: a shape the contract refuses (an empty or nested list, an unknown source or + * key, a value that is not a parameter), a valid parameter with no response location (`egress.*`), or no + * parameter at all. Such a condition masks nothing, and the rule that carries it is reported when the + * rules load. + * + * A group (`parameter: "rules"`) is not a scope of its own: the engine evaluates each of its conditions + * with that condition's parameter, and each yields its own redactor with its own scope. + */ +function redactorScope(parameter) { + if (parameter === undefined || parameter === null || parameterProblem(parameter) !== null) return null; + const scope = { body: false, allHeaders: false, headers: new Set() }; + for (const name of Array.isArray(parameter) ? parameter : [parameter]) { + // Checked here rather than left to the contract, which answers a different question: a member that is + // not a parameter name has no place to mask, whatever the validator makes of it. + if (typeof name !== 'string' || name === '') return null; + if (name.startsWith('response.header.')) scope.headers.add(name.slice('response.header.'.length).toLowerCase()); + else if (name === 'response.headers') scope.allHeaders = true; + else if (name === 'response.body' || name === 'response.status' || BROAD_SCOPE_SOURCES.has(name.split('.')[0])) { + scope.body = true; + scope.allHeaders = true; + } else return null; + } + + return scope; +} + +// Request sources, whose conditions in a response rule mask their match in the body and every header. +const BROAD_SCOPE_SOURCES = new Set(['get', 'post', 'request', 'cookie', 'files', 'server', 'raw', 'all']); + +/** Does a span redactor apply to this header? `name` is lower-cased, as every screened header name is. */ +function masksHeader(redactor, name) { + return redactor.scope.allHeaders || redactor.scope.headers.has(name); +} + // Derive redaction targets from a rule's own conditions: regex → mask every match; // contains/stripos → mask the literal. (Other match types can't identify a span → the // rule falls back to block.) -function extractRedactors(rule) { +function extractRedactors(rule, onUnscoped) { const out = []; + // A span redactor needs a place to mask. One whose parameter names none is left out and reported. + const scoped = (c) => { + const scope = redactorScope(c.parameter); + if (scope === null) onUnscoped?.(c.parameter); + + return scope; + }; const walk = (conds) => { - for (const c of conds ?? []) { + for (const c of Array.isArray(conds) ? conds : []) { + if (!c || typeof c !== 'object') continue; if (Array.isArray(c.rules)) walk(c.rules); const m = c.match; if (!m) continue; @@ -2102,21 +2305,26 @@ function extractRedactors(rule) { const safe = safeRegExp(m.value); if (safe) { const flags = safe.flags.includes('g') ? safe.flags : safe.flags + 'g'; + const scope = scoped(c); try { - out.push({ re: new RegExp(safe.source, flags) }); + if (scope) out.push({ re: new RegExp(safe.source, flags), scope }); } catch { /* skip invalid */ } } } else if ((m.type === 'contains' || m.type === 'stripos') && m.value != null) { - out.push({ literal: String(m.value) }); + const scope = scoped(c); + if (scope) out.push({ literal: String(m.value), scope }); } else if (m.type === 'jwt_claim_equals' && typeof m.claim === 'string') { // A span-producing target, not a predicate. A boolean-only matcher would leave `redact` with // no span to mask, so the rule would fall back to withholding the WHOLE response — turning a // one-token leak into an outage. The spans come from the same `jwtClaimSpans` the matcher // used, so what is reported and what is masked cannot diverge. - out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') } }); - } else if (m.type === 'array_key_value' && m.match && isBodyParam(c.parameter)) { + const scope = scoped(c); + if (scope) out.push({ jwtClaim: { claim: m.claim, value: String(m.value ?? '') }, scope }); + } else if (m.type === 'array_key_value' && m.match && Array.isArray(c.parameter) && !readsBody(c.parameter) && redactorScope(c.parameter) === null) { + onUnscoped?.(c.parameter); + } else if (m.type === 'array_key_value' && m.match && readsBody(c.parameter)) { // Structural redaction: mask the value at a JSON path (fanning out over arrays) rather than // a text span — e.g. key "orders.customers.email" masks that field in every array element. const keys = Array.isArray(m.key) ? m.key : [m.key]; @@ -2289,6 +2497,15 @@ function applyRedactors(body, redactors, mask, transform) { // A response-body redaction target (array_key_value masks the JSON body). A bare condition with no // parameter also defaults to the body. +/** Does a condition read the body, alone or as a member of a parameter list the contract accepts? */ +function readsBody(parameter) { + if (!Array.isArray(parameter)) return isBodyParam(parameter); + + const names = parameter.every((member) => typeof member === 'string' && member !== ''); + + return names && parameterProblem(parameter) === null && parameter.some((member) => isBodyParam(member)); +} + function isBodyParam(parameter) { return parameter == null || parameter === 'response.body' || parameter === 'raw' || parameter === 'response.raw'; } diff --git a/tests/protect/header-block-without-a-body.test.ts b/tests/protect/header-block-without-a-body.test.ts new file mode 100644 index 00000000..5ebd7d36 --- /dev/null +++ b/tests/protect/header-block-without-a-body.test.ts @@ -0,0 +1,302 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { gzipSync } from 'node:zlib'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A `block` rule that reads response headers alone is decided without the body, so it is enforced on a + * response whose body was not screened — over the cap, binary, a live stream, or content-encoded — as + * long as the head has not gone out. When it has, the match is reported and the limitation is recorded + * as a `headers-sent` skip. A rule that reads the body, or has no explicit `block` action, is left out + * when there is no body. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const BINARY = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0x02, 0x03]); +const WITHHELD = { error: 'Response withheld by Patchstack (sensitive data detected)' }; + +const blockRule = (extra: Record = {}) => ({ + id: 'header-block', + phase: 'response', + category: 'secret-exposure', + action: 'block', + rule_v2: [{ parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }], + ...extra, +}); + +async function guard(rules: object[], mode = 'block') { + const detections: any[] = []; + const skips: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode, + responseRules: rules, + onDetect: (event: any) => detections.push(event), + onSkip: (skip: any) => skips.push(skip), + onError: () => {}, + }); + + return { protection, detections, skips }; +} + +const unscreened: Array<[string, () => Response]> = [ + ['a binary body', () => new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } })], + ['a body over the cap', () => new Response('x'.repeat(600 * 1024), { headers: { 'content-type': 'text/plain', 'x-sample': SAMPLE } })], + ['a live stream', () => new Response('data: 1\n\n', { headers: { 'content-type': 'text/event-stream', 'x-sample': SAMPLE } })], + ['an encoded body', () => new Response(gzipSync('{"a":1}'), { headers: { 'content-type': 'application/json', 'content-encoding': 'gzip', 'x-sample': SAMPLE } })], +]; + +describe('a header block when the body is not screened (fetch)', () => { + it.each(unscreened)('withholds %s', async (_label, make) => { + const { protection, detections } = await guard([blockRule()]); + const out = await protection.screenResponse(make()); + + expect(out.status).toBe(500); + expect(await out.json()).toEqual(WITHHELD); + expect(out.headers.get('x-sample')).toBeNull(); + expect(detections).toHaveLength(1); + }); + + it('wins over a header redaction on the same response', async () => { + const redact = { ...blockRule(), id: 'header-redact', action: 'redact' }; + const { protection, detections } = await guard([redact, blockRule()]); + const out = await protection.screenResponse(unscreened[0][1]()); + + expect(out.status).toBe(500); + expect(detections).toHaveLength(2); + }); + + it('records the match in dry-run and sends the response', async () => { + const { protection, detections } = await guard([blockRule()], 'dry-run'); + const original = unscreened[0][1](); + + expect(await protection.screenResponse(original)).toBe(original); + expect(detections).toHaveLength(1); + }); + + it.each([ + ['one that reads the body', { rule_v2: [{ parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }] }], + ['one that reads the body beside a header', { rule_v2: [ + { parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + { parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + ] }], + ['one with no explicit action', { action: undefined }], + ])('leaves out %s', async (_label, shape) => { + const { protection, detections } = await guard([blockRule(shape)]); + const original = unscreened[0][1](); + + expect(await protection.screenResponse(original)).toBe(original); + expect(detections).toHaveLength(0); + }); +}); + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + + return `http://127.0.0.1:${(server.address() as { port: number }).port}/`; +} + +async function rawGet(url: string): Promise<{ status: number; headers: Record; body: Buffer }> { + const { request } = await import('node:http'); + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: Buffer.concat(chunks) })); + }).on('error', reject).end(); + }); +} + +describe('a header block when the body is not screened (Node)', () => { + const bodies: Array<[string, string, Buffer, Record]> = [ + ['a binary body', 'image/png', Buffer.from(BINARY), {}], + ['a sniffed binary body', 'application/octet-stream', Buffer.from(BINARY), {}], + ['an encoded body', 'application/json', gzipSync('{"a":1}'), { 'content-encoding': 'gzip' }], + ['a body over the cap', 'text/plain', Buffer.from('x'.repeat(600 * 1024)), {}], + ]; + + function expectWithheld(got: { status: number; headers: Record; body: Buffer }) { + expect(got.status).toBe(500); + expect(Number(got.headers['content-length'])).toBe(got.body.length); + expect(JSON.parse(got.body.toString())).toEqual(WITHHELD); + expect(got.headers['x-sample']).toBeUndefined(); + expect(got.headers['content-encoding']).toBeUndefined(); + } + + it.each(bodies)('withholds %s, headers set before the body', async (_label, type, body, extra) => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(extra)) res.setHeader(name, value); + res.setHeader('x-sample', SAMPLE); + res.end(body); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); + + it.each(bodies)('withholds %s, headers supplied through writeHead', async (_label, type, body, extra) => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.writeHead(200, 'OK', { 'content-type': type, 'x-sample': SAMPLE, ...extra }); + res.end(body); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); + + it('withholds a body that crosses the cap across writes, and drops what follows', async () => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const errors: unknown[] = []; + const written: boolean[] = []; + const ended = new Promise((resolve) => { + void listen((req, res) => + node(req, res, () => { + res.on('error', (error) => errors.push(error)); + res.setHeader('content-type', 'text/plain'); + res.setHeader('x-sample', SAMPLE); + for (let i = 0; i < 4; i++) written.push(res.write('x'.repeat(200 * 1024))); + res.end('tail', () => resolve()); + }), + ).then((url) => rawGet(url).then((got) => { + expectWithheld(got); + })); + }); + + await ended; + expect(errors).toEqual([]); + expect(written.every(Boolean)).toBe(true); + expect(detections).toHaveLength(1); + }); + + it('completes the application\'s end once when the cap is crossed by its final chunk', async () => { + const { protection } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const calls: unknown[] = []; + const errors: unknown[] = []; + const url = await listen((req, res) => + node(req, res, () => { + res.on('error', (error) => errors.push(error)); + res.setHeader('content-type', 'text/plain'); + res.setHeader('x-sample', SAMPLE); + res.end('x'.repeat(600 * 1024), (error?: unknown) => calls.push(error ?? null)); + }), + ); + + expectWithheld(await rawGet(url)); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(calls).toEqual([null]); + expect(errors).toEqual([]); + }); + + it('reports a match whose head went out before the guard saw it, and records that it could not be enforced', async () => { + const { protection, detections, skips } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + node(req, res, () => res.end(Buffer.from(BINARY))); + }); + + const got = await rawGet(url); + expect(got.status).toBe(200); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips).toContainEqual(expect.objectContaining({ reason: 'headers-sent', detail: { action: 'block' } })); + }); + + it.each([ + ['a streamed text body', 'text/plain', ['first ', 'second ', 'third']], + ['a streamed binary body', 'application/octet-stream', [Buffer.from(BINARY), Buffer.from(BINARY)]], + ])('withholds on flushHeaders before %s', async (_label, type, parts) => { + for (const viaWriteHead of [false, true]) { + const { protection, detections, skips } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + if (viaWriteHead) res.writeHead(200, { 'content-type': type, 'x-sample': SAMPLE }); + else { + res.setHeader('content-type', type); + res.setHeader('x-sample', SAMPLE); + } + res.flushHeaders(); + for (const part of parts.slice(0, -1)) res.write(part); + res.end(parts[parts.length - 1]); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).not.toContain('headers-sent'); + await close?.(); + close = null; + } + }); + + it('withholds a body written before the application ends it, with no explicit head', async () => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/octet-stream'); + res.setHeader('x-sample', SAMPLE); + res.write(Buffer.from(BINARY)); + res.end(Buffer.from(BINARY)); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); + + it('leaves the response alone in dry-run', async () => { + const { protection, detections } = await guard([blockRule()], 'dry-run'); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.end(Buffer.from(BINARY)); + }), + ); + + const got = await rawGet(url); + expect(got.status).toBe(200); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + }); + + it('withholds a screened text response once', async () => { + const { protection, detections } = await guard([blockRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('x-sample', SAMPLE); + res.end('{"ok":true}'); + }), + ); + + expectWithheld(await rawGet(url)); + expect(detections).toHaveLength(1); + }); +}); diff --git a/tests/protect/header-redaction-without-a-body.test.ts b/tests/protect/header-redaction-without-a-body.test.ts new file mode 100644 index 00000000..3eab87fa --- /dev/null +++ b/tests/protect/header-redaction-without-a-body.test.ts @@ -0,0 +1,293 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { gzipSync } from 'node:zlib'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A redaction keyed on a response header reads the header alone, so it is decided — and carried out — + * whether or not the body was screened: over the cap, binary, a live stream, or content-encoded. A rule + * that reads the body is still left out when there is no body to read. + * + * A rule's `prefilter` anchors are looked for in the header values as well as the body, since a rule + * may read either. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const BINARY = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0x02, 0x03]); + +const headerRule = (extra: Record = {}) => ({ + id: 'header-value', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [{ parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }], + ...extra, +}); + +async function guard(rules: object[], mode = 'block') { + const detections: any[] = []; + const skips: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode, + responseRules: rules, + onDetect: (event: any) => detections.push(event), + onSkip: (skip: any) => skips.push(skip), + onError: () => {}, + }); + + return { protection, detections, skips }; +} + +describe('a header redaction when the body is not screened (fetch)', () => { + it.each([ + ['a binary body', () => new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } }), 'non-text-content-type'], + ['a body over the cap', () => new Response('x'.repeat(600 * 1024), { headers: { 'content-type': 'text/plain', 'x-sample': SAMPLE } }), 'body-cap'], + ['a live stream', () => new Response('data: 1\n\n', { headers: { 'content-type': 'text/event-stream', 'x-sample': SAMPLE } }), 'live-stream'], + ['an encoded body', () => new Response(gzipSync('{"a":1}'), { headers: { 'content-type': 'application/json', 'content-encoding': 'gzip', 'x-sample': SAMPLE } }), 'encoded-body'], + ])('masks the header on %s and leaves the body alone', async (_label, make, reason) => { + const { protection, detections, skips } = await guard([headerRule()]); + const original = make(); + const expected = new Uint8Array(await original.clone().arrayBuffer()); + const out = await protection.screenResponse(original); + + expect(out.headers.get('x-sample')).toBe('[REDACTED]'); + expect(new Uint8Array(await out.arrayBuffer())).toEqual(expected); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).toContain(reason); + }); + + it('records the match in dry-run without changing the header', async () => { + const { protection, detections } = await guard([headerRule()], 'dry-run'); + const out = await protection.screenResponse(new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } })); + + expect(out.headers.get('x-sample')).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it.each([ + ['one that reads the body', { rule_v2: [{ parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }] }], + ['one that reads the status alone', { rule_v2: [{ parameter: 'response.status', match: { type: 'contains', value: '200' } }] }], + ['one that reads the body beside a header', { rule_v2: [ + { parameter: 'response.header.x-sample', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + { parameter: 'response.body', match: { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' } }, + ] }], + ['an encoding rule', { action: 'encode' }], + ['one whose match is decoded first', { rule_v2: [{ parameter: 'response.header.x-sample', mutations: ['urldecode'], match: { type: 'contains', value: SAMPLE } }] }], + ['one with no span to mask', { rule_v2: [{ parameter: 'response.header.x-sample', match: { type: 'isset' } }] }], + ])('leaves out %s', async (_label, shape) => { + const { protection, detections } = await guard([headerRule(shape)]); + const original = new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } }); + const out = await protection.screenResponse(original); + + expect(out).toBe(original); + expect(detections).toHaveLength(0); + }); +}); + +describe('a prefilter on a header redaction (fetch)', () => { + it.each([ + ['a screened body', () => new Response('{"ok":true}', { headers: { 'content-type': 'application/json', 'x-sample': SAMPLE } })], + ['an unscreened body', () => new Response(BINARY, { headers: { 'content-type': 'image/png', 'x-sample': SAMPLE } })], + ])('finds its anchor in the header value with %s', async (_label, make) => { + const { protection, detections } = await guard([headerRule({ prefilter: ['sample-token'] })]); + const out = await protection.screenResponse(make()); + + expect(out.headers.get('x-sample')).toBe('[REDACTED]'); + expect(detections).toHaveLength(1); + }); + + it('still skips the rule when the anchor is nowhere in the response', async () => { + const { protection, detections } = await guard([headerRule({ prefilter: ['absent-anchor'] })]); + const out = await protection.screenResponse(new Response('{"ok":true}', { headers: { 'content-type': 'application/json', 'x-sample': SAMPLE } })); + + expect(out.headers.get('x-sample')).toBe(SAMPLE); + expect(detections).toHaveLength(0); + }); +}); + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + + return `http://127.0.0.1:${(server.address() as { port: number }).port}/`; +} + +async function rawGet(url: string): Promise<{ headers: Record; body: Buffer }> { + const { request } = await import('node:http'); + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ headers: res.headers, body: Buffer.concat(chunks) })); + }).on('error', reject).end(); + }); +} + +describe('a header redaction when the body is not screened (Node)', () => { + const bodies: Array<[string, string, Buffer, Record, string]> = [ + ['a binary body', 'image/png', Buffer.from(BINARY), {}, 'non-text-content-type'], + ['a sniffed binary body', 'application/octet-stream', Buffer.from(BINARY), {}, 'binary-body'], + ['an encoded body', 'application/json', gzipSync('{"a":1}'), { 'content-encoding': 'gzip' }, 'encoded-body'], + ['a body over the cap', 'text/plain', Buffer.from('x'.repeat(600 * 1024)), {}, 'body-cap'], + ]; + + it.each(bodies)('masks the header on %s, set before the body', async (_label, type, body, extra, reason) => { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(extra)) res.setHeader(name, value); + res.setHeader('x-sample', SAMPLE); + res.end(body); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.equals(body)).toBe(true); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).toContain(reason); + }); + + it.each(bodies)('masks the header on %s, supplied through writeHead', async (_label, type, body, extra) => { + const { protection, detections } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.writeHead(200, { 'content-type': type, 'x-sample': SAMPLE, ...extra }); + res.end(body); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.equals(body)).toBe(true); + expect(detections).toHaveLength(1); + }); + + it.each([ + ['a streamed text body', 'text/plain', ['first ', 'second ', 'third']], + ['a streamed binary body', 'image/png', [Buffer.from(BINARY), Buffer.from(BINARY)]], + ])('masks the header on flushHeaders before %s', async (_label, type, parts) => { + for (const viaWriteHead of [false, true]) { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + if (viaWriteHead) res.writeHead(200, { 'content-type': type, 'x-sample': SAMPLE }); + else { + res.setHeader('content-type', type); + res.setHeader('x-sample', SAMPLE); + } + res.flushHeaders(); + for (const part of parts.slice(0, -1)) res.write(part); + res.end(parts[parts.length - 1]); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.equals(Buffer.concat(parts.map((p) => Buffer.from(p))))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips.map((s: any) => s.reason)).not.toContain('headers-sent'); + await close?.(); + close = null; + } + }); + + it.each([ + ['a text body', 'text/plain', 'plain text'], + ['a binary body', 'image/png', Buffer.from(BINARY)], + ])('reports a match once in dry-run when flushHeaders comes before %s', async (_label, type, body) => { + const { protection, detections } = await guard([headerRule()], 'dry-run'); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + res.end(body); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it('reports a match whose header went out before the guard saw it, and records that it could not be masked', async () => { + const { protection, detections, skips } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.flushHeaders(); + node(req, res, () => res.end(Buffer.from(BINARY))); + }); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe(SAMPLE); + expect(got.body.equals(Buffer.from(BINARY))).toBe(true); + expect(detections).toHaveLength(1); + expect(skips).toContainEqual(expect.objectContaining({ reason: 'headers-sent', detail: { headers: ['x-sample'] } })); + }); + + it('reports a screened text response once', async () => { + const { protection, detections } = await guard([headerRule()]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('x-sample', SAMPLE); + res.end('{"ok":true}'); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.body.toString()).toBe('{"ok":true}'); + expect(detections).toHaveLength(1); + }); + + it('leaves the header alone in dry-run', async () => { + const { protection, detections } = await guard([headerRule()], 'dry-run'); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'image/png'); + res.setHeader('x-sample', SAMPLE); + res.end(Buffer.from(BINARY)); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it('finds a prefilter anchor in the header value', async () => { + const { protection, detections } = await guard([headerRule({ prefilter: ['sample-token'] })]); + const node = protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('x-sample', SAMPLE); + res.end('{"ok":true}'); + }), + ); + + const got = await rawGet(url); + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(detections).toHaveLength(1); + }); +}); diff --git a/tests/protect/redaction-scope-malformed-parameters.test.ts b/tests/protect/redaction-scope-malformed-parameters.test.ts new file mode 100644 index 00000000..9f68bae1 --- /dev/null +++ b/tests/protect/redaction-scope-malformed-parameters.test.ts @@ -0,0 +1,227 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A response rule whose parameter list holds something other than a parameter name is loaded, not + * rejected: the condition masks nothing, the rule is reported through `onError`, and a response the rule + * matches follows the fallback for a match with nothing to mask. That holds for the first load and for a + * rule arriving on a refresh, which must not throw either. + */ + +afterEach(() => vi.restoreAllMocks()); + +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const PATTERN = { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' }; +const EMPTY = { firewall: [], whitelists: [], whitelist_keys: {} }; +const URL_OPT = 'https://x.test/monitor/pulse'; + +// `matches`: whether the engine can match the rule at all. A list whose readable member reads the header +// matches, and with nothing it can mask, withholds; one with no readable member never matches; and a +// member the engine cannot resolve fails the rule's evaluation, which the engine treats as no match. +const malformed: Array<[string, unknown[], boolean]> = [ + ['[null]', [null], false], + ['[undefined]', [undefined], false], + ['a header list with null', ['response.header.x-sample', null], true], + ['a body list with null', ['response.body', null], true], + ['a list with a number', ['response.header.x-sample', 42], false], + ['a list with an object', ['response.header.x-sample', { name: 'response.body' }], false], + ['a list with an empty string', ['response.header.x-sample', ''], true], +]; + +const SCOPE_REPORT = 'names no place in the response to mask'; +const scopeReports = (errors: Error[]) => errors.filter((e) => e.message.includes(SCOPE_REPORT)); + +/** Withheld, or sent without anything masked beyond what is known: never widened. */ +async function expectFallback(out: Response, matches: boolean) { + expect(out.status).toBe(matches ? 500 : 200); + if (!matches) { + expect(out.headers.get('x-sample')).toBe(SAMPLE); + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(await out.text()).toBe(`{"note":"${SAMPLE}"}`); + } +} + +const badRule = (parameter: unknown) => ({ + id: 'malformed', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [{ parameter, match: PATTERN }], +}); + +const goodRule = { + id: 'well-formed', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [{ parameter: 'response.header.x-good', match: PATTERN }], +}; + +const response = () => + new Response(`{"note":"${SAMPLE}"}`, { + headers: { 'content-type': 'application/json', 'x-sample': SAMPLE, 'x-good': SAMPLE, 'x-other': SAMPLE }, + }); + +describe('a malformed parameter list on the first load', () => { + it.each(malformed)('loads with %s, reports it, and follows the fallback', async (_label, parameter, matches) => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [badRule(parameter)], + }); + + expect(scopeReports(errors)).toHaveLength(1); + await expectFallback(await protection.screenResponse(response()), matches); + }); + + it('keeps a well-formed rule beside it working', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [badRule([null]), goodRule], + }); + const out = await protection.screenResponse(response()); + + expect(out.status).toBe(200); + expect(out.headers.get('x-good')).toBe('[REDACTED]'); + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(errors).toHaveLength(1); + }); + + it.each([ + ['a null condition', [null, { parameter: 'response.header.x-good', match: PATTERN }]], + ['a condition that is not an object', ['text', { parameter: 'response.header.x-good', match: PATTERN }]], + ])('loads a rule with %s without widening a mask', async (_label, rule_v2) => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ ...goodRule, rule_v2 }], + }); + const out = await protection.screenResponse(response()); + + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(out.headers.get('x-sample')).toBe(SAMPLE); + // Loading reads past the condition it cannot use. What is reported comes from evaluating the rule, + // once per evaluation, and not from loading it. + const fromLoading = errors.filter((e) => /reading 'rules'|reading 'match'|reading 'mutations'/.test(e.message)); + expect(fromLoading).toEqual([]); + }); + + it.each([['match'], ['mutations']])('loads a rule whose condition cannot be read (%s) and reports it', async (property) => { + const errors: Error[] = []; + const condition: Record = { parameter: 'response.header.x-good', match: PATTERN }; + Object.defineProperty(condition, property, { get() { throw new Error('unreadable condition'); }, enumerable: true }); + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ ...goodRule, rule_v2: [condition] }], + }); + // Reported by loading, before any response is screened. + expect(errors.some((e) => e.message === 'unreadable condition')).toBe(true); + + const out = await protection.screenResponse(response()); + expect(out.headers.get('x-other')).toBe(SAMPLE); + }); + + it('reports a structural condition over a list with null, and withholds a response it matches', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: EMPTY, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ + ...goodRule, + rule_v2: [{ parameter: ['response.body', null], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } }], + }], + }); + const out = await protection.screenResponse(response()); + + expect(scopeReports(errors)).toHaveLength(1); + expect(out.status).toBe(500); + }); +}); + +describe('a malformed parameter list arriving on a refresh', () => { + // The rule source validates a delivered bundle against the rule contract, and an update carrying a rule + // it refuses is rejected whole: the previous rules stay in force. A `null` member passes that check, so + // it is applied, and the scope handling reports it. + const refused = new Set(['a list with a number', 'a list with an object', 'a list with an empty string']); + const previous = { ...goodRule, id: 'previous', rule_v2: [{ parameter: 'response.header.x-other', match: PATTERN }] }; + + it.each(malformed.filter(([label]) => label !== '[undefined]'))('refreshes with %s without throwing', async (label, parameter, matches) => { + const bundle = { firewall: [badRule(parameter), goodRule], whitelists: [], whitelist_keys: {} }; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ ...EMPTY, firewall: [previous] }), { status: 200 })) + .mockResolvedValue(new Response(JSON.stringify(bundle), { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const errors: Error[] = []; + const protection: any = await createProtection({ + siteUuid: 's', + pulseRulesUrl: URL_OPT, + mode: 'block', + reportManifest: false, + onError: (error: Error) => errors.push(error), + }); + expect(scopeReports(errors)).toHaveLength(0); + + const outcome = await protection.refresh(); + const out = await protection.screenResponse(response()); + + if (refused.has(label)) { + expect(outcome).toMatchObject({ ok: false, reason: 'update rejected' }); + expect(errors.some((e) => e.message.includes('rejected the entire update'))).toBe(true); + expect(scopeReports(errors)).toHaveLength(0); + // The previous rules are still the ones in force. + expect(out.status).toBe(200); + expect(out.headers.get('x-other')).toBe('[REDACTED]'); + expect(out.headers.get('x-good')).toBe(SAMPLE); + + return; + } + + expect(outcome).toMatchObject({ ok: true }); + expect(scopeReports(errors)).toHaveLength(1); + expect(out.status).toBe(matches ? 500 : 200); + if (!matches) { + // The well-formed rule the same refresh delivered is in force beside it, and the previous one is not. + expect(out.headers.get('x-good')).toBe('[REDACTED]'); + expect(out.headers.get('x-other')).toBe(SAMPLE); + } + }); +}); + +describe('a malformed parameter list arriving through the push endpoint', () => { + it('refreshes and reports it', async () => { + const bundle = { firewall: [badRule(['response.header.x-sample', null]), goodRule], whitelists: [], whitelist_keys: {} }; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(JSON.stringify(EMPTY), { status: 200 })) + .mockResolvedValue(new Response(JSON.stringify(bundle), { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const errors: Error[] = []; + const protection: any = await createProtection({ + siteUuid: 's', + pulseRulesUrl: URL_OPT, + mode: 'block', + reportManifest: false, + refreshSecret: 'sample-secret', + onError: (error: Error) => errors.push(error), + }); + + const handled = await protection.refreshHandler()( + new Request('https://app.example.test/_ps/refresh', { headers: { 'x-patchstack-refresh': 'sample-secret' } }), + ); + + expect(handled.status).toBe(200); + expect(scopeReports(errors)).toHaveLength(1); + expect((await protection.screenResponse(response())).status).toBe(500); + }); +}); diff --git a/tests/protect/redaction-scope-parameter-forms.test.ts b/tests/protect/redaction-scope-parameter-forms.test.ts new file mode 100644 index 00000000..50b4ce51 --- /dev/null +++ b/tests/protect/redaction-scope-parameter-forms.test.ts @@ -0,0 +1,235 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A redaction's scope does not depend on how its parameter is written. A list masks the union of its + * members' scopes, a single-item list masks what the string does, and a condition inside a group masks + * where it reads. The same holds on every path a response can take: screened on fetch or Node, with the + * head flushed early, or with a body that is not screened. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const PATTERN = { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' }; +const HEADERS = { 'x-sample': SAMPLE, 'x-extra': SAMPLE, 'x-other': SAMPLE }; +const BINARY = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01]); + +type Form = { label: string; condition: Record; masked: string[]; readsBody: boolean }; + +const leaf = (parameter: unknown) => ({ parameter, match: PATTERN }); +const forms: Form[] = [ + { label: 'a header, as a string', condition: leaf('response.header.x-sample'), masked: ['x-sample'], readsBody: false }, + { label: 'a header, as a one-item list', condition: leaf(['response.header.x-sample']), masked: ['x-sample'], readsBody: false }, + { label: 'two headers, as a list', condition: leaf(['response.header.x-sample', 'response.header.X-Extra']), masked: ['x-sample', 'x-extra'], readsBody: false }, + { label: 'all headers, as a string', condition: leaf('response.headers'), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: false }, + { label: 'all headers, as a list', condition: leaf(['response.headers']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: false }, + { label: 'a header list in a group', condition: { parameter: 'rules', rules: [leaf(['response.header.x-sample'])] }, masked: ['x-sample'], readsBody: false }, + { label: 'the body, as a one-item list', condition: leaf(['response.body']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: true }, + { label: 'the body and a header, as a list', condition: leaf(['response.body', 'response.header.x-sample']), masked: ['x-sample', 'x-extra', 'x-other'], readsBody: true }, +]; + +const ruleOf = (condition: Record) => ({ + id: 'scoped', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: [condition], +}); + +async function guard(condition: Record) { + const detections: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + responseRules: [ruleOf(condition)], + onDetect: (event: any) => detections.push(event), + onError: () => {}, + }); + + return { protection, detections }; +} + +type Seen = { status: number; body: Buffer; headers: Record }; + +const seenHeaders = (get: (name: string) => string | null | undefined) => + Object.fromEntries(Object.keys(HEADERS).map((name) => [name, get(name) ?? null])); + +async function viaFetch(condition: Record, body: string | Buffer, type: string): Promise { + const { protection } = await guard(condition); + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': type, ...HEADERS } })); + + return { status: out.status, body: Buffer.from(await out.arrayBuffer()), headers: seenHeaders((n) => out.headers.get(n)) }; +} + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function viaNode(condition: Record, body: string | Buffer, type: string, flush: boolean): Promise { + const { protection } = await guard(condition); + const node = protection.node({ screenResponses: true }); + const server = createServer((req: IncomingMessage, res: ServerResponse) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(HEADERS)) res.setHeader(name, value); + if (flush) res.flushHeaders(); + res.end(body); + }), + ); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + const { request } = await import('node:http'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}/`; + + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const got = res.headers; + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks), headers: seenHeaders((n) => got[n] as string | undefined) }); + }); + }).on('error', reject).end(); + }); +} + +const expectedHeaders = (masked: string[]) => + Object.fromEntries(Object.keys(HEADERS).map((name) => [name, masked.includes(name) ? '[REDACTED]' : SAMPLE])); + +describe.each([ + ['fetch', (c: Record, b: string | Buffer, t: string) => viaFetch(c, b, t)], + ['Node', (c: Record, b: string | Buffer, t: string) => viaNode(c, b, t, false)], +])('parameter forms on a screened body (%s)', (_path, run) => { + it.each(forms)('$label', async ({ condition, masked, readsBody }) => { + const body = `{"note":"${SAMPLE}"}`; + const got = await run(condition, body, 'application/json'); + + expect(got.status).toBe(200); + expect(got.body.toString()).toBe(readsBody ? '{"note":"[REDACTED]"}' : body); + expect(got.headers).toEqual(expectedHeaders(masked)); + }); + + it.each(forms.filter((f) => !f.readsBody))('$label leaves a JSON key alone', async ({ condition, masked }) => { + const body = `{"${SAMPLE}":1}`; + const got = await run(condition, body, 'application/json'); + + expect(got.status).toBe(200); + expect(got.body.toString()).toBe(body); + expect(got.headers).toEqual(expectedHeaders(masked)); + }); +}); + +describe.each([ + ['fetch, a body that is not screened', (c: Record) => viaFetch(c, BINARY, 'image/png')], + ['Node, a body that is not screened', (c: Record) => viaNode(c, BINARY, 'image/png', false)], + ['Node, an early flush', (c: Record) => viaNode(c, 'plain text', 'text/plain', true)], +])('header-only parameter forms (%s)', (_path, run) => { + it.each(forms.filter((f) => !f.readsBody))('$label', async ({ condition, masked }) => { + const got = await run(condition); + + expect(got.status).toBe(200); + expect(got.headers).toEqual(expectedHeaders(masked)); + }); +}); + +describe('a legacy parameter that names no place in the response', () => { + it.each([ + ['the status', 'response.status'], + ['a request source', 'get.q'], + ['a request source in a list', ['cookie.session', 'server.HTTP_HOST']], + ])('keeps its broad mask with %s', async (_label, parameter) => { + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + responseRules: [{ ...ruleOf(leaf('response.body')), rule_v2: [leaf('response.body'), { parameter, match: { type: 'regex', value: '/EXTRA-\\d+/' } }] }], + }); + const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}","more":"EXTRA-1"}`, { + headers: { 'content-type': 'application/json', 'x-other': 'EXTRA-1' }, + })); + + expect(JSON.parse(await out.text())).toEqual({ note: '[REDACTED]', more: '[REDACTED]' }); + expect(out.headers.get('x-other')).toBe('[REDACTED]'); + }); +}); + +describe('a parameter shape with no place to mask', () => { + const unsupported: Array<[string, unknown]> = [ + ['a nested list', ['response.header.x-extra', ['response.body']]], + ['an empty list', []], + ['a value that is not a parameter', 42], + ['an unknown source', 'nope.value'], + ['an unknown response key', 'response.nope'], + ['an outbound source', 'egress.url'], + ['a keyed source without its key', 'get'], + ['a key the source does not answer for', 'server.NOT_A_KEY'], + ['no parameter', undefined], + ]; + + async function withUnsupported(parameter: unknown) { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [{ + ...ruleOf(leaf('response.header.x-sample')), + rule_v2: [leaf('response.header.x-sample'), { ...(parameter === undefined ? {} : { parameter }), match: PATTERN }], + }], + }); + + return { protection, errors }; + } + + it.each(unsupported)('does not widen the mask for %s, and reports it', async (_label, parameter) => { + const { protection, errors } = await withUnsupported(parameter); + const body = `{"note":"${SAMPLE}"}`; + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json', ...HEADERS } })); + + expect(out.status).toBe(200); + expect(await out.text()).toBe(body); + expect(seenHeaders((n) => out.headers.get(n))).toEqual(expectedHeaders(['x-sample'])); + expect(errors.map((e) => e.message)).toEqual([expect.stringContaining('names no place in the response to mask')]); + }); + + it('withholds a response whose only condition cannot be masked, rather than report a mask it did not make', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [ruleOf(leaf(['response.header.x-sample', ['response.body']]))], + }); + const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}"}`, { headers: { 'content-type': 'application/json', ...HEADERS } })); + + expect(out.status).toBe(500); + expect(errors).toHaveLength(1); + }); + + it('does not mask a JSON path read through a refused list', async () => { + const errors: Error[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode: 'block', + onError: (error: Error) => errors.push(error), + responseRules: [ruleOf({ parameter: ['response.body', ['response.body']], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } })], + }); + const body = `{"note":"${SAMPLE}","other":1}`; + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json' } })); + + expect(out.status).toBe(500); + expect(await out.text()).not.toContain(SAMPLE); + }); +}); + +describe('structural masking with a parameter list', () => { + it('masks a JSON path read through a one-item list', async () => { + const condition = { parameter: ['response.body'], mutations: ['json_decode'], match: { type: 'array_key_value', key: 'note', match: { type: 'isset' } } }; + const got = await viaFetch(condition, `{"note":"${SAMPLE}","other":1}`, 'application/json'); + + expect(got.status).toBe(200); + expect(JSON.parse(got.body.toString())).toEqual({ note: '[REDACTED]', other: 1 }); + }); +}); diff --git a/tests/protect/response-redaction-scope.test.ts b/tests/protect/response-redaction-scope.test.ts new file mode 100644 index 00000000..57b47610 --- /dev/null +++ b/tests/protect/response-redaction-scope.test.ts @@ -0,0 +1,244 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createProtection } from '../../src/protect/runtime.js'; + +/** + * A redaction masks where its condition read. A `response.header.` condition masks that header + * only — each entry of a multi-valued one — and `response.headers` masks the headers only; neither + * touches the body. A `response.body` condition masks the body, and the same text in the headers. A rule + * with conditions on both masks each where it reads. + */ + +const emptyBundle = { firewall: [], whitelists: [], whitelist_keys: {} }; +const SAMPLE = 'SAMPLE-TOKEN-0123456789'; +const PATTERN = { type: 'regex', value: '/SAMPLE-TOKEN-\\d+/' }; +const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url'); +const TOKEN = `${encode({ alg: 'none', typ: 'JWT' })}.${encode({ role: 'sample' })}.c2lnbmF0dXJl`; + +const rule = (...parameters: string[]) => ({ + id: 'scoped', + phase: 'response', + category: 'secret-exposure', + action: 'redact', + rule_v2: parameters.map((parameter) => ({ parameter, match: PATTERN })), +}); + +async function guard(rules: object[], mode = 'block') { + const detections: any[] = []; + const protection: any = await createProtection({ + rules: emptyBundle, + mode, + responseRules: rules, + onDetect: (event: any) => detections.push(event), + onError: () => {}, + }); + + return { protection, detections }; +} + +type Case = { + label: string; + rules: object[]; + body: string; + headers: Record; + status: number; + expectBody: string; + expectHeaders: Record; +}; + +const cases: Case[] = [ + { + label: 'a header rule leaves the body alone', + rules: [rule('response.header.x-sample')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE }, + status: 200, + expectBody: `{"note":"${SAMPLE}"}`, + expectHeaders: { 'x-sample': '[REDACTED]' }, + }, + { + label: 'a header rule leaves an unrelated header alone', + rules: [rule('response.header.x-sample')], + body: '{"ok":true}', + headers: { 'x-sample': SAMPLE, 'x-other': SAMPLE }, + status: 200, + expectBody: '{"ok":true}', + expectHeaders: { 'x-sample': '[REDACTED]', 'x-other': SAMPLE }, + }, + { + label: 'a header rule leaves a JSON key alone', + rules: [rule('response.header.x-sample')], + body: `{"${SAMPLE}":1}`, + headers: { 'x-sample': SAMPLE }, + status: 200, + expectBody: `{"${SAMPLE}":1}`, + expectHeaders: { 'x-sample': '[REDACTED]' }, + }, + { + label: 'an all-headers rule masks every header and not the body', + rules: [rule('response.headers')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE, 'x-other': SAMPLE }, + status: 200, + expectBody: `{"note":"${SAMPLE}"}`, + expectHeaders: { 'x-sample': '[REDACTED]', 'x-other': '[REDACTED]' }, + }, + { + label: 'a body rule masks the body, and the same text in the headers', + rules: [rule('response.body')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE }, + status: 200, + expectBody: '{"note":"[REDACTED]"}', + expectHeaders: { 'x-sample': '[REDACTED]' }, + }, + { + label: 'a rule on a header and the body masks both', + rules: [rule('response.header.x-sample', 'response.body')], + body: `{"note":"${SAMPLE}"}`, + headers: { 'x-sample': SAMPLE, 'x-other': SAMPLE }, + status: 200, + expectBody: '{"note":"[REDACTED]"}', + expectHeaders: { 'x-sample': '[REDACTED]', 'x-other': '[REDACTED]' }, + }, +]; + +describe('redaction scope (fetch)', () => { + it.each(cases)('$label', async ({ rules, body, headers, status, expectBody, expectHeaders }) => { + const { protection } = await guard(rules); + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json', ...headers } })); + + expect(out.status).toBe(status); + expect(await out.text()).toBe(expectBody); + for (const [name, value] of Object.entries(expectHeaders)) expect(out.headers.get(name)).toBe(value); + }); + + it('masks a header rule entry by entry on a multi-valued header', async () => { + const { protection } = await guard([rule('response.header.set-cookie')]); + const headers = new Headers({ 'content-type': 'application/json' }); + headers.append('set-cookie', `a=${SAMPLE}`); + headers.append('set-cookie', 'b=plain'); + headers.append('x-other', SAMPLE); + const out = await protection.screenResponse(new Response(`{"note":"${SAMPLE}"}`, { headers })); + + expect(out.headers.getSetCookie()).toEqual(['a=[REDACTED]', 'b=plain']); + expect(out.headers.get('x-other')).toBe(SAMPLE); + expect(await out.text()).toBe(`{"note":"${SAMPLE}"}`); + }); + + it('encodes the body only for a rule on the body', async () => { + const encode = (parameter: string) => ({ ...rule(parameter), action: 'encode', rule_v2: [{ parameter, match: { type: 'contains', value: '' } }] }); + const body = JSON.stringify({ note: 'x' }); + for (const [parameter, expected] of [['response.header.x-sample', body], ['response.body', JSON.stringify({ note: '<b>x' })]]) { + const { protection } = await guard([encode(parameter)]); + const out = await protection.screenResponse(new Response(body, { headers: { 'content-type': 'application/json', 'x-sample': '' } })); + + expect(await out.text()).toBe(expected); + expect(out.headers.get('x-sample')).toBe(''); + } + }); + + it.each([ + ['a mixed-case header name', { parameter: 'response.header.X-Sample', match: PATTERN }, SAMPLE], + ['a literal match', { parameter: 'response.header.x-sample', match: { type: 'contains', value: SAMPLE } }, SAMPLE], + ['a token claim', { parameter: 'response.header.x-sample', match: { type: 'jwt_claim_equals', claim: 'role', value: 'sample' } }, TOKEN], + ])('keeps %s to its header', async (_label, condition, value) => { + const { protection } = await guard([{ ...rule(), rule_v2: [condition] }]); + const out = await protection.screenResponse(new Response(`{"note":"${value}"}`, { + headers: { 'content-type': 'application/json', 'x-sample': value, 'x-other': value }, + })); + + expect(out.headers.get('x-sample')).not.toContain(value); + expect(out.headers.get('x-other')).toBe(value); + expect(await out.text()).toBe(`{"note":"${value}"}`); + }); + + it('keeps the scope when the body is not screened', async () => { + const { protection } = await guard([rule('response.header.x-sample')]); + const out = await protection.screenResponse(new Response(new Uint8Array([0x89, 0x50, 0x00, 0x01]), { + headers: { 'content-type': 'image/png', 'x-sample': SAMPLE, 'x-other': SAMPLE }, + })); + + expect(out.headers.get('x-sample')).toBe('[REDACTED]'); + expect(out.headers.get('x-other')).toBe(SAMPLE); + }); +}); + +let close: (() => Promise) | null = null; +afterEach(async () => { + await close?.(); + close = null; +}); + +async function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + close = () => new Promise((resolve) => server.close(() => resolve())); + + return `http://127.0.0.1:${(server.address() as { port: number }).port}/`; +} + +async function rawGet(url: string): Promise<{ status: number; headers: Record; body: Buffer }> { + const { request } = await import('node:http'); + return new Promise((resolve, reject) => { + request(url, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: Buffer.concat(chunks) })); + }).on('error', reject).end(); + }); +} + +describe('redaction scope (Node)', () => { + async function serve(rules: object[], type: string, headers: Record, body: string | Buffer, flush: boolean) { + const g = await guard(rules); + const node = g.protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', type); + for (const [name, value] of Object.entries(headers)) res.setHeader(name, value); + if (flush) res.flushHeaders(); + res.end(body); + }), + ); + + return { ...g, got: await rawGet(url) }; + } + + it.each(cases)('$label', async ({ rules, body, headers, status, expectBody, expectHeaders }) => { + const { got } = await serve(rules, 'application/json', headers, body, false); + + expect(got.status).toBe(status); + expect(got.body.toString()).toBe(expectBody); + for (const [name, value] of Object.entries(expectHeaders)) expect(got.headers[name]).toBe(value); + }); + + it.each([ + ['an early flush', 'text/plain', 'plain text', true], + ['a body that is not screened', 'image/png', Buffer.from([0x89, 0x50, 0x00, 0x01]), false], + ])('keeps a header rule to its header on %s', async (_label, type, body, flush) => { + const { got, detections } = await serve([rule('response.header.x-sample')], type, { 'x-sample': SAMPLE, 'x-other': SAMPLE }, body, flush); + + expect(got.headers['x-sample']).toBe('[REDACTED]'); + expect(got.headers['x-other']).toBe(SAMPLE); + expect(detections).toHaveLength(1); + }); + + it('masks a header rule entry by entry on a multi-valued header', async () => { + const g = await guard([rule('response.header.set-cookie')]); + const node = g.protection.node({ screenResponses: true }); + const url = await listen((req, res) => + node(req, res, () => { + res.setHeader('content-type', 'application/json'); + res.setHeader('set-cookie', [`a=${SAMPLE}`, 'b=plain']); + res.setHeader('x-other', SAMPLE); + res.end(`{"note":"${SAMPLE}"}`); + }), + ); + const got = await rawGet(url); + + expect(got.headers['set-cookie']).toEqual(['a=[REDACTED]', 'b=plain']); + expect(got.headers['x-other']).toBe(SAMPLE); + expect(got.body.toString()).toBe(`{"note":"${SAMPLE}"}`); + }); +});