From 1e6e1ef42bbf4aacf6374a92d84c09f7ecd1b453 Mon Sep 17 00:00:00 2001 From: Dave Jong Date: Mon, 28 Sep 2026 16:24:26 +0200 Subject: [PATCH] Screen the scaffolded rule sets in the regex cost gate The regex cost gate now screens every regex the package ships: the compiled defaults plus the rule sets the installer scaffolds (`rules.json`, and `demo-rules.json` for `protect --demo`). Each clause has a declared worst case. Two demo patterns did not stay within the budget and are rewritten to stay linear with the same demo behaviour: - the response email pattern now has bounded parts and must start at the beginning of a local part; - the XXE pattern bounds the text between ` --- examples/protect/demo-rules.json | 4 +- scripts/regex-cost.mjs | 12 +++- src/protect/templates/demo-rules.json | 4 +- tests/protect/regex-cost.test.ts | 82 +++++++++++++++++++++++++-- 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/examples/protect/demo-rules.json b/examples/protect/demo-rules.json index cfeec61c..50dcfae4 100644 --- a/examples/protect/demo-rules.json +++ b/examples/protect/demo-rules.json @@ -91,7 +91,7 @@ "title": "XML external entity (XXE) in request body", "category": "xxe", "rule_v2": [ - { "parameter": "raw", "match": { "type": "regex", "value": "/]*SYSTEM/i" } } + { "parameter": "raw", "match": { "type": "regex", "value": "/]{0,256}SYSTEM/i" } } ], "_demo": { "desc": "External DOCTYPE/ENTITY referencing a local file", @@ -119,7 +119,7 @@ "category": "pii-exposure", "action": "redact", "rule_v2": [ - { "parameter": "response.body", "match": { "type": "regex", "value": "/[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\\.[A-Za-z0-9-]+){0,8}\\.[A-Za-z]{2,}/" } } + { "parameter": "response.body", "match": { "type": "regex", "value": "/(?]*SYSTEM/i" + "value": "/]{0,256}SYSTEM/i" } } ] @@ -195,7 +195,7 @@ "parameter": "response.body", "match": { "type": "regex", - "value": "/[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\\.[A-Za-z0-9-]+){0,8}\\.[A-Za-z]{2,}/" + "value": "/(? = { 'resp-stack-trace::rule_v2[0].match': { lead: ' at f (', fill: 'a:1:1' }, 'resp-sql-error::rule_v2[0].match': 'SQLSTATE[', 'resp-exception-trace::rule_v2[0].match': 'at a.b(c.java:1', + + // The scaffolded starter and demo sets. Each fill is a near-miss of one alternative, repeated. + 'ps-fallback-ssrf-internal::rule_v2[0].match': 'localhos ', + 'ps-fallback-xss::rule_v2[0].match': 'onerror ', + 'demo-sqli::rule_v2[0].match': 'union or 1 ', + 'demo-xss::rule_v2[0].match': 'onerror ', + 'demo-command-injection::rule_v2[0].match': 'a', + 'demo-nosql-injection::rule_v2[0].match': '"$ne" ', + // Declarations that never reach the keyword, each one a new place for the pattern to start. + 'demo-xxe::rule_v2[0].match': ' + JSON.parse(readFileSync(new URL(`../../src/protect/templates/${name}`, import.meta.url), 'utf8')).firewall; + +// Every rule the package ships. The example bundle is not read separately: `demo-rules.test.ts` holds it +// in lockstep with the scaffolded demo set. +const compiled = [ + ...DEFAULT_RESPONSE_RULES, + ...DEFAULT_EGRESS_RULES, + ...scaffolded('rules.json'), + ...scaffolded('demo-rules.json'), +] as any[]; const keyOf = (clause: { ruleId: string | null; path: string }) => `${clause.ruleId}::${clause.path}`; const clauses = compiled.flatMap((rule) => regexClausesOf(rule)); @@ -155,6 +195,23 @@ describe('the measurement itself', () => { } }); + it('measures the declared candidate alone when derivation is switched off', async () => { + // The derived candidate for this pattern matches it, which would invalidate the screening. + const pattern = '/(?:\\d[ -]?){13,16}/'; + const withDerived = await screenPatternCost(pattern, { candidate: '1-1-a', bytes: 4096, budgetMs: BUDGET_MS }); + const declaredOnly = await screenPatternCost(pattern, { candidate: '1-1-a', bytes: 4096, budgetMs: BUDGET_MS, derive: false }); + + expect(withDerived.screened).toBe(false); + expect(declaredOnly.screened, declaredOnly.reason).toBe(true); + expect(declaredOnly.measurements.map((m: any) => m.source)).toEqual(['declared']); + }, 30_000); + + it('refuses to switch off derivation without a declared candidate', async () => { + const result = await screenPatternCost('/abc/', { derive: false }); + expect(result.screened).toBe(false); + expect(result.reason).toContain('declared candidate'); + }); + it('reports a pattern it cannot compile rather than passing it', async () => { expect((await screenPatternCost('not a pattern')).screened).toBe(false); expect((await screenPatternCost('/[/')).screened).toBe(false); @@ -226,7 +283,7 @@ describe('finding the patterns a rule carries', () => { }); }); -describe('every regex the compiled rules carry, at the size it may be screened at', () => { +describe('every regex the shipped rules carry, at the size it may be screened at', () => { // The budget is generous — orders above what a linear pattern spends here — so this fails on a shape // change rather than on timing noise. it.each(clauses.map((clause) => [keyOf(clause), clause] as const))('%s', async (key, clause) => { @@ -234,6 +291,7 @@ describe('every regex the compiled rules carry, at the size it may be screened a candidate: WORST_CASE[key], bytes: CAP, budgetMs: BUDGET_MS, + derive: !DERIVED_MATCHES.has(key), }); expect(result.screened, result.reason).toBe(true); @@ -243,7 +301,7 @@ describe('every regex the compiled rules carry, at the size it may be screened a ).toBe(true); }, 30_000); - it('screens at least one regex from every compiled rule that carries any', () => { + it('screens at least one regex from every shipped rule that carries any', () => { // The set, not just the members: a rule whose patterns all went missing from the walk would leave // the cases above passing over a shorter list. const carrying = compiled.filter((rule) => regexClausesOf(rule).length > 0).map((rule) => rule.id); @@ -260,11 +318,23 @@ describe('every regex the compiled rules carry, at the size it may be screened a expect(missing, 'these compiled regex clauses have no declared worst case').toEqual([]); }); - it('declares a worst case for nothing that is not compiled in', () => { + it('declares a worst case for nothing that is not shipped', () => { // The other direction: a stale entry leaves a real clause uncovered while the count looks right. const live = new Set(clauses.map(keyOf)); const stale = Object.keys(WORST_CASE).filter((key) => !live.has(key)); expect(stale, 'these worst cases name no compiled regex clause').toEqual([]); }); + + it('screens without the derived candidate only where it matches', async () => { + const live = new Set(clauses.map(keyOf)); + for (const key of DERIVED_MATCHES) { + expect(live.has(key), `${key} names no shipped regex clause`).toBe(true); + const clause = clauses.find((c) => keyOf(c) === key)!; + const derived = deriveCandidate(clause.pattern, 4096); + expect(derived, `${key} derives no candidate at all`).not.toBeNull(); + const measured = (await measurePatternCost(clause.pattern, derived!, { timeoutMs: BUDGET_MS })) as any; + expect(measured.matched, `${key}'s derived candidate does not match, so it should be measured`).toBe(true); + } + }, 30_000); });