[ENG-4059] Resolve request fields the way applications read them

src/protect/engine/cookies.js

@@ -0,0 +1,36 @@
+import { setOwn } from './own.js';
+
+/**
+ * A `Cookie` header as name → value.
+ *
+ * A value wrapped in double quotes loses them, as cookie parsers strip them before an application reads
+ * the value. A name sent more than once keeps every value, as an array in the order sent: parsers differ
+ * on whether the first or the last one wins, so a rule on that cookie inspects all of them.
+ */
+export function parseCookieHeader(header) {
+  const cookies = {};
+  if (typeof header !== 'string' || header === '') return cookies;
+
+  const repeated = new Map();
+  for (const pair of header.split(';')) {
+    const eq = pair.indexOf('=');
+    if (eq === -1) continue;
+    const name = pair.slice(0, eq).trim();
+    if (name === '') continue;
+    let value = pair.slice(eq + 1).trim();
+    if (value.length >= 2 && value[0] === '"' && value[value.length - 1] === '"') value = value.slice(1, -1);
+
+    if (!Object.hasOwn(cookies, name)) {
+      setOwn(cookies, name, value);
+    } else {
+      let values = repeated.get(name);
+      if (values === undefined) {
+        values = [cookies[name]];
+        repeated.set(name, values);
+        setOwn(cookies, name, values);
+      }
+      values.push(value);
+    }
+  }
+  return cookies;
+}

src/protect/engine/engine.js

@@ -514,26 +514,31 @@ const WHOLE_VALUE_MATCH_TYPES = new Set(['isset', 'array_in_array', 'array_key_v
 // Iteratively collect every scalar (non-object) leaf of a structured value. Iterative + bounded
 // (depth and node caps) so a pathologically deep/large attacker payload STOPS at the bound rather
 // than throwing a RangeError that the per-rule catch would swallow into a fail-open bypass.
+// `truncated` says the bound was reached: containers past it contributed no leaves.
 function collectLeafValues(root, nodeCap = 20000, maxDepth = 1000) {
-  const out = [];
+  const leaves = [];
   const stack = [[root, 0]];
   let visited = 0;
+  let truncated = false;
   while (stack.length) {
     const [node, depth] = stack.pop();
     if (node === null || node === undefined) continue;
     if (typeof node !== 'object') {
-      out.push(node);
+      leaves.push(node);
+      continue;
+    }
+    if (depth >= maxDepth || visited >= nodeCap) {
+      truncated = true;
       continue;
     }
-    if (depth >= maxDepth || visited >= nodeCap) continue;
     visited++;
     if (Array.isArray(node)) {
       for (let i = node.length - 1; i >= 0; i--) stack.push([node[i], depth + 1]);
     } else {
       for (const k of Object.keys(node)) stack.push([node[k], depth + 1]);
     }
   }
-  return out;
+  return { leaves, truncated };
 }
 
 // The inspection limits one evaluation reached, as `{ skips: [reason, …] }`, or nothing when it
@@ -543,6 +548,16 @@ function skipsOf(resolver) {
   return skips && skips.length > 0 ? { skips } : {};
 }
 
+// The whole value as text, for a container too large to walk leaf by leaf. `undefined` when it has
+// no text form (a cycle, or nesting deeper than the serialiser allows).
+function serialisedValue(value) {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return undefined;
+  }
+}
+
 // Emit a warning at most once per distinct key (keeps a persistent misconfiguration from spamming).
 const warnedKeys = new Set();
 function warnOnce(key, message) {
@@ -585,16 +600,34 @@ function warnUnsupportedMatchType(type) {
  * value that is already a bare host passes through untouched, which is what keeps the egress path and
  * the built-in default rule behaving exactly as before.
  */
+// Schemes a URL parser treats as hierarchical whatever follows the colon: `http:x`, `http:/x` and
+// `http:\\x` all name host `x`.
+const SPECIAL_SCHEMES = new Set(['http', 'https', 'ws', 'wss', 'ftp', 'file']);
+
+// Strip C0 controls and spaces (U+0000–U+0020) from both ends, in linear time.
+function trimControls(text) {
+  let start = 0;
+  let end = text.length;
+  while (start < end && text.charCodeAt(start) <= 0x20) start++;
+  while (end > start && text.charCodeAt(end - 1) <= 0x20) end--;
+  return text.slice(start, end);
+}
+
 function hostFromValue(value) {
-  const raw = String(value ?? '').trim();
+  // A URL parser drops tabs and newlines anywhere, and C0 controls and spaces at either end, before it
+  // reads anything else — so they are dropped here too, or they would hide the scheme.
+  const raw = trimControls(String(value ?? '').replace(/[\t\n\r]/g, '')).trim();
   if (raw === '') return '';
 
-  // A scheme (`http://`, and deliberately any other) or a protocol-relative URL. Parsing rather than
-  // string-slicing is what makes the userinfo evasion (`http://trusted@169.254.169.254/`) resolve to the
-  // host actually contacted, and keeps `http://evil.com#@127.0.0.1` resolving to evil.com.
-  if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw) || raw.startsWith('//')) {
+  // A scheme (`http://`, and deliberately any other), a special scheme in any of the shorter spellings a
+  // URL parser still resolves to a host, or a protocol-relative URL (either slash direction). Parsing
+  // rather than string-slicing is what makes the userinfo evasion (`http://trusted@169.254.169.254/`)
+  // resolve to the host actually contacted, and keeps `http://evil.com#@127.0.0.1` resolving to evil.com.
+  const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(raw)?.[1]?.toLowerCase();
+  const relative = /^[\\/]{2}/.test(raw);
+  if (relative || /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) || (scheme !== undefined && SPECIAL_SCHEMES.has(scheme))) {
     try {
-      return new URL(raw.startsWith('//') ? `http:${raw}` : raw).hostname;
+      return new URL(relative ? `http:${raw}` : raw).hostname;
     } catch {
       // Unparseable: hand the raw value on, where the host check rejects it rather than guessing.
       return raw;
@@ -1328,9 +1361,17 @@ export class RuleEngine {
         if (WHOLE_VALUE_MATCH_TYPES.has(match.type)) {
           if (matchValue(match.type, value, match.value, match)) return true;
         } else {
-          for (const leaf of collectLeafValues(value)) {
+          const { leaves, truncated } = collectLeafValues(value);
+          for (const leaf of leaves) {
             if (matchValue(match.type, leaf, match.value, match)) return true;
           }
+          // Past the walk's bound, the rest of the value is matched as its serialised text, and the
+          // bound is reported: the leaves beyond it were not inspected individually.
+          if (truncated) {
+            resolver.noteSkip('container-cap');
+            const text = serialisedValue(value);
+            if (text !== undefined && matchValue(match.type, text, match.value, match)) return true;
+          }
         }
         continue;
       }

src/protect/engine/fetch.js

@@ -9,6 +9,7 @@
 import { resolveClientIp } from '../client-ip.js';
 import { RuleEngine } from './engine.js';
 import { notify } from '../notify.js';
+import { parseCookieHeader } from './cookies.js';
 import { appendOwn, setOwn } from './own.js';
 
 // Cap how much request body we buffer for inspection. A larger body is left UNSCANNED
@@ -67,7 +68,7 @@ export async function fromFetchRequest(request, options = {}) {
     // that guessed differently would attribute one request to two addresses.
     ip: client.ip ?? '',
     _clientIp: client,
-    cookies: parseCookies(headers.cookie),
+    cookies: parseCookieHeader(headers.cookie),
     // Verbatim body text: preserves literal keys (e.g. `__proto__`) that JSON.stringify
     // drops, so prototype-pollution rules on `raw` are robust.
     _rawBody: rawBody,
@@ -272,20 +273,6 @@ function decodeExtendedValue(value) {
   }
 }
 
-function parseCookies(header) {
-  const cookies = {};
-  if (!header) {
-    return cookies;
-  }
-  for (const pair of header.split(';')) {
-    const idx = pair.indexOf('=');
-    if (idx === -1) {
-      continue;
-    }
-    setOwn(cookies, pair.slice(0, idx).trim(), pair.slice(idx + 1).trim());
-  }
-  return cookies;
-}
 
 function defaultBlockResponse(result) {
   return new Response(
@@ -319,6 +306,9 @@ export function createFetchMiddleware(rulesData, options = {}) {
         notify(options.onSkip, { phase: 'request', reason: req._bodyInspectionSkip }, 'onSkip');
       }
       result = engine.evaluate(req);
+      for (const reason of result.skips ?? []) {
+        notify(options.onSkip, { phase: 'request', reason }, 'onSkip');
+      }
     } catch (err) {
       notify(options.onError, err, 'onError');
       return null; // fail open

src/protect/engine/index.d.ts

@@ -124,6 +124,7 @@ export declare function normalizeRequest(req: any, options?: NormalizeOptions):
   query: Record<string, any>;
   body: Record<string, any>;
   headers: Record<string, string>;
+  cookies: Record<string, any>;
   url: string;
   originalUrl: string;
   _rawBody: string;

src/protect/engine/node.js

@@ -10,6 +10,7 @@ import { resolveClientIp } from '../client-ip.js';
 import { RuleEngine } from './engine.js';
 import { parseBody } from './fetch.js';
 import { notify } from '../notify.js';
+import { parseCookieHeader } from './cookies.js';
 import { appendOwn, setOwn } from './own.js';
 
 // Build the engine's request shape from a Node IncomingMessage + its raw body text.
@@ -70,26 +71,12 @@ export function fromNodeRequest(req, rawBody = '', options = {}) {
     headers,
     ip: client.ip ?? '',
     _clientIp: client,
-    cookies: parseCookies(headers.cookie),
+    cookies: parseCookieHeader(headers.cookie),
     // Verbatim body text: preserves literal keys (e.g. `__proto__`) that JSON.stringify drops.
     _rawBody: rawBody
   };
 }
 
-function parseCookies(header) {
-  const cookies = {};
-  if (!header) {
-    return cookies;
-  }
-  for (const pair of header.split(';')) {
-    const idx = pair.indexOf('=');
-    if (idx === -1) {
-      continue;
-    }
-    setOwn(cookies, pair.slice(0, idx).trim(), pair.slice(idx + 1).trim());
-  }
-  return cookies;
-}
 
 function defaultBlock(res, result) {
   res.statusCode = 403;

src/protect/engine/normalizer.js

@@ -6,6 +6,7 @@
  * obfuscation techniques that attackers use to evade pattern matching.
  */
 import { setOwn } from './own.js';
+import { parseCookieHeader } from './cookies.js';
 
 
 const HTML_ENTITIES = {
@@ -388,16 +389,29 @@ export function normalizeRequest(req, options = {}) {
         ? req._rawBody
         : serializeForRawDetection(body ?? null);
 
+    const headers = requestField(req, 'headers') || {};
+
     return {
         query: normalizeObject(requestField(req, 'query') || {}, options),
         body: normalizeObject(body || {}, options),
-        headers: normalizeObject(requestField(req, 'headers') || {}, options),
+        headers: normalizeObject(headers, options),
+        // Cookies are split into name/value pairs BEFORE their values are normalised, and normalised the
+        // same way whether a framework parsed them or they come from the header — so a rule on a cookie
+        // sees one value, independent of which cookie parser (if any) ran first.
+        cookies: normalizeObject(cookieValues(requestField(req, 'cookies'), headers), options),
         url: normalize(decodeQueryPlus(url || ''), options),
         originalUrl: normalize(decodeQueryPlus(requestField(req, 'originalUrl') || url || ''), options),
         _rawBody: rawBody
     };
 }
 
+// The request's cookies as name → value: the ones a cookie parser already produced when there are any,
+// otherwise the `Cookie` header's pairs.
+function cookieValues(parsed, headers) {
+    if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed;
+    return parseCookieHeader(headers?.cookie);
+}
+
 // The same depth bound as the engine's leaf walk, so every value that walk reaches is normalized. Past
 // it a sub-value is kept as it is (still matched, in its raw form) and `options.onLimit` is called.
 const MAX_NORMALIZE_DEPTH = 1000;