[ENG-4057] Classify more non-public addresses and keep fetch options across redirects

src/protect/egress.js

@@ -196,22 +196,26 @@ export async function installEgressGuard({ shouldBlock, onBlock, onSkip, dnsScre
       const headers = new Headers(cur.headers);
       const signal = cur.signal;
       const replay = captureReplayBody(cur);
+      // Every hop is sent with the caller's own options (a dispatcher, keepalive, referrer policy, …).
+      // Only what this loop owns per hop is replaced; the body is consumed by the first hop's Request.
+      const { body: _callerBody, ...callerOptions } = init || {};
       let body;
 
       for (let hop = 0; ; hop++) {
         let resp;
         try {
-          resp = await originalFetch(
-            hop === 0 ? cur : new Request(url, { method, headers, body, redirect: 'manual', signal }),
-          );
+          resp =
+            hop === 0
+              ? await originalFetch(cur, { ...callerOptions, redirect: 'manual' })
+              : await originalFetch(url, { ...callerOptions, method, headers, body, redirect: 'manual', signal });
         } catch (error) {
           replay.cancel();
           throw error;
         }
         const location = REDIRECT_STATUSES.has(resp.status) ? resp.headers.get('location') : null;
         if (!location) {
           replay.cancel();
-          return resp;
+          return hop === 0 ? resp : markRedirected(resp);
         }
         await discardResponseBody(resp);
         if (hop >= MAX_REDIRECTS) {
@@ -303,6 +307,17 @@ export async function installEgressGuard({ shouldBlock, onBlock, onSkip, dnsScre
   };
 }
 
+// A response reached by following redirects reports it, as native `follow` would. The final hop was
+// itself fetched with `redirect: 'manual'`, so its own flag reads false.
+function markRedirected(response) {
+  try {
+    Object.defineProperty(response, 'redirected', { value: true, configurable: true });
+  } catch {
+    /* a response that cannot take the property is returned as it is */
+  }
+  return response;
+}
+
 // Wrap http(s).request/get — and, on node:http, the ClientRequest constructor they build — so a
 // blocked destination throws before the socket opens.
 function patchHttpModule(http, block, screen, skip) {

src/protect/engine/engine.js

@@ -562,7 +562,8 @@ function warnUnsupportedMatchType(type) {
 // match type. It CANONICALIZES the host before classifying —
 // a textual/prefix check is bypassable by alternate encodings (decimal/hex/octal IPv4, expanded or
 // IPv4-mapped IPv6), which is a classic SSRF evasion. Handles localhost / *.local / GCP metadata
-// names, every IPv4 spelling inet_aton accepts, and IPv6 loopback/link-local/unique-local/mapped.
+// names, every IPv4 spelling inet_aton accepts, and IPv6 loopback/link-local/site-local/unique-local/
+// multicast plus the transition forms that carry a v4 destination (mapped, NAT64, 6to4).
 /**
  * The host to classify out of a rule parameter's value.
  *
@@ -626,11 +627,22 @@ function isInternalHost(hostname) {
     const allZeroHi = g[0] === 0 && g[1] === 0 && g[2] === 0 && g[3] === 0 && g[4] === 0;
     if (allZeroHi && g[5] === 0 && g[6] === 0 && (g[7] === 0 || g[7] === 1)) return true; // ::, ::1 loopback
     if ((g[0] & 0xffc0) === 0xfe80) return true; // link-local fe80::/10
+    if ((g[0] & 0xffc0) === 0xfec0) return true; // site-local fec0::/10 (deprecated, still routed locally)
     if ((g[0] & 0xfe00) === 0xfc00) return true; // unique-local fc00::/7
+    if ((g[0] & 0xff00) === 0xff00) return true; // multicast ff00::/8
+    const low32 = (((g[6] << 16) >>> 0) | g[7]) >>> 0;
     if (allZeroHi && (g[5] === 0xffff || g[5] === 0)) {
       // IPv4-mapped (::ffff:a.b.c.d) / IPv4-compatible (::a.b.c.d) — classify the embedded v4.
-      return isPrivateV4Int((((g[6] << 16) >>> 0) | g[7]) >>> 0);
+      return isPrivateV4Int(low32);
     }
+    // NAT64 well-known prefix 64:ff9b::/96 — a translator forwards to the embedded v4, so classify that.
+    if (g[0] === 0x64 && g[1] === 0xff9b && g[2] === 0 && g[3] === 0 && g[4] === 0 && g[5] === 0) {
+      return isPrivateV4Int(low32);
+    }
+    // NAT64 local-use prefix 64:ff9b:1::/48 — operator-defined translation, never a public destination.
+    if (g[0] === 0x64 && g[1] === 0xff9b && g[2] === 1) return true;
+    // 6to4 2002::/16 — the relay forwards to the v4 address in the next 32 bits.
+    if (g[0] === 0x2002) return isPrivateV4Int((((g[1] << 16) >>> 0) | g[2]) >>> 0);
     return false;
   }
 
@@ -640,10 +652,15 @@ function isInternalHost(hostname) {
   return false;
 }
 
-// Private / loopback / link-local / this-host / CGNAT test on a 32-bit IPv4 integer.
+// Non-public test on a 32-bit IPv4 integer: private, loopback, link-local, this-host, CGNAT, the IETF
+// protocol block, benchmarking, multicast, and reserved space (incl. limited broadcast).
 function isPrivateV4Int(n) {
   const a = (n >>> 24) & 0xff;
   const b = (n >>> 16) & 0xff;
+  const c = (n >>> 8) & 0xff;
+  if (a >= 224) return true; // 224.0.0.0/4 multicast, 240.0.0.0/4 reserved, 255.255.255.255 broadcast
+  if (a === 192 && b === 0 && c === 0) return true; // 192.0.0.0/24 IETF protocol assignments
+  if (a === 198 && (b === 18 || b === 19)) return true; // 198.18.0.0/15 benchmarking
   if (a === 127 || a === 10 || a === 0) return true; // loopback / private / this-host
   if (a === 169 && b === 254) return true; // link-local incl. 169.254.169.254 metadata
   if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12

tests/protect/egress-redirect-options.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { createServer } from 'node:http';
+import { createProtection } from '../../src/protect/runtime.js';
+
+// Following redirects on the caller's behalf must look like native `follow` from the outside: every
+// hop is sent with the caller's own fetch options, and the final response says it was redirected.
+
+type Call = { url: string; init: any };
+const originalFetch = globalThis.fetch;
+let protection: any;
+
+afterEach(() => {
+  protection?.uninstallEgress?.();
+  protection?.stop?.();
+  protection = undefined;
+  globalThis.fetch = originalFetch;
+});
+
+async function guardOver(responses: Response[]) {
+  const calls: Call[] = [];
+  globalThis.fetch = (async (input: any, init?: any) => {
+    calls.push({ url: typeof input === 'string' ? input : input.url, init });
+    return responses.shift() ?? new Response('unexpected');
+  }) as any;
+  protection = await createProtection({ egress: true, mode: 'block' });
+  return calls;
+}
+
+const START = 'http://203.0.113.10/start';
+
+describe('egress redirect following keeps the caller options', () => {
+  it('sends every hop with the caller options and reports the redirect', async () => {
+    const calls = await guardOver([
+      new Response(null, { status: 302, headers: { location: '/middle' } }),
+      new Response(null, { status: 301, headers: { location: 'http://203.0.113.11/end' } }),
+      new Response('done'),
+    ]);
+    const dispatcher = { name: 'caller-dispatcher' };
+    const response = await fetch(START, { dispatcher, keepalive: true, referrerPolicy: 'no-referrer' } as any);
+
+    expect(await response.text()).toBe('done');
+    expect(response.redirected).toBe(true);
+    expect(calls.map((c) => c.url)).toEqual([START, 'http://203.0.113.10/middle', 'http://203.0.113.11/end']);
+    for (const call of calls) {
+      expect(call.init).toMatchObject({ dispatcher, keepalive: true, referrerPolicy: 'no-referrer', redirect: 'manual' });
+    }
+  });
+
+  it('does not report a redirect for a direct response', async () => {
+    await guardOver([new Response('direct')]);
+    const response = await fetch(START, { keepalive: true });
+    expect(await response.text()).toBe('direct');
+    expect(response.redirected).toBe(false);
+  });
+
+  it('replays a 307 body alongside the caller options', async () => {
+    const calls = await guardOver([
+      new Response(null, { status: 307, headers: { location: '/again' } }),
+      new Response('ok'),
+    ]);
+    const dispatcher = { name: 'caller-dispatcher' };
+    await fetch(START, { method: 'POST', body: 'payload', dispatcher } as any);
+
+    expect(calls[1].init).toMatchObject({ method: 'POST', dispatcher, redirect: 'manual' });
+    expect(new TextDecoder().decode(calls[1].init.body)).toBe('payload');
+  });
+
+  it('still drops the body on a 303', async () => {
+    const calls = await guardOver([
+      new Response(null, { status: 303, headers: { location: '/see-other' } }),
+      new Response('ok'),
+    ]);
+    await fetch(START, { method: 'POST', body: 'payload', keepalive: true });
+    expect(calls[1].init).toMatchObject({ method: 'GET', keepalive: true });
+    expect(calls[1].init.body).toBeUndefined();
+  });
+
+  it('uses the caller dispatcher for every hop of a real redirect', async () => {
+    const server = createServer((req, res) => {
+      if (req.url === '/start') {
+        res.writeHead(302, { location: '/next' });
+        res.end();
+        return;
+      }
+      const chunks: Buffer[] = [];
+      req.on('data', (chunk) => chunks.push(chunk));
+      req.on('end', () => res.end(['final', req.url, Buffer.concat(chunks).toString()].filter(Boolean).join(' ')));
+    });
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+    try {
+      const { port } = server.address() as { port: number };
+      protection = await createProtection({ egress: true, mode: 'block', allowHosts: ['127.0.0.1'] });
+      const global = (globalThis as any)[Symbol.for('undici.globalDispatcher.1')];
+      const paths: string[] = [];
+      const dispatcher = {
+        dispatch: (options: any, handler: any) => {
+          paths.push(options.path);
+          return global.dispatch(options, handler);
+        },
+      };
+      const response = await fetch(`http://127.0.0.1:${port}/start`, { dispatcher } as any);
+      expect(await response.text()).toBe('final /next');
+      expect(response.redirected).toBe(true);
+      expect(paths).toEqual(['/start', '/next']);
+
+      // A streamed body belongs to the first hop's Request and is sent once.
+      const stream = new ReadableStream({
+        start(controller) {
+          controller.enqueue(new TextEncoder().encode('streamed'));
+          controller.close();
+        },
+      });
+      const posted = await fetch(`http://127.0.0.1:${port}/echo`, { method: 'POST', body: stream, duplex: 'half' } as any);
+      expect(await posted.text()).toBe('final /echo streamed');
+    } finally {
+      await new Promise<void>((resolve) => server.close(() => resolve()));
+    }
+  });
+});

tests/protect/internal-address-ranges.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from 'vitest';
+import { _testExports } from '../../src/protect/engine/engine.js';
+
+const { isInternalHost, matchValue } = _testExports as {
+  isInternalHost: (host: string) => boolean;
+  matchValue: (type: string, value: string, expected: unknown) => boolean;
+};
+
+// Each non-public range, next to the closest public neighbour, so a boundary that is off by one
+// in either direction fails a case.
+describe('internal_host classifies non-public address space', () => {
+  it.each([
+    ['198.18.0.1', '198.17.255.255'],
+    ['198.19.255.254', '198.20.0.1'],
+    ['192.0.0.170', '192.0.1.1'],
+    ['224.0.0.251', '223.255.255.255'],
+    ['239.255.255.250', '223.1.1.1'],
+    ['240.0.0.1', '223.255.255.254'],
+    ['255.255.255.255', '8.8.8.8'],
+  ])('IPv4 %s is internal, %s is not', (internal, external) => {
+    expect(isInternalHost(internal)).toBe(true);
+    expect(isInternalHost(external)).toBe(false);
+  });
+
+  it.each([
+    ['fec0::1', 'fe00::1'],
+    ['feff::1', 'fe7f::1'],
+    ['ff02::1', 'fe7f:ffff::1'],
+    ['64:ff9b::a9fe:a9fe', '64:ff9b::808:808'],
+    ['64:ff9b::10.0.0.1', '64:ff9b::1.1.1.1'],
+    ['64:ff9b:1::1', '64:ff9c::a9fe:a9fe'],
+    ['2002:7f00:1::', '2002:808:808::'],
+    ['2002:c0a8:101::1', '2003:c0a8:101::1'],
+    ['[2002:a9fe:a9fe::]', '[2002:0101:0101::]'],
+  ])('IPv6 %s is internal, %s is not', (internal, external) => {
+    expect(isInternalHost(internal)).toBe(true);
+    expect(isInternalHost(external)).toBe(false);
+  });
+
+  it('applies to a full URL parameter on the request phase', () => {
+    expect(matchValue('internal_host', 'http://[64:ff9b::a9fe:a9fe]/latest/', null)).toBe(true);
+    expect(matchValue('internal_host', 'http://198.18.0.1:8080/', null)).toBe(true);
+    expect(matchValue('internal_host', 'http://[64:ff9b::808:808]/', null)).toBe(false);
+  });
+});