diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index db1273d8..b01e572d 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -317,7 +317,9 @@ export function humanClosedCode(text: string): string | null { /** * Closed exit codes of the in-container probe scripts. Every script failure * exits with one of them, so a failing `hack exec` that exits 1 is the product - * (or the runtime), never an uncaught probe error. + * (or the runtime), never an uncaught probe error. `rejected` and `timeout` + * are the HTTP phase (connect or response); the `dns*` codes are name + * resolution, so a probe timeout names which half stalled (HACK-1419). */ export const NATIVE_NETWORK_PROBE_EXIT = { environment: 23, @@ -327,18 +329,212 @@ export const NATIVE_NETWORK_PROBE_EXIT = { timeout: 27, unreadable: 28, unwritable: 29, + dnsRejected: 30, + dnsTimeout: 31, } as const; const PROBE = NATIVE_NETWORK_PROBE_EXIT; +/** The peer probe's per-URL budget: name resolution plus the HTTP exchange share it. */ +export const NATIVE_NETWORK_PEER_PROBE_BUDGET_MS = 4000; -/** Fetches the endpoint origin and its alias; both must answer the exact marker. */ +/** + * Resolves, then fetches, the endpoint origin and its alias; both must answer + * the exact marker. Each URL has one budget shared by resolution and the HTTP + * exchange, and the HTTP request goes to the address just resolved, so a + * failure is attributed to exactly one phase. Before a failing exit after the + * environment check it writes one closed stderr line (see + * `nativeNetworkFixtureProbeEvidence`): URL label, phase, resolution and total + * milliseconds, and the resolved address count. Never an address or body. + */ export function nativeNetworkFixturePeerProbe(opts: { readonly origin: string; readonly alias: string; readonly expected: string; readonly timeoutMs?: number; }): string { - const timeout = opts.timeoutMs ?? 4000; - return `if(process.env.PEER_ORIGIN!==${JSON.stringify(opts.origin)})process.exit(${PROBE.environment});for(const url of [process.env.PEER_ORIGIN,${JSON.stringify(opts.alias)}]){let ok=false;try{const r=await fetch(url,{signal:AbortSignal.timeout(${timeout})});ok=r.ok&&await r.text()===${JSON.stringify(opts.expected)}}catch(e){process.exit(e?.name==="TimeoutError"?${PROBE.timeout}:${PROBE.rejected})}if(!ok)process.exit(${PROBE.response})}`; + const timeout = opts.timeoutMs ?? NATIVE_NETWORK_PEER_PROBE_BUDGET_MS; + return [ + `if(process.env.PEER_ORIGIN!==${JSON.stringify(opts.origin)})process.exit(${PROBE.environment});`, + "const ms=(s)=>Math.round(performance.now()-s);", + "const fail=(u,p,d,s,n,c)=>{console.error(`probe-evidence ${u} ${p} ${d} ${ms(s)} ${n}`);process.exit(c)};", + `for(const [u,url] of [["origin",process.env.PEER_ORIGIN],["alias",${JSON.stringify(opts.alias)}]]){`, + "const s=performance.now();const target=new URL(url);let a=[];let t;", + `try{a=await Promise.race([Bun.dns.lookup(target.hostname),new Promise((_,j)=>{t=setTimeout(()=>j({name:"TimeoutError"}),${timeout})})])}catch(e){fail(u,"dns",ms(s),s,0,e?.name==="TimeoutError"?${PROBE.dnsTimeout}:${PROBE.dnsRejected})}finally{clearTimeout(t)}`, + `const d=ms(s);const x=a.find((r)=>r.family===4)??a[0];if(!x)fail(u,"dns",d,s,0,${PROBE.dnsRejected});`, + "target.hostname=x.family===6?`[${x.address}]`:x.address;let ok=false;", + `try{const r=await fetch(target,{signal:AbortSignal.timeout(Math.max(1,${timeout}-d))});ok=r.ok&&await r.text()===${JSON.stringify(opts.expected)}}catch(e){fail(u,"http",d,s,a.length,e?.name==="TimeoutError"?${PROBE.timeout}:${PROBE.rejected})}`, + `if(!ok)fail(u,"http",d,s,a.length,${PROBE.response})}`, + ].join(""); +} + +const PROBE_EVIDENCE = + /^probe-evidence (origin|alias) (dns|http) (\d{1,7}) (\d{1,7}) (\d{1,3})$/m; + +/** The peer probe's closed stderr evidence; null for anything else. */ +export function nativeNetworkFixtureProbeEvidence(stderr: string): { + readonly url: "origin" | "alias"; + readonly phase: "dns" | "http"; + readonly dnsMs: number; + readonly elapsedMs: number; + readonly addresses: number; +} | null { + const match = PROBE_EVIDENCE.exec(stderr); + if (!match) { + return null; + } + const [, url, phase, dnsMs, elapsedMs, addresses] = match; + return { + url: url === "alias" ? "alias" : "origin", + phase: phase === "dns" ? "dns" : "http", + dnsMs: Number(dnsMs), + elapsedMs: Number(elapsedMs), + addresses: Number(addresses), + }; +} + +const HEALTH_STATES = ["starting", "healthy", "unhealthy"] as const; +const IPV4 = /^(?:\d{1,3}\.){3}\d{1,3}$/; + +function attachments( + value: unknown +): readonly { readonly id: string; readonly ip: string }[] { + const settings = isRecord(value) ? own(value, "NetworkSettings") : undefined; + const networks = isRecord(settings) ? own(settings, "Networks") : undefined; + if (!isRecord(networks)) { + return []; + } + return Object.values(networks).flatMap((entry) => { + const id = isRecord(entry) ? own(entry, "NetworkID") : undefined; + const ip = isRecord(entry) ? own(entry, "IPAddress") : undefined; + return typeof id === "string" && ID.test(id) + ? [{ id, ip: typeof ip === "string" ? ip : "" }] + : []; + }); +} + +function containerState(value: unknown): { + readonly state: string; + readonly health: string; +} { + const state = isRecord(value) ? own(value, "State") : undefined; + if (!isRecord(state)) { + return { state: "unknown", health: "unknown" }; + } + const health = own(state, "Health"); + const status = isRecord(health) ? own(health, "Status") : undefined; + return { + state: + CONTAINER_STATES.find((entry) => entry === own(state, "Status")) ?? + "unknown", + health: + health === undefined || health === null + ? "none" + : (HEALTH_STATES.find((entry) => entry === status) ?? "unknown"), + }; +} + +/** + * Value-free route state behind a failed peer probe, from the two containers' + * engine inspections, the source container's `getent ahostsv4` answer for the + * probed name (empty when the name does not resolve, null when the lookup + * could not run) and the ingress network ID: + * closed states, attachment counts, shared bridges, ingress attachment, how + * many addresses the name resolved to and whether every one is the target's + * address on a bridge both containers share. Never an ID, address or name. + */ +export function nativeNetworkFixtureProbeRoute(opts: { + readonly source: unknown; + readonly target: unknown; + readonly resolved: string | null; + readonly ingressId: string | null; +}) { + const source = attachments(opts.source); + const target = attachments(opts.target); + const sourceIds = new Set(source.map((entry) => entry.id)); + const shared = target.filter((entry) => sourceIds.has(entry.id)); + const addresses = + opts.resolved === null + ? null + : [ + ...new Set( + opts.resolved + .split("\n") + .map((line) => line.trim().split(/\s+/)[0] ?? "") + .filter((address) => IPV4.test(address)) + ), + ]; + const reachable = new Set(shared.map((entry) => entry.ip)); + return { + source: containerState(opts.source), + target: containerState(opts.target), + sourceAttachments: source.length, + targetAttachments: target.length, + sharedBridges: shared.length, + sourceIngress: opts.ingressId !== null && sourceIds.has(opts.ingressId), + addresses: addresses === null ? null : addresses.length, + resolvesToSharedTarget: + addresses === null + ? null + : addresses.length > 0 && + addresses.every((address) => reachable.has(address)), + }; +} + +/** Outcome label of a probe exit: pass, a closed probe code name, or other (the product or runtime). */ +function probeOutcome(exitCode: number): string { + if (exitCode === 0) { + return "pass"; + } + return ( + Object.entries(PROBE).find(([, code]) => code === exitCode)?.[0] ?? "other" + ); +} + +/** + * Failed peer probe evidence: checkout role, probing service and target + * (fixed fixture names), the first attempt's exit, outcome and closed probe + * evidence, one diagnostic-only rerun's outcome (it never turns the step + * green) and the value-free route state. Never argv, output text or values. + */ +export function nativeNetworkFixtureProbeDiagnostic(opts: { + readonly role: FixtureRole; + readonly service: string; + readonly target: string; + readonly first: Pick; + readonly rerun: Pick | null; + readonly route: ReturnType | null; +}): string { + const attempt = ( + result: Pick + ) => ({ + exit: result.exitCode, + timedOut: result.timedOut, + outcome: probeOutcome(result.exitCode), + evidence: nativeNetworkFixtureProbeEvidence(result.stderr), + }); + if ( + !( + NATIVE_NETWORK_FIXTURE_ROLES.includes(opts.role) && + NAMES.includes(opts.service as Name) && + NAMES.includes(opts.target as Name) && + Number.isSafeInteger(opts.first.exitCode) && + typeof opts.first.timedOut === "boolean" && + typeof opts.first.stderr === "string" && + (opts.rerun === null || + (Number.isSafeInteger(opts.rerun.exitCode) && + typeof opts.rerun.timedOut === "boolean" && + typeof opts.rerun.stderr === "string")) + ) + ) { + return "Network probe diagnostic unavailable"; + } + return `Network probe diagnostic ${JSON.stringify({ + role: opts.role, + service: opts.service, + target: opts.target, + ...attempt(opts.first), + rerun: opts.rerun ? attempt(opts.rerun) : null, + route: opts.route, + })}`; } /** Reads the retained marker; a missing or unreadable file has its own code. */ @@ -1849,6 +2045,89 @@ export const nativeConfigNetworksScenario: Scenario = { ): Promise => { ok(await raw(checkout, ["exec", service, "--", "bun", "-e", script])); }; + // A failed peer probe logs closed probe evidence, the value-free route + // state and one diagnostic-only rerun before the step fails (HACK-1419). + const probeDiagnostic = async (opts: { + readonly checkout: Checkout; + readonly service: Name; + readonly target: Name; + readonly script: string; + readonly first: CliResult; + }): Promise => { + try { + const project = (await saved(opts.checkout.root)).identity + .composeProject; + const container = (service: Name) => `${project}-${service}-1`; + const inspected = async (service: Name): Promise => { + const result = await execute([ + engine, + "container", + "inspect", + container(service), + ]); + if (result.exitCode !== 0) { + return null; + } + const value: unknown = JSON.parse(result.stdout); + return Array.isArray(value) && value.length === 1 ? value[0] : null; + }; + // The probe writes to stderr; accept the closed line on either stream. + const streams = (result: CliResult) => ({ + ...result, + stderr: `${result.stderr}\n${result.stdout}`, + }); + const first = streams(opts.first); + const evidence = nativeNetworkFixtureProbeEvidence(first.stderr); + const lookup = await execute([ + engine, + "exec", + container(opts.service), + "getent", + "ahostsv4", + evidence?.url === "alias" ? `${opts.target}-alias` : opts.target, + ]); + const ingress = await execute([ + engine, + "network", + "inspect", + "--format", + "{{.Id}}", + "hack-dev", + ]); + const route = nativeNetworkFixtureProbeRoute({ + source: await inspected(opts.service), + target: await inspected(opts.target), + // getent exits 2 when the name does not resolve; other failures are unknown. + resolved: + lookup.exitCode === 0 + ? lookup.stdout + : lookup.exitCode === 2 + ? "" + : null, + ingressId: ingress.exitCode === 0 ? ingress.stdout.trim() : null, + }); + const rerun = await raw(opts.checkout, [ + "exec", + opts.service, + "--", + "bun", + "-e", + opts.script, + ]); + ctx.log( + nativeNetworkFixtureProbeDiagnostic({ + role: roles.get(opts.checkout.root) ?? "fixture", + service: opts.service, + target: opts.target, + first, + rerun: streams(rerun), + route, + }) + ); + } catch { + ctx.log("Network probe diagnostic unavailable"); + } + }; const markers = async (checkout: Checkout): Promise => { for (const [service, target, alias] of [ ["web", "peer", "peer-alias"], @@ -1856,15 +2135,23 @@ export const nativeConfigNetworksScenario: Scenario = { ["vault", "reader", "reader-alias"], ["reader", "vault", "vault-alias"], ] as const) { - await exec( - checkout, + const script = nativeNetworkFixturePeerProbe({ + origin: `http://${target}:3000`, + alias: `http://${alias}:3000/`, + expected: `${checkout.marker}:${target}`, + }); + const first = await raw(checkout, [ + "exec", service, - nativeNetworkFixturePeerProbe({ - origin: `http://${target}:3000`, - alias: `http://${alias}:3000/`, - expected: `${checkout.marker}:${target}`, - }) - ); + "--", + "bun", + "-e", + script, + ]); + if (first.exitCode !== 0 || first.timedOut) { + await probeDiagnostic({ checkout, service, target, script, first }); + } + ok(first); } await exec( checkout, diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index 62bf795c..433324a8 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -33,6 +33,9 @@ import { nativeNetworkFixtureNetworkMatches, nativeNetworkFixturePeerProbe, nativeNetworkFixturePreservedContainerMatches, + nativeNetworkFixtureProbeDiagnostic, + nativeNetworkFixtureProbeEvidence, + nativeNetworkFixtureProbeRoute, nativeNetworkFixtureProjectKind, nativeNetworkFixtureProtocolMatches, nativeNetworkFixtureRefusalDiagnostic, @@ -406,6 +409,285 @@ test("probe scripts exit with a closed code for every in-container failure", asy } }); +// HACK-1419: a hosted probe exited 27 with no way to tell which URL or phase stalled. +test("peer probe attributes a failure to one URL and phase in closed stderr evidence", async () => { + const root = await mkdtemp(join(tmpdir(), "hack-network-probe-")); + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + return new URL(request.url).pathname === "/hang" + ? new Promise(() => undefined) + : new Response("primary:peer"); + }, + }); + const closed = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response(""), + }); + const refused = `http://127.0.0.1:${closed.port}`; + closed.stop(true); + const origin = `http://127.0.0.1:${server.port}`; + const run = async (opts: { + readonly origin: string; + readonly alias: string; + readonly timeoutMs: number; + }): Promise<{ readonly code: number; readonly stderr: string }> => { + const child = Bun.spawn( + [ + process.execPath, + "-e", + nativeNetworkFixturePeerProbe({ ...opts, expected: "primary:peer" }), + ], + { + cwd: root, + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + PEER_ORIGIN: opts.origin, + }, + stdout: "ignore", + stderr: "pipe", + } + ); + const [code, stderr] = await Promise.all([ + child.exited, + new Response(child.stderr).text(), + ]); + return { code, stderr }; + }; + try { + // A name resolving to several families still reaches the IPv4 listener. + const named = `http://localhost:${server.port}`; + expect( + await run({ origin: named, alias: `${named}/a`, timeoutMs: 4000 }) + ).toEqual({ code: 0, stderr: "" }); + + const hung = await run({ origin, alias: `${origin}/hang`, timeoutMs: 300 }); + expect(hung.code).toBe(NATIVE_NETWORK_PROBE_EXIT.timeout); + const hungEvidence = nativeNetworkFixtureProbeEvidence(hung.stderr); + expect(hungEvidence).toMatchObject({ + url: "alias", + phase: "http", + addresses: 1, + }); + // Resolution and the HTTP exchange share one budget per URL. + expect(hungEvidence?.elapsedMs).toBeGreaterThanOrEqual(250); + expect(hungEvidence?.elapsedMs).toBeLessThan(2000); + + const rejected = await run({ + origin: refused, + alias: `${origin}/a`, + timeoutMs: 4000, + }); + expect(rejected.code).toBe(NATIVE_NETWORK_PROBE_EXIT.rejected); + expect(nativeNetworkFixtureProbeEvidence(rejected.stderr)).toMatchObject({ + url: "origin", + phase: "http", + }); + + // RFC 6761 `.invalid` never resolves; an unreachable resolver times out instead. + const unresolved = await run({ + origin, + alias: "http://probe-1419.invalid:3000/", + timeoutMs: 2000, + }); + expect([ + NATIVE_NETWORK_PROBE_EXIT.dnsRejected, + NATIVE_NETWORK_PROBE_EXIT.dnsTimeout, + ]).toContain(unresolved.code); + expect(nativeNetworkFixtureProbeEvidence(unresolved.stderr)).toMatchObject({ + url: "alias", + phase: "dns", + addresses: 0, + }); + expect(unresolved.stderr).not.toContain("probe-1419"); + + expect(new Set(Object.values(NATIVE_NETWORK_PROBE_EXIT)).size).toBe( + Object.values(NATIVE_NETWORK_PROBE_EXIT).length + ); + } finally { + server.stop(true); + await rm(root, { recursive: true, force: true }); + } +}); + +test("probe evidence admits only the closed stderr line", () => { + const secret = "/private/fixture/secret-path"; + expect( + nativeNetworkFixtureProbeEvidence( + `warning ${secret}\nprobe-evidence origin dns 4001 4003 0\n` + ) + ).toEqual({ + url: "origin", + phase: "dns", + dnsMs: 4001, + elapsedMs: 4003, + addresses: 0, + }); + for (const text of [ + "", + secret, + "probe-evidence other http 1 2 1", + "probe-evidence alias connect 1 2 1", + `probe-evidence alias http 1 2 1 ${secret}`, + "probe-evidence alias http -1 2 1", + "probe-evidence alias http 1 2 1000", + ]) { + expect(nativeNetworkFixtureProbeEvidence(text)).toBeNull(); + } +}); + +test("probe route state reduces inspections to closed counts and booleans", () => { + const outbound = "a".repeat(64); + const ingress = "b".repeat(64); + const foreign = "c".repeat(64); + const source = { + State: { Status: "running", Health: { Status: "healthy" } }, + NetworkSettings: { + Networks: { + project_outbound: { NetworkID: outbound, IPAddress: "10.9.0.2" }, + "hack-dev": { NetworkID: ingress, IPAddress: "10.8.0.5" }, + }, + }, + }; + const target = { + State: { Status: "running", Health: { Status: "starting" } }, + NetworkSettings: { + Networks: { + project_outbound: { NetworkID: outbound, IPAddress: "10.9.0.3" }, + other: { NetworkID: foreign, IPAddress: "10.7.0.3" }, + }, + }, + }; + const shared = nativeNetworkFixtureProbeRoute({ + source, + target, + resolved: + "10.9.0.3 STREAM peer\n10.9.0.3 DGRAM\n10.9.0.3 RAW\n", + ingressId: ingress, + }); + expect(shared).toEqual({ + source: { state: "running", health: "healthy" }, + target: { state: "running", health: "starting" }, + sourceAttachments: 2, + targetAttachments: 2, + sharedBridges: 1, + sourceIngress: true, + addresses: 1, + resolvesToSharedTarget: true, + }); + for (const value of [outbound, ingress, foreign, "10.9.0", "hack-dev"]) { + expect(JSON.stringify(shared)).not.toContain(value); + } + // A name answering an address off the shared bridge (another network, a + // stale container) is the misroute a timeout would hide. + expect( + nativeNetworkFixtureProbeRoute({ + source, + target, + resolved: "10.9.0.3 STREAM peer\n10.7.0.3 STREAM peer\n", + ingressId: null, + }) + ).toMatchObject({ + addresses: 2, + resolvesToSharedTarget: false, + sourceIngress: false, + }); + expect( + nativeNetworkFixtureProbeRoute({ + source, + target, + resolved: "", + ingressId: ingress, + }) + ).toMatchObject({ addresses: 0, resolvesToSharedTarget: false }); + expect( + nativeNetworkFixtureProbeRoute({ + source: null, + target: { State: { Status: "secret" } }, + resolved: null, + ingressId: ingress, + }) + ).toEqual({ + source: { state: "unknown", health: "unknown" }, + target: { state: "unknown", health: "none" }, + sourceAttachments: 0, + targetAttachments: 0, + sharedBridges: 0, + sourceIngress: false, + addresses: null, + resolvesToSharedTarget: null, + }); +}); + +test("probe diagnostic names role, pair, outcomes and route state only", () => { + const secret = "/private/fixture/secret-path"; + const route = nativeNetworkFixtureProbeRoute({ + source: null, + target: null, + resolved: null, + ingressId: null, + }); + const text = nativeNetworkFixtureProbeDiagnostic({ + role: "primary", + service: "web", + target: "peer", + first: { + exitCode: NATIVE_NETWORK_PROBE_EXIT.timeout, + timedOut: false, + stderr: `${secret}\nprobe-evidence alias http 2 4001 1\n`, + }, + rerun: { exitCode: 0, timedOut: false, stderr: "" }, + route, + }); + expect(text).toBe( + `Network probe diagnostic ${JSON.stringify({ + role: "primary", + service: "web", + target: "peer", + exit: 27, + timedOut: false, + outcome: "timeout", + evidence: { + url: "alias", + phase: "http", + dnsMs: 2, + elapsedMs: 4001, + addresses: 1, + }, + rerun: { + exit: 0, + timedOut: false, + outcome: "pass", + evidence: null, + }, + route, + })}` + ); + expect(text).not.toContain(secret); + expect( + nativeNetworkFixtureProbeDiagnostic({ + role: "sibling-a", + service: "reader", + target: "vault", + first: { exitCode: 1, timedOut: false, stderr: secret }, + rerun: null, + route: null, + }) + ).toContain('"exit":1,"timedOut":false,"outcome":"other","evidence":null'); + expect( + nativeNetworkFixtureProbeDiagnostic({ + role: "primary", + service: secret, + target: "peer", + first: { exitCode: 27, timedOut: false, stderr: "" }, + rerun: null, + route: null, + }) + ).toBe("Network probe diagnostic unavailable"); +}); + test("saved-state evidence reports closed flags and pending operation only", () => { const secret = "secret-generation-token"; expect(