Skip to content

Add Connect tunnel retry jitter and request-correlated diagnostics #2064

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jshph wants to merge 1 commit into
base: main
Choose a base branch
from
+354 −56
Open

Add Connect tunnel retry jitter and request-correlated diagnostics #2064

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 99 additions & 1 deletion apps/connect/src/worker.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ const mockHandleAssignMachineLabel = vi.mocked(handleAssignMachineLabel);
/** A resolved server row; overrides let a test tweak one field. */
function resolvedServer(
over: Partial<{
credentialHash: string | null;
lastSeenAt: Date | null;
userId: string;
}> = {},
Expand All @@ -189,7 +190,8 @@ function resolvedServer(
userId: over.userId ?? OWNER,
server: {
id: "srv1",
credentialHash: "abc",
credentialHash:
over.credentialHash === undefined ? "abc" : over.credentialHash,
revokedAt: null,
lastSeenAt: over.lastSeenAt ?? null,
},
Expand Down Expand Up @@ -426,6 +428,102 @@ describe("gate tunnel authentication", () => {
expect(captured[0].headers.get("x-bb-cloud-dev-host")).toBeNull();
});

it("returns a request-correlated 500 when Durable Object dispatch fails", async () => {
const credential = "bbcred_server_secret";
const digest = await crypto.subtle.digest(
"SHA-256",
new TextEncoder().encode(credential),
);
const hash = [...new Uint8Array(digest)]
.map((byte) => byte.toString(16).padStart(2, "0"))
.join("");
mockResolveLabel.mockResolvedValue(
resolvedServer({ credentialHash: hash }),
);
const upstreamError = Object.assign(new Error("DO unavailable"), {
remote: true,
retryable: true,
overloaded: false,
});
const { env, ctx, captured } = makeEnv(() => Promise.reject(upstreamError));
const errorLog = vi.spyOn(console, "error").mockImplementation(() => {});

try {
const response = await worker.fetch(
visitorRequest("sawyer.getbb.app", "/__tunnel?v=1", {
headers: {
authorization: `Bearer ${credential}`,
"cf-ray": "incident-ray",
upgrade: "websocket",
},
}),
env as never,
ctx,
);

expect(response.status).toBe(500);
const requestId = response.headers.get("x-bb-request-id");
expect(requestId).toMatch(/^[0-9a-f-]{36}$/u);
expect(await response.text()).toContain(`request ${requestId}`);
expect(captured[0].headers.get("x-bb-request-id")).toBe(requestId);
const logLine = String(errorLog.mock.calls[0]?.[0]);
expect(JSON.parse(logLine)).toMatchObject({
event: "tunnel_dial_failed",
requestId,
cfRay: "incident-ray",
label: "sawyer",
ownerKind: "server",
ownerId: "srv1",
stage: "durable_object_dispatch",
errorName: "Error",
errorMessage: "DO unavailable",
remote: true,
retryable: true,
overloaded: false,
});
expect(logLine).not.toContain(credential);
} finally {
errorLog.mockRestore();
}
});

it("correlates a fresh D1 resolution failure without dispatching to the DO", async () => {
mockResolveLabel.mockRejectedValue(new Error("D1 unavailable"));
const { env, ctx, captured } = makeEnv(() => new Response("origin"));
const errorLog = vi.spyOn(console, "error").mockImplementation(() => {});

try {
const response = await worker.fetch(
visitorRequest("sawyer.getbb.app", "/__tunnel?v=1", {
headers: {
authorization: "Bearer not-logged",
"cf-ray": "d1-ray",
upgrade: "websocket",
},
}),
env as never,
ctx,
);

expect(response.status).toBe(500);
expect(captured).toHaveLength(0);
const requestId = response.headers.get("x-bb-request-id");
expect(JSON.parse(String(errorLog.mock.calls[0]?.[0]))).toMatchObject({
event: "tunnel_dial_failed",
requestId,
cfRay: "d1-ray",
label: "sawyer",
ownerKind: null,
ownerId: null,
stage: "resolve_label",
errorMessage: "D1 unavailable",
});
expect(String(errorLog.mock.calls[0]?.[0])).not.toContain("not-logged");
} finally {
errorLog.mockRestore();
}
});

it("dials immediately after a negative resolve and label assignment", async () => {
const credential = "bbcm_new_machine";
const digest = await crypto.subtle.digest(
Expand Down
134 changes: 113 additions & 21 deletions apps/connect/src/worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
parseCookie,
markMachineSeen,
resolveLabel,
type ResolvedLabel,
verifyMachineCredentialDetails,
verifySessionCookie,
} from "./session.js";
Expand Down Expand Up @@ -55,6 +56,63 @@ function text(body: string, status: number): Response {
});
}

const TUNNEL_REQUEST_ID_HEADER = "x-bb-request-id";

type TunnelDialFailureStage =
| "resolve_label"
| "credential_hash"
| "durable_object_binding"
| "durable_object_dispatch";

function errorBoolean(error: unknown, property: string): boolean | null {
if (typeof error !== "object" || error === null) return null;
try {
const value = Reflect.get(error, property);
return typeof value === "boolean" ? value : null;
} catch {
return null;
}
}

/** Log a tunnel failure without including the Authorization credential. */
function tunnelDialFailure(args: {
request: Request;
requestId: string;
label: string;
ownerKind: "server" | "machine" | null;
ownerId: string | null;
stage: TunnelDialFailureStage;
error: unknown;
}): Response {
console.error(
JSON.stringify({
event: "tunnel_dial_failed",
requestId: args.requestId,
cfRay: args.request.headers.get("cf-ray"),
label: args.label,
ownerKind: args.ownerKind,
ownerId: args.ownerId,
stage: args.stage,
errorName: args.error instanceof Error ? args.error.name : null,
errorMessage:
args.error instanceof Error ? args.error.message : String(args.error),
remote: errorBoolean(args.error, "remote"),
retryable: errorBoolean(args.error, "retryable"),
overloaded: errorBoolean(args.error, "overloaded"),
}),
);
return new Response(
`bb connect: tunnel handshake failed (request ${args.requestId})\n`,
{
status: 500,
headers: {
"content-type": "text/plain; charset=utf-8",
[TUNNEL_REQUEST_ID_HEADER]: args.requestId,
},
},
);
}

// Matches the bb dashboard's visual language (Inter, --canvas/--ink tokens,
// dark primary button, bb logo) since this plain worker can't bundle React.
export function dashboardSignInUrl(appUrl: string, returnTo: string): string {
Expand Down Expand Up @@ -334,21 +392,36 @@ export default {
// avoids both stale credentials and a cached negative immediately after a
// machine label is assigned.
const isTunnelDial = url.pathname === "/__tunnel";
const resolved = await resolveLabel(
label,
db,
isTunnelDial ? { fresh: true } : undefined,
);
const tunnelRequestId = isTunnelDial ? crypto.randomUUID() : null;
let resolved: ResolvedLabel | null;
try {
resolved = await resolveLabel(
label,
db,
isTunnelDial ? { fresh: true } : undefined,
);
} catch (error) {
if (tunnelRequestId === null) throw error;
return tunnelDialFailure({
request,
requestId: tunnelRequestId,
label,
ownerKind: null,
ownerId: null,
stage: "resolve_label",
error,
});
}
if (!resolved) return text(`bb connect: no server for "${label}"\n`, 404);

// Server routing stays exactly as on main (the bare label). Machine labels
// are new and use ownership-generation identity from their first dial.
const routingKey =
resolved.kind === "machine" ? resolved.routingKey : label;
const stub = env.TUNNEL_DO.get(env.TUNNEL_DO.idFromName(routingKey));

// Tunnel client connection — bare label only (share hosts are visitor-facing).
if (url.pathname === "/__tunnel") {
const requestId = tunnelRequestId ?? crypto.randomUUID();
if (target !== null) return text("bb connect: not found\n", 404);
const auth = request.headers.get("authorization") ?? "";
const credential = auth.startsWith("Bearer ") ? auth.slice(7) : "";
Expand All @@ -362,24 +435,43 @@ export default {
403,
);
}
if ((await sha256Hex(credential)) !== owner.credentialHash) {
return text("bb connect: invalid credential\n", 401);
}
const forward = new URL(request.url);
forward.searchParams.delete("serverId");
forward.searchParams.delete("machineId");
if (resolved.kind === "server") {
forward.searchParams.set("serverId", owner.id);
} else {
forward.searchParams.set("machineId", owner.id);
let stage: TunnelDialFailureStage = "credential_hash";
try {
if ((await sha256Hex(credential)) !== owner.credentialHash) {
return text("bb connect: invalid credential\n", 401);
}
const forward = new URL(request.url);
forward.searchParams.delete("serverId");
forward.searchParams.delete("machineId");
if (resolved.kind === "server") {
forward.searchParams.set("serverId", owner.id);
} else {
forward.searchParams.set("machineId", owner.id);
}
const headers = new Headers(request.headers);
stripCloudDevHeader(headers);
headers.set(TUNNEL_REQUEST_ID_HEADER, requestId);
stage = "durable_object_binding";
const stub = env.TUNNEL_DO.get(env.TUNNEL_DO.idFromName(routingKey));
stage = "durable_object_dispatch";
return await stub.fetch(
new Request(new Request(forward, request), { headers }),
);
} catch (error) {
return tunnelDialFailure({
request,
requestId,
label,
ownerKind: resolved.kind,
ownerId: owner.id,
stage,
error,
});
}
const headers = new Headers(request.headers);
stripCloudDevHeader(headers);
return stub.fetch(
new Request(new Request(forward, request), { headers }),
);
}

const stub = env.TUNNEL_DO.get(env.TUNNEL_DO.idFromName(routingKey));

// Reserve the /__ namespace: never proxy internal paths from outside.
if (url.pathname.startsWith("/__"))
return text("bb connect: not found\n", 404);
Expand Down
Loading
Loading