From a15b6bd75971b2026b3c1206eecaa898cd45a029 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 20:04:59 +0330 Subject: [PATCH 01/14] server: prepare database queries and batch dialog owner locks --- server/BENCHMARKING.md | 25 +++- server/src/__tests__/performance/catalog.ts | 6 +- .../performance/wire.postgres.test.ts | 9 +- server/src/db/connectionPolicy.ts | 4 +- server/src/db/preparedAccessQueries.ts | 22 ---- .../src/db/preparedQueries.postgres.test.ts | 121 ++++++++++++++++++ server/src/db/preparedQueries.ts | 68 ++++++++++ .../src/modules/dialogOpen.postgres.test.ts | 62 +++++++++ server/src/modules/dialogOpen.ts | 13 +- 9 files changed, 291 insertions(+), 39 deletions(-) delete mode 100644 server/src/db/preparedAccessQueries.ts create mode 100644 server/src/db/preparedQueries.postgres.test.ts create mode 100644 server/src/db/preparedQueries.ts create mode 100644 server/src/modules/dialogOpen.postgres.test.ts diff --git a/server/BENCHMARKING.md b/server/BENCHMARKING.md index ea235783b..3eecef699 100644 --- a/server/BENCHMARKING.md +++ b/server/BENCHMARKING.md @@ -123,10 +123,12 @@ Do not disable domain jobs to make a path pass its budget. `catalog.ts` holds reviewed maximum command counts. They represent current costs, including costly linear fanout; they are not desired targets. Lower them with a proven optimization. Do not automatically regenerate budgets or loosen one to -hide a regression. The 1/10/100 cases expose scaling: current public sends cost -`32 + recipients`, reply sends `66 + 7 × recipients`, and user update batches -`2 + 4 × updates` for these fixtures. Chat-list queries remain constant at 11 -while result volume grows. These are local fixture observations, not production +hide a regression. The 1/10/100 cases expose scaling: the reviewed public-send budget is now constant at +`32` after batching sidebar owner locks, reply sends retain a conservative +`66 + 7 × recipients` bound, and user update batches +`2 + 4 × updates` for these fixtures. Unchanged dialog batches cost 5 commands at 1/10/100 owners. Chat-list +command budgets include their snapshot and admission overhead while remaining +constant as result volume grows. These are local fixture observations, not production traffic statistics. `test:backend` also selects the existing send, chat-list, history, message integrity, @@ -191,3 +193,18 @@ matrix, runtime/driver/database/platform/CPU/pool settings, warmups and sample counts. They report saved commands, exchanges and milliseconds separately, along with before/after RTT calibration. A comparison is evidence to inspect, not an automatic correctness proof or a promise of production latency savings. + +## Prepared query policy + +Ordinary Drizzle postgres-js calls now opt into preparation for bounded scalar +SQL/type signatures, including transactions, nested savepoints and reserved +clients. Warm prepared calls avoid the extra Describe/Flush exchange; result +rows and authorization decisions are never cached. Explicit per-query choices +are preserved. After 256 admitted signatures, previously seen shapes remain +prepared and new shapes execute unprepared. Queries longer than 32 KiB, more +than 301 parameters, and custom typed/array parameters retain their prior mode. +PgBouncer must support protocol-level prepared statements. The pool's existing +prepare setting remains authoritative, including an explicit global opt-out. + +The real-wire regression tests distinguish SQL submissions from protocol +exchanges and cover new bind values, nested rollback and bounded admission. diff --git a/server/src/__tests__/performance/catalog.ts b/server/src/__tests__/performance/catalog.ts index 618d3d191..3807a0c8b 100644 --- a/server/src/__tests__/performance/catalog.ts +++ b/server/src/__tests__/performance/catalog.ts @@ -18,10 +18,10 @@ const snapshotCommands = 3 const catalogAdmissionCommands = 2 const historySnapshotChatCommands = 1 export const scenarios: readonly ScenarioSpec[] = [ - { id: "send.dm.open", kind: "sendDm", size: 1, maxCommands: 26 }, + { id: "send.dm.open", kind: "sendDm", size: 1, maxCommands: 25 }, { id: "send.dm.closed", kind: "sendDm", size: 1, variant: "closed", maxCommands: 42 }, { id: "send.dm.retry", kind: "sendDm", size: 1, variant: "retry", maxCommands: 7 }, - ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `send.public.${size}`, kind: "sendThread", size, maxCommands: 32 + size })), + ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `send.public.${size}`, kind: "sendThread", size, maxCommands: 32 })), ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `send.reply.${size}`, kind: "sendThread", size, variant: "reply", maxCommands: 66 + 7 * size })), ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `getChats.${size}`, kind: "getChats", size, maxCommands: 11 + snapshotCommands + catalogAdmissionCommands })), { id: "history.50", kind: "history", size: 50, maxCommands: 7 + snapshotCommands + historySnapshotChatCommands }, @@ -29,7 +29,7 @@ export const scenarios: readonly ScenarioSpec[] = [ { id: "updates.replay.message", kind: "replay", size: 1, maxCommands: 23 }, { id: "updates.checkpoint", kind: "checkpoint", size: 1, maxCommands: 6 }, ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `updates.enqueue.${size}`, kind: "enqueue", size, maxCommands: 4 * size + 2 })), - ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `dialogs.noop.${size}`, kind: "dialog", size, maxCommands: size + 5 })), + ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `dialogs.noop.${size}`, kind: "dialog", size, maxCommands: 5 })), { id: "read.advance", kind: "read", size: 1, maxCommands: 12 }, { id: "read.noop", kind: "read", size: 1, variant: "noop", maxCommands: 5 }, ] diff --git a/server/src/__tests__/performance/wire.postgres.test.ts b/server/src/__tests__/performance/wire.postgres.test.ts index f4a0518a6..db4ad7cd8 100644 --- a/server/src/__tests__/performance/wire.postgres.test.ts +++ b/server/src/__tests__/performance/wire.postgres.test.ts @@ -23,11 +23,10 @@ test("real Drizzle queries expose protocol exchanges independently of SQL comman }, async () => {}, proxy) expect(sample.sql.commands).toBe(1) expect(sample.sql.catalogCommands).toBe(0) - // Drizzle currently calls unsafe() without per-query prepare:true. Its - // Describe/Flush and Bind/Execute/Sync produce two exchange boundaries. - // A future driver optimization may reduce this to one, never to zero. - expect(sample.wire!.exchangeBoundaries).toBeGreaterThanOrEqual(1) - expect(sample.wire!.exchangeBoundaries).toBeLessThanOrEqual(2) + // Warm parameterized Drizzle reads must skip Describe/Flush, not merely + // keep the SQL command count unchanged. + expect(sample.wire!.exchangeBoundaries).toBe(1) + expect(sample.wire!.frames["H"] ?? 0).toBe(0) expect(sample.wire!.frames["S"]).toBe(1) expect(sample.wire!.clientBytes).toBeGreaterThan(0) expect(sample.wire!.serverBytes).toBeGreaterThan(0) diff --git a/server/src/db/connectionPolicy.ts b/server/src/db/connectionPolicy.ts index b2e920b2c..52103b6b8 100644 --- a/server/src/db/connectionPolicy.ts +++ b/server/src/db/connectionPolicy.ts @@ -1,5 +1,5 @@ import postgres from "postgres" -import { prepareAccessQueries } from "./preparedAccessQueries" +import { prepareQueries } from "./preparedQueries" export const QUERY_TIMEOUTS = { statement_timeout: 30_000, @@ -155,7 +155,7 @@ export const makeDatabaseClients = (databaseUrl: string, environment: DatabaseEn max: 1, connect_timeout: 2, idle_timeout: 30, connection: { application_name: "inline-health", ...HEALTH_TIMEOUTS }, }) - prepareAccessQueries(queryClient) + prepareQueries(queryClient) const close = () => Promise.all([queryClient.end({ timeout: 5 }), healthClient.end({ timeout: 5 })]).then(() => {}) const queryTimeoutSettings = () => queryClient` select name, setting, unit from pg_settings diff --git a/server/src/db/preparedAccessQueries.ts b/server/src/db/preparedAccessQueries.ts deleted file mode 100644 index 5431123a9..000000000 --- a/server/src/db/preparedAccessQueries.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type postgres from "postgres" -import { accessQueryComment } from "./databaseDiagnostics" - -const comments = [accessQueryComment("sync.replay"), accessQueryComment("sync.sidecars")] - -/** - * Drizzle's postgres-js adapter uses unsafe(), which disables preparation even - * when the client enables it. Opt in only user replay's access queries on the - * shared client: at most 100 chat IDs repeated in three clauses plus one user. - * The two sources therefore produce at most 200 statement shapes per client. - * Larger batches and transaction-owned clients retain their existing behavior. - * This caches plans, never authorization results. PgBouncer must support - * protocol-level prepared statements (production max_prepared_statements=200). - */ -export function prepareAccessQueries(client: postgres.Sql): void { - const unsafe = client.unsafe - client.unsafe = (query, parameters, options) => unsafe(query, parameters, - options?.prepare === undefined && parameters !== undefined && parameters.length <= 301 && - comments.some((comment) => query.trimEnd().endsWith(comment)) - ? { ...options, prepare: true } - : options) -} diff --git a/server/src/db/preparedQueries.postgres.test.ts b/server/src/db/preparedQueries.postgres.test.ts new file mode 100644 index 000000000..e017abee9 --- /dev/null +++ b/server/src/db/preparedQueries.postgres.test.ts @@ -0,0 +1,121 @@ +import { expect, test } from "bun:test" +import postgres from "postgres" +import { drizzle } from "drizzle-orm/postgres-js" +import { sql } from "drizzle-orm" +import { setupTestLifecycle } from "../__tests__/database" +import { createWireProxy } from "../__tests__/performance/wire" +import { MAX_PREPARED_QUERY_SHAPES, prepareQueries } from "./preparedQueries" + +setupTestLifecycle() + +async function withClient(run: (client: postgres.Sql, proxy: Awaited>) => Promise) { + const url = new URL(process.env["DATABASE_URL"]!) + const proxy = await createWireProxy({ host: url.hostname, port: Number(url.port || 5432) }, 0) + url.hostname = "127.0.0.1" + url.port = String(proxy.port) + const client = postgres(url.toString(), { max: 1, ssl: false, fetch_types: false }) + prepareQueries(client) + prepareQueries(client) // Installing twice must not stack wrappers or reset the shape budget. + try { + await client`select 1` + await run(client, proxy) + } finally { + await client.end() + await proxy.close() + } +} + +test("Drizzle reuses plans with new values in the pool, transactions and savepoints", async () => { + await withClient(async (client, proxy) => { + const db = drizzle(client) + const read = async (value: number) => { + const rows = await db.execute(sql`select ${value}::int as value`) + expect(rows[0]?.["value"]).toBe(value) + } + await read(1) + proxy.reset() + await read(2) + expect(proxy.snapshot().frames["H"] ?? 0).toBe(0) + expect(proxy.snapshot().frames["P"] ?? 0).toBe(0) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + + const nestedRead = (value: number) => db.transaction(async (tx) => { + const rows = await tx.execute(sql`select ${value}::int + 10 as value`) + expect(rows[0]?.["value"]).toBe(value + 10) + await tx.transaction(async (nested) => { + const inner = await nested.execute(sql`select ${value}::int + 20 as value`) + expect(inner[0]?.["value"]).toBe(value + 20) + }) + }, { isolationLevel: "repeatable read", accessMode: "read only" }) + await nestedRead(3) + proxy.reset() + await nestedRead(4) + expect(proxy.snapshot().frames["H"] ?? 0).toBe(0) + expect(proxy.snapshot().frames["P"] ?? 0).toBe(0) + + const reserved = await client.reserve() + try { + await reserved.unsafe("select $1::int + 30 as value", [5]) + proxy.reset() + const rows = await reserved.unsafe("select $1::int + 30 as value", [6]) + expect(rows[0]?.["value"]).toBe(36) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + } finally { reserved.release() } + }) +}) + +test("prepared writes preserve savepoint and outer transaction rollback", async () => { + await withClient(async (client) => { + const db = drizzle(client) + await db.execute(sql`create temporary table prepared_write_test (id integer primary key)`) + await db.transaction(async (tx) => { + await tx.execute(sql`insert into prepared_write_test values (${1})`) + await expect(tx.transaction(async (nested) => { + await nested.execute(sql`insert into prepared_write_test values (${2})`) + throw new Error("rollback inner") + })).rejects.toThrow("rollback inner") + await tx.execute(sql`insert into prepared_write_test values (${3})`) + }) + await expect(db.transaction(async (tx) => { + await tx.execute(sql`insert into prepared_write_test values (${4})`) + throw new Error("rollback outer") + })).rejects.toThrow("rollback outer") + const rows = await db.execute(sql`select id from prepared_write_test order by id`) + expect(rows.map((row) => row["id"])).toEqual([1, 3]) + }) +}) + +test("explicit opt-out and shape-budget overflow still execute correctly without retaining new plans", async () => { + await withClient(async (client, proxy) => { + const query = "select $1::int as value" + await client.unsafe(query, [1]) + proxy.reset() + expect((await client.unsafe(query, [2], { prepare: false }))[0]?.["value"]).toBe(2) + expect(proxy.snapshot().frames["H"]).toBe(1) + for (let i = 1; i < MAX_PREPARED_QUERY_SHAPES; i++) { + await client.unsafe(`select $1::int + ${i} as value`, [1]) + } + const overflow = `select $1::int + ${MAX_PREPARED_QUERY_SHAPES} as value` + await client.unsafe(overflow, [1]) + proxy.reset() + expect((await client.unsafe(overflow, [2]))[0]?.["value"]).toBe(MAX_PREPARED_QUERY_SHAPES + 2) + expect(proxy.snapshot().frames["H"]).toBe(1) + proxy.reset() + expect((await client.unsafe(query, [3]))[0]?.["value"]).toBe(3) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + }) +}) + + +test("nullable scalar type changes and custom typed parameters keep their original values", async () => { + await withClient(async (client, proxy) => { + for (const value of [true, null, false, null]) { + expect((await client.unsafe("select $1::boolean as value", [value]))[0]?.["value"]).toBe(value) + } + const typed = () => client.unsafe("select $1::int as typed", [client.typed(7, 23)]) + await typed() + proxy.reset() + expect((await typed())[0]?.["typed"]).toBe(7) + expect(proxy.snapshot().frames["H"]).toBe(1) + }) +}) diff --git a/server/src/db/preparedQueries.ts b/server/src/db/preparedQueries.ts new file mode 100644 index 000000000..791342732 --- /dev/null +++ b/server/src/db/preparedQueries.ts @@ -0,0 +1,68 @@ +import type postgres from "postgres" + +// Bound automatically retained SQL/type signatures per pool. Once full, new shapes keep +// postgres.js's unprepared behavior; existing hot shapes still reuse their plans. +export const MAX_PREPARED_QUERY_SHAPES = 256 +const MAX_QUERY_LENGTH = 32_768 +const MAX_PARAMETERS = 301 +const installed = new WeakSet() +type TransactionCallback = (sql: postgres.TransactionSql) => T | Promise + +/** Drizzle uses unsafe(), whose call-level default disables preparation even + * when the pool enables it. Apply the same bounded policy to borrowed/transaction + * clients, including savepoints. Cache plans only, never rows or authority. + * Requires protocol-level prepared statement support when using PgBouncer. */ +export function prepareQueries(client: postgres.Sql): void { + const shapes = new Set() + const wrap = (scope: T): T => { + if (installed.has(scope)) return scope + installed.add(scope) + const unsafe = scope.unsafe + scope.unsafe = (query, parameters, options) => { + if (options?.prepare !== undefined || !parameters?.length || parameters.length > MAX_PARAMETERS || + query.length > MAX_QUERY_LENGTH || !/^\s*(select|insert|update|delete|with)\b/i.test(query)) { + return unsafe(query, parameters, options) + } + // Match the driver's ordinary scalar type categories. Custom typed/array + // parameters retain their original execution path rather than admitting + // unbounded type variants under one SQL-text entry. + const types = parameters.map(parameterType) + if (types.some((type) => type === undefined)) return unsafe(query, parameters, options) + const signature = `${types.join(",")}\0${query}` + const prepare = shapes.has(signature) || shapes.size < MAX_PREPARED_QUERY_SHAPES + if (prepare) shapes.add(signature) + return unsafe(query, parameters, { ...options, prepare }) + } + if (typeof scope.begin === "function") { + const begin = scope.begin + scope.begin = (first: string | TransactionCallback, callback?: TransactionCallback) => { + if (typeof first === "function") return begin((tx) => first(wrap(tx))) + if (!callback) throw new Error("Missing transaction callback") + return begin(first, (tx) => callback(wrap(tx))) + } + } + if ("savepoint" in scope && typeof scope.savepoint === "function") { + const savepoint = scope.savepoint as postgres.TransactionSql["savepoint"] + scope.savepoint = (first: string | TransactionCallback, callback?: TransactionCallback) => { + if (typeof first === "function") return savepoint((tx) => first(wrap(tx))) + if (!callback) throw new Error("Missing savepoint callback") + return savepoint(first, (tx) => callback(wrap(tx))) + } + } + if (typeof scope.reserve === "function") { + const reserve = scope.reserve + scope.reserve = async () => wrap(await reserve()) + } + return scope + } + wrap(client) +} + +function parameterType(value: unknown): number | undefined { + if (value instanceof Date) return 1184 + if (value instanceof Uint8Array) return 17 + if (typeof value === "boolean") return 16 + if (typeof value === "bigint") return 20 + if (value === null || typeof value === "string" || typeof value === "number") return 0 + return undefined +} diff --git a/server/src/modules/dialogOpen.postgres.test.ts b/server/src/modules/dialogOpen.postgres.test.ts new file mode 100644 index 000000000..7b37742bd --- /dev/null +++ b/server/src/modules/dialogOpen.postgres.test.ts @@ -0,0 +1,62 @@ +import { expect, test } from "bun:test" +import { asc, eq } from "drizzle-orm" +import { db } from "@in/server/db" +import { dialogs, users } from "@in/server/db/schema" +import { setupTestLifecycle, testUtils } from "../__tests__/setup" +import { measureOperation } from "../__tests__/performance/measure" +import { setDialogOpenForUsers } from "./dialogOpen" + +setupTestLifecycle() + +test("one hundred unchanged dialogs use a constant query budget and preserve their state", async () => { + const actors = await db.insert(users).values(Array.from({ length: 100 }, (_, i) => ({ email: `batch-${i}@example.test` }))).returning() + const chat = await testUtils.createChat(null) + if (!chat) throw new Error("Missing chat") + await db.insert(dialogs).values(actors.map((user) => ({ + userId: user.id, chatId: chat.id, open: true, order: "a0", archived: false, + }))) + const before = await db.select().from(dialogs).where(eq(dialogs.chatId, chat.id)).orderBy(asc(dialogs.userId)) + const sample = await measureOperation(db.$client.options, async () => { + const result = await setDialogOpenForUsers({ chat, userIds: actors.map((user) => user.id), open: true }) + expect(result.changedDialogs).toEqual([]) + expect(result.dialogs.toSorted((a, b) => a.userId - b.userId)).toEqual(before) + }, async () => {}) + // Active owners, BEGIN, one sorted owner-lock batch, existing dialogs, COMMIT. + expect(sample.sql.commands).toBe(5) + expect(await db.select().from(dialogs).where(eq(dialogs.chatId, chat.id)).orderBy(asc(dialogs.userId))).toEqual(before) +}) + +test("overlapping reversed owner batches keep independent chat allocations ordered without deadlock", async () => { + const actors = await db.insert(users).values(Array.from({ length: 10 }, (_, i) => ({ email: `overlap-${i}@example.test` }))).returning() + const first = await testUtils.createChat(null) + const second = await testUtils.createChat(null) + if (!first || !second) throw new Error("Missing chats") + const ids = actors.map((user) => user.id) + const results = await Promise.allSettled([ + setDialogOpenForUsers({ chat: first, userIds: [...ids, ids[0]!], open: true }), + setDialogOpenForUsers({ chat: second, userIds: ids.toReversed(), open: true }), + ]) + for (const result of results) { + if (result.status === "rejected") throw result.reason + expect(result.value.dialogs).toHaveLength(10) + expect(result.value.changedDialogs).toHaveLength(10) + } + for (const user of actors) { + const rows = await db.select().from(dialogs).where(eq(dialogs.userId, user.id)) + expect(rows).toHaveLength(2) + expect(rows.every((row) => row.open === true && row.order !== null)).toBe(true) + expect(new Set(rows.map((row) => row.order)).size).toBe(2) + } +}) + +test("closing metadata-only state returns persisted cleared fields even without a visibility change", async () => { + const actor = await testUtils.createUser() + const chat = await testUtils.createChat(null) + if (!chat) throw new Error("Missing chat") + await db.insert(dialogs).values({ userId: actor.id, chatId: chat.id, open: false, order: "a0" }) + const result = await setDialogOpenForUsers({ chat, userIds: [actor.id], open: false }) + expect(result.changedDialogs).toEqual([]) + expect(result.dialogs[0]?.order).toBeNull() + const repeated = await setDialogOpenForUsers({ chat, userIds: [actor.id], open: false }) + expect(repeated.dialogs).toEqual(result.dialogs) +}) diff --git a/server/src/modules/dialogOpen.ts b/server/src/modules/dialogOpen.ts index 0bbc5804d..6e16082c0 100644 --- a/server/src/modules/dialogOpen.ts +++ b/server/src/modules/dialogOpen.ts @@ -262,9 +262,10 @@ async function setDialogOpenForUsersInTransaction( // Acquire all user owners before touching any dialog rows. Projection // writers use the same users -> dialogs order, so overlapping batches do // not deadlock or derive a stale fractional order. - for (const userId of userIds) { - await tx.select({ id: users.id }).from(users).where(eq(users.id, userId)).for("no key update").limit(1) - } + await tx.select({ id: users.id }).from(users) + .where(inArray(users.id, userIds)) + .orderBy(asc(users.id)) + .for("no key update") const existingDialogs = await tx .select() @@ -289,6 +290,9 @@ async function setDialogOpenForUsersInTransaction( return showInChatList && dialog.chatListHidden === true }) const missingUserIds = userIds.filter((userId) => !existingUserIds.has(userId)) + if (dialogsToOpen.length === 0 && missingUserIds.length === 0) { + return { dialogs: existingDialogs, changedDialogs: [] } + } for (const dialog of dialogsToOpen) { const placement = input.openPlacementByUserId?.get(dialog.userId) ?? defaultDialogOpenPlacement @@ -346,6 +350,9 @@ async function setDialogOpenForUsersInTransaction( .filter((dialog) => effectiveDialogOpenForDialog(dialog) || dialog.folderId != null) .map((dialog) => dialog.userId) const missingUserIds = userIds.filter((userId) => !existingUserIds.has(userId)) + if (persistCloseUserIds.length === 0 && missingUserIds.length === 0) { + return { dialogs: existingDialogs, changedDialogs: [] } + } if (persistCloseUserIds.length > 0) { await tx From 5dc49a0f2dd6cd759823b7557248e25695b95495 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 20:22:22 +0330 Subject: [PATCH 02/14] server: preserve prepared plans for ordinary array bindings --- server/BENCHMARKING.md | 4 ++-- server/src/db/preparedQueries.postgres.test.ts | 18 +++++++++++++++++- server/src/db/preparedQueries.ts | 8 +++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/server/BENCHMARKING.md b/server/BENCHMARKING.md index 3eecef699..041c41ed9 100644 --- a/server/BENCHMARKING.md +++ b/server/BENCHMARKING.md @@ -196,13 +196,13 @@ automatic correctness proof or a promise of production latency savings. ## Prepared query policy -Ordinary Drizzle postgres-js calls now opt into preparation for bounded scalar +Ordinary Drizzle postgres-js calls now opt into preparation for bounded scalar/ordinary-array SQL/type signatures, including transactions, nested savepoints and reserved clients. Warm prepared calls avoid the extra Describe/Flush exchange; result rows and authorization decisions are never cached. Explicit per-query choices are preserved. After 256 admitted signatures, previously seen shapes remain prepared and new shapes execute unprepared. Queries longer than 32 KiB, more -than 301 parameters, and custom typed/array parameters retain their prior mode. +than 301 parameters, and custom typed parameters retain their prior mode. PgBouncer must support protocol-level prepared statements. The pool's existing prepare setting remains authoritative, including an explicit global opt-out. diff --git a/server/src/db/preparedQueries.postgres.test.ts b/server/src/db/preparedQueries.postgres.test.ts index e017abee9..05c2f1f13 100644 --- a/server/src/db/preparedQueries.postgres.test.ts +++ b/server/src/db/preparedQueries.postgres.test.ts @@ -13,7 +13,7 @@ async function withClient(run: (client: postgres.Sql, proxy: Awaited { + await withClient(async (client, proxy) => { + // JSON readback avoids the driver's numeric-array NULL parser so this + // checks the bound database values, including NULL and an empty array. + const query = "select to_json($1::integer[]) as ids" + await client.unsafe(query, [[1, 2]]) + for (const ids of [[3], [4, 5, 6], [], [null, 7]]) { + proxy.reset() + expect((await client.unsafe(query, [ids]))[0]?.["ids"]).toEqual(ids) + expect(proxy.snapshot().frames["P"] ?? 0).toBe(0) + expect(proxy.snapshot().frames["H"] ?? 0).toBe(0) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + } + }) +}) diff --git a/server/src/db/preparedQueries.ts b/server/src/db/preparedQueries.ts index 791342732..8afe10f36 100644 --- a/server/src/db/preparedQueries.ts +++ b/server/src/db/preparedQueries.ts @@ -23,7 +23,7 @@ export function prepareQueries(client: postgres.Sql): void { query.length > MAX_QUERY_LENGTH || !/^\s*(select|insert|update|delete|with)\b/i.test(query)) { return unsafe(query, parameters, options) } - // Match the driver's ordinary scalar type categories. Custom typed/array + // Match the driver's ordinary scalar/array type categories. Custom typed // parameters retain their original execution path rather than admitting // unbounded type variants under one SQL-text entry. const types = parameters.map(parameterType) @@ -59,6 +59,12 @@ export function prepareQueries(client: postgres.Sql): void { } function parameterType(value: unknown): number | undefined { + // postgres.js infers ordinary arrays from their first element. Bound the + // traversal so malformed/cyclic inputs do not recurse inside this policy. + for (let depth = 0; Array.isArray(value); depth++) { + if (depth >= 32) return undefined + value = value[0] ?? null + } if (value instanceof Date) return 1184 if (value instanceof Uint8Array) return 17 if (typeof value === "boolean") return 16 From 7034a5fcc9d440cbe6553c0b3e3a1806a3091ef8 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 20:39:17 +0330 Subject: [PATCH 03/14] server: reset replay rewind backoff after recovery --- .../modules/updates/replayLoopGuard.test.ts | 19 +++++++++++++++++++ server/src/modules/updates/replayLoopGuard.ts | 1 + 2 files changed, 20 insertions(+) diff --git a/server/src/modules/updates/replayLoopGuard.test.ts b/server/src/modules/updates/replayLoopGuard.test.ts index 3b6836260..3b111c8e0 100644 --- a/server/src/modules/updates/replayLoopGuard.test.ts +++ b/server/src/modules/updates/replayLoopGuard.test.ts @@ -58,4 +58,23 @@ describe("replay loop guard", () => { expect(guard.allow("a", 4000)).toBe(true) expect(guard.allow("a", 0)).toBe(true) }) + + test("resets rewind backoff after passing the previous high-water mark on the same socket", () => { + let now = 0 + const guard = new ReplayLoopGuard(() => now) + for (const seq of [4000, 0, 4000]) expect(guard.allow("a", seq)).toBe(true) + expect(guard.allow("a", 0)).toBe(false) + + // Recovery is possible during backoff, without an idle expiry or reconnect. + expect(guard.allow("a", 4001)).toBe(true) + now += 1000 + expect(guard.allow("a", 0)).toBe(true) + expect(guard.allow("a", 4001)).toBe(true) + expect(guard.allow("a", 0)).toBe(false) + // A new failure episode starts at 60 seconds, not the old 120-second step. + now += 59_999 + expect(guard.allow("a", 0)).toBe(false) + now += 1 + expect(guard.allow("a", 0)).toBe(true) + }) }) diff --git a/server/src/modules/updates/replayLoopGuard.ts b/server/src/modules/updates/replayLoopGuard.ts index cf1c22ff8..0ec831bdc 100644 --- a/server/src/modules/updates/replayLoopGuard.ts +++ b/server/src/modules/updates/replayLoopGuard.ts @@ -25,6 +25,7 @@ export class ReplayLoopGuard { // A recovered client may continue beyond the old failed pass immediately. if (startSeq > state.highWater) { state.highWater = startSeq + state.rewinds = 0 state.blockedUntil = 0 } if (now < state.blockedUntil) return false From 0b51ed38f2dd521d256cc553e014f6121ed8e298 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 20:48:00 +0330 Subject: [PATCH 04/14] server: qualify broad preparation and bound query shapes --- server/BENCHMARKING.md | 74 +++++++++++++--- server/bench/backend.test.ts | 49 ++++++++++- server/docs/database-observability.md | 19 ++++- server/scripts/benchmark-backend.test.ts | 35 ++++++++ server/scripts/benchmark-backend.ts | 9 +- server/src/__tests__/performance/report.ts | 26 ++++++ .../performance/wire.postgres.test.ts | 15 ++-- .../db/connectionPolicy.integration.test.ts | 49 ++++++++++- server/src/db/connectionPolicy.test.ts | 5 ++ server/src/db/connectionPolicy.ts | 38 ++++++--- server/src/db/models/users.ts | 4 +- .../src/db/preparedQueries.postgres.test.ts | 84 +++++++++++++++++-- server/src/db/preparedQueries.ts | 37 ++++++-- .../src/modules/dialogOpen.postgres.test.ts | 18 ++++ server/src/modules/dialogOpen.ts | 13 +-- 15 files changed, 417 insertions(+), 58 deletions(-) diff --git a/server/BENCHMARKING.md b/server/BENCHMARKING.md index 041c41ed9..7ef0e8561 100644 --- a/server/BENCHMARKING.md +++ b/server/BENCHMARKING.md @@ -36,6 +36,10 @@ bun run bench:backend --compare .test-results/before.json --output .test-results # Explore sensitivity to additional DB transport latency on one path. bun run bench:backend --scenario send.dm.open --rtt 0,3,5 +# Compare safe default preparation with qualified broad preparation. +bun run bench:backend --preparation access --scenario send.dm.open --rtt 0,3,5 --output .test-results/access.json +bun run bench:backend --preparation all-custom --scenario send.dm.open --rtt 0,3,5 --compare .test-results/access.json --output .test-results/all-custom.json + # Check the entire benchmark workflow quickly; this is not a tail-latency study. bun run bench:backend --all --samples 1 --warmup 1 ``` @@ -47,6 +51,20 @@ never overwritten. Unknown selections, invalid arguments, failed assertions, background failures, failed cleanup and incomplete runs exit unsuccessfully. Interrupted worker artifacts are candidates, not accepted comparison baselines. +`--preparation access` is the default; shell rollout flags are deliberately not +inherited. `--preparation all-custom` sets `force_custom_plan` only as a database +default on the lifecycle's disposable local database. It then recreates the +application pools and validates startup before measuring. The provisioning +database and role defaults are never changed. Use at least three warmups when +studying retained broad statements, which require two observations for admission. + +Reports record the requested mode, the actual `plan_cache_mode`, and statement/ +execution counters from one sampled backend after the matrix. These counters +include driver-owned statements and are not an admission count or pool total. +Access-to-all-custom comparisons display both treatments and planning settings. +Legacy reports without preparation metadata remain readable but cannot compare +to new qualified reports; rerun those baselines with an explicit mode. + Run timing experiments alone, on the same otherwise idle machine, database version and power configuration. Avoid concurrent builds and tests. Repeat A/B measurements before claiming a small improvement. Keep reports with the change; @@ -77,6 +95,9 @@ Each iteration follows this sequence: | `commandsBeforeReturn` | Commands submitted by return time; background work can overlap, so this is **not** a foreground-only count | | `wire.exchangeBoundaries` | Frontend `Query`, `Flush`, and `Sync` frames observed at the proxy | | `calibrationMs` | Ten real `SELECT 1` timings at the requested delay, outside the operation samples | +| `preparation.mode` / `planCacheMode` | Explicit benchmark treatment and actual backend planning policy | +| `preparation.sampledBackendStatements` | Retained statements visible on one backend after the matrix; includes driver-owned queries | +| `preparation.sampledBackendCustomPlans` / `sampledBackendGenericPlans` | Execution counters on that sampled backend, rather than pool-wide totals | One driver submission can contain several SQL statements, so `sql.commands` is not a SQL parser's statement count. A transaction does not make its statements @@ -196,15 +217,44 @@ automatic correctness proof or a promise of production latency savings. ## Prepared query policy -Ordinary Drizzle postgres-js calls now opt into preparation for bounded scalar/ordinary-array -SQL/type signatures, including transactions, nested savepoints and reserved -clients. Warm prepared calls avoid the extra Describe/Flush exchange; result -rows and authorization decisions are never cached. Explicit per-query choices -are preserved. After 256 admitted signatures, previously seen shapes remain -prepared and new shapes execute unprepared. Queries longer than 32 KiB, more -than 301 parameters, and custom typed parameters retain their prior mode. -PgBouncer must support protocol-level prepared statements. The pool's existing -prepare setting remains authoritative, including an explicit global opt-out. - -The real-wire regression tests distinguish SQL submissions from protocol -exchanges and cover new bind values, nested rollback and bounded admission. +The default `INLINE_DATABASE_PREPARED_QUERIES=access` prepares only the existing +`sync.replay` and `sync.sidecars` access family. Its 100 batch lengths per source +fit a 200-signature budget. Other Drizzle queries retain unnamed execution and +custom planning. Rows and authorization decisions are never cached. + +`INLINE_DATABASE_PREPARED_QUERIES=all-custom` opts into broader scalar/ordinary-array +preparation, including transactions, savepoints and reserved clients. Startup +must first observe `plan_cache_mode=force_custom_plan` on the actual query path; +readiness checks it again. This requires an operator-qualified application role +default on every backend, with existing pooled sessions reestablished after the +role change. PgBouncer does not accept this as an ordinary startup parameter. +The application never changes roles or session settings. A sampled startup check +cannot establish that every existing backend has inherited the role default. + +Broad admission requires a second observation, with at most 200 pending candidates. +This avoids retaining one-off SQL shapes, such as transient `inArray` lengths. +It admits at most 100 broad signatures and leaves 100 slots for access queries, +within the same total cap of 200. Retained signatures live until pool shutdown; +they are not an LRU. Once a partition or total budget fills, new shapes use the +original unnamed path. Changing hot lists to a single array binding avoids a +new signature for each list length. The default access mode keeps its full 200 +slots. Explicit per-query options and the pool-level preparation switch remain +authoritative. Queries longer than 32 KiB, more than 301 parameters, and custom +typed parameters retain their prior mode. + +Warm admitted calls skip Describe/Flush while PostgreSQL builds custom plans +from current bindings. The wire fixture opts into `all-custom` using a dedicated +direct connection with the setting established before startup. The benchmark +launcher provides its own disposable-database qualification via `--preparation`. Database tests +assert partial-index selection and zero generic plans after more than five +executions, plus rollback, changed array bindings and admission pressure. + +Measurements from commit `5dc49a0f2` used unconditional broad preparation under +the database's prior planning policy. They do not describe this safe default or +qualify `all-custom` latency. New comparisons must record the preparation mode, +role setting, admission limits and enough warmups (at least three) for repeated +shapes; inspect `pg_prepared_statements` counters to verify prepared reuse and planning policy. + +Role-wide custom planning also applies to the previously tuned access queries. +Qualify representative replay latency as well as simple point reads before +enabling broad mode; faster protocol exchanges alone do not prove a net gain. diff --git a/server/bench/backend.test.ts b/server/bench/backend.test.ts index 597034fc8..2ed4aa730 100644 --- a/server/bench/backend.test.ts +++ b/server/bench/backend.test.ts @@ -4,18 +4,20 @@ import { createHash } from "node:crypto" import { cpus } from "node:os" import { resolve } from "node:path" import { sql } from "drizzle-orm" -import { closeDb, db, initDb } from "@in/server/db" +import { closeDb, db, initDb, validateDatabaseStartup } from "@in/server/db" import { setupTestDatabase, teardownTestDatabase } from "@in/server/__tests__/database" import { scenarios, scenarioVersion, selectScenarios } from "@in/server/__tests__/performance/catalog" import { prepareScenario } from "@in/server/__tests__/performance/scenarios" import { observeScenarioWork } from "@in/server/__tests__/performance/observe" import { assertQueryBudget, measureOperation, type Sample } from "@in/server/__tests__/performance/measure" import { createWireProxy } from "@in/server/__tests__/performance/wire" -import { summarize, type BenchmarkReport } from "@in/server/__tests__/performance/report" +import { summarize, type BenchmarkReport, type BenchmarkPreparation } from "@in/server/__tests__/performance/report" const encoded = process.env["INLINE_BACKEND_BENCH_CONFIG"] if (!encoded) throw new Error("Use bun run bench:backend; this worker needs the isolated benchmark launcher") -const config = JSON.parse(encoded) as { ids: string[]; samples: number; warmup: number; rtts: number[]; output: string } +const config = JSON.parse(encoded) as { ids: string[]; samples: number; warmup: number; rtts: number[]; preparation: BenchmarkPreparation["mode"]; output: string } +if (config.preparation !== "access" && config.preparation !== "all-custom") throw new Error("Invalid benchmark preparation mode") +const previousPreparation = process.env["INLINE_DATABASE_PREPARED_QUERIES"] const selected = selectScenarios(config.ids) const background = observeScenarioWork() let proxy: Awaited> | undefined @@ -23,15 +25,36 @@ let proxy: Awaited> | undefined beforeAll(async () => { await setupTestDatabase() const url = new URL(process.env["DATABASE_URL"]!) + if (config.preparation === "all-custom") { + // Only the lifecycle's freshly cloned database may receive this default. + // No role change, provisioning database change, or session-only shortcut. + const template = process.env["INLINE_TEST_DATABASE_TEMPLATE"] + const name = url.pathname.slice(1) + if (!template || !/^inline_test_template_[0-9]+_[0-9a-f]{32}$/.test(template) || + !new RegExp(`^test_db_${template.slice(-32)}_[0-9a-f]{12}$`).test(name) || + !["localhost", "127.0.0.1"].includes(url.hostname)) { + throw new Error("Refusing to configure an unowned benchmark database") + } + const [current] = await db.execute<{ name: string }>(sql`select current_database() as name`) + if (current?.name !== name) throw new Error("Benchmark database identity changed") + await db.$client.unsafe(`ALTER DATABASE "${name}" SET plan_cache_mode = force_custom_plan`) + } + process.env["INLINE_DATABASE_PREPARED_QUERIES"] = config.preparation proxy = await createWireProxy({ host: url.hostname, port: Number(url.port || 5432) }, 0) url.hostname = "127.0.0.1" url.port = String(proxy.port) await closeDb() initDb(url.toString()) + await validateDatabaseStartup() }) afterAll(async () => { try { await background.close() } finally { - try { await teardownTestDatabase() } finally { await proxy?.close() } + try { await teardownTestDatabase() } finally { + try { await proxy?.close() } finally { + if (previousPreparation === undefined) Reflect.deleteProperty(process.env, "INLINE_DATABASE_PREPARED_QUERIES") + else process.env["INLINE_DATABASE_PREPARED_QUERIES"] = previousPreparation + } + } } }) @@ -107,6 +130,24 @@ test("verified backend benchmark matrix", async () => { if (report.results.length !== selected.length * config.rtts.length || selected.some((entry) => !scenarios.includes(entry))) { throw new Error("Incomplete benchmark matrix") } + const [settings] = await db.execute<{ mode: string; statements: number; custom: string; generic: string }>(sql` + select current_setting('plan_cache_mode') as mode, count(*)::integer as statements, + coalesce(sum(custom_plans), 0)::text as custom, coalesce(sum(generic_plans), 0)::text as generic + from pg_prepared_statements + `) + if (!settings || !["auto", "force_custom_plan", "force_generic_plan"].includes(settings.mode)) { + throw new Error("Cannot record benchmark planning policy") + } + if (config.preparation === "all-custom" && settings.mode !== "force_custom_plan") { + throw new Error("Benchmark lost custom-plan qualification") + } + report.preparation = { + mode: config.preparation, planCacheMode: settings.mode as BenchmarkPreparation["planCacheMode"], + sampledBackendStatements: settings.statements, + sampledBackendCustomPlans: Number(settings.custom), sampledBackendGenericPlans: Number(settings.generic), + } + console.info(`Preparation ${report.preparation.mode}; plan_cache_mode=${report.preparation.planCacheMode}; ` + + `sampled backend: ${settings.statements} retained statements, ${settings.custom} custom / ${settings.generic} generic executions`) if (await sourceDigest() !== digest) throw new Error("Backend sources changed during the benchmark; rerun on a stable checkout") // The launcher promotes this only after Bun hooks and DB disposal also pass. writeFileSync(config.output, JSON.stringify({ ...report, status: "candidate" }, null, 2) + "\n", { flag: "wx" }) diff --git a/server/docs/database-observability.md b/server/docs/database-observability.md index c56acafb3..3fe146ecb 100644 --- a/server/docs/database-observability.md +++ b/server/docs/database-observability.md @@ -84,15 +84,26 @@ transactions and deployment overlap. The shared query client now prepares only tagged `sync.replay` and `sync.sidecars` access queries with at most 301 parameters (100 chat IDs used three times, plus one user). Their two sources yield at most 200 statement -shapes per connection. Larger batches and transaction-local queries retain the -existing driver behavior. Explicit `prepare: false` is respected. This caches +shapes per connection. Larger batches retain the existing driver behavior. The same bounded policy +also applies to transactions, savepoints and reserved clients. Explicit `prepare: false` is respected. This caches plans, never permissions or results. Production PgBouncer was verified to use transaction mode with `max_prepared_statements=200`; keep this capability when changing providers or pool configuration. Local integration tests exercise actual Drizzle prepared reads through PgBouncer, and authorization tests cover revocation after warm repeated reads. Array-binding alternatives were rejected -after live probes showed PostgreSQL kept replanning them instead of selecting -the much faster generic plan for the existing query shape. +after historical live probes showed PostgreSQL kept replanning them instead of +selecting the faster generic plan for that query shape. This is evidence for the +narrow family, not for enabling generic plans across all queries. + +Broader preparation requires explicit `INLINE_DATABASE_PREPARED_QUERIES=all-custom` +and an operator-established `force_custom_plan` default on the query role. The +startup and readiness probes inspect `plan_cache_mode` through the query pool; +startup rejects unqualified opt-ins, and readiness disables broad admission if +the sampled backend loses qualification. Establish the role default and recycle +existing PgBouncer backend sessions before enabling it: sampling one backend +does not prove the setting on every backend. No role or session settings are +changed by this code. See [the prepared-query policy](../BENCHMARKING.md#prepared-query-policy) +for the bounded second-observation admission policy and benchmark limits. Replay access checks execute the canonical SELECT directly on the shared client. The former single-query explicit transaction added BEGIN/SET/COMMIT diff --git a/server/scripts/benchmark-backend.test.ts b/server/scripts/benchmark-backend.test.ts index 3c7ef67f2..5a7da91b5 100644 --- a/server/scripts/benchmark-backend.test.ts +++ b/server/scripts/benchmark-backend.test.ts @@ -25,6 +25,7 @@ test.each([ ["--warmup", "NaN", "--warmup must"], ["--rtt", "0,,3", "Empty RTT"], ["--rtt", "-1", "RTT must"], + ["--preparation", "invalid", "--preparation must be access or all-custom"], ["--output", ".env.json", "never environment files"], ["--retry", "3", "Unknown option"], ])("invalid %s %s fails before provisioning: %s", async (flag, value, message) => { @@ -37,6 +38,7 @@ function report(): BenchmarkReport { return { schemaVersion: 1, status: "passed", createdAt: "2026-01-01T00:00:00.000Z", revision: { commit: "fixture", dirty: false, sourceDigest: "a".repeat(64) }, environment: { bun: "1.4.0", postgres: "15", postgresJs: "3.4.7", drizzle: "fixture", platform: "darwin", arch: "arm64", cpu: "fixture", poolMax: 10, clientPrepareDefault: true, concurrency: 1, cache: "cold-application-warm-pool" }, + preparation: { mode: "access", planCacheMode: "auto", sampledBackendStatements: 2, sampledBackendCustomPlans: 5, sampledBackendGenericPlans: 0 }, warmup: 1, sampleCount: 1, results: [{ id: "send.dm.open", version: 1, size: 1, rttMs: 3, maxCommands: 27, calibrationMs: [3], samples: [{ operationMs: 10, settledMs: 12, commandsBeforeReturn: 2, @@ -73,3 +75,36 @@ test("failed, empty, incomplete and incompatible runs cannot be compared as succ expect(() => compareReports(report(), changed)).toThrow() } }) + + +test.each(["access", "all-custom"])("explicit %s preparation can list without provisioning", async (mode) => { + const result = await run(`--preparation=${mode}`, "--list") + expect(result.code).toBe(0) + expect(result.output).toContain("send.public.100") +}) + +test("preparation comparisons identify the treatment and its actual planning policy", () => { + const before = report(), after = report() + after.preparation = { mode: "all-custom", planCacheMode: "force_custom_plan", sampledBackendStatements: 10, sampledBackendCustomPlans: 20, sampledBackendGenericPlans: 0 } + expect(compareReports(before, after)[0]).toMatchObject({ + preparationBefore: "access", preparationAfter: "all-custom", + planCacheModeBefore: "auto", planCacheModeAfter: "force_custom_plan", + }) +}) + +test("preparation metadata rejects unqualified modes and malformed backend counters", () => { + for (const metadata of [ + { mode: "unknown", planCacheMode: "auto", sampledBackendStatements: 0, sampledBackendCustomPlans: 0, sampledBackendGenericPlans: 0 }, + { mode: "all-custom", planCacheMode: "auto", sampledBackendStatements: 0, sampledBackendCustomPlans: 0, sampledBackendGenericPlans: 0 }, + { mode: "access", planCacheMode: "auto", sampledBackendStatements: -1, sampledBackendCustomPlans: 0, sampledBackendGenericPlans: 0 }, + { mode: "access", planCacheMode: "auto", sampledBackendStatements: 0 }, + ]) expect(() => validateReport({ ...report(), preparation: metadata })).toThrow("preparation") +}) + +test("legacy reports remain readable but cannot silently compare to a qualified mode", () => { + const legacy = report() + delete legacy.preparation + expect(() => validateReport(legacy)).not.toThrow() + expect(() => compareReports(legacy, report())).toThrow("Preparation policy is unknown") + expect(() => compareReports(legacy, structuredClone(legacy))).not.toThrow() +}) diff --git a/server/scripts/benchmark-backend.ts b/server/scripts/benchmark-backend.ts index ba4f9eb2e..6bcdb7aa3 100644 --- a/server/scripts/benchmark-backend.ts +++ b/server/scripts/benchmark-backend.ts @@ -11,16 +11,21 @@ const { values } = parseArgs({ args: process.argv.slice(2), strict: true, allowPositionals: false, options: { scenario: { type: "string", multiple: true }, all: { type: "boolean" }, list: { type: "boolean" }, - samples: { type: "string" }, warmup: { type: "string" }, rtt: { type: "string" }, + samples: { type: "string" }, warmup: { type: "string" }, rtt: { type: "string" }, preparation: { type: "string" }, output: { type: "string" }, compare: { type: "string" }, help: { type: "boolean", short: "h" }, }, }) +const preparation = values.preparation ?? "access" +if (preparation !== "access" && preparation !== "all-custom") { + throw new Error("--preparation must be access or all-custom") +} if (values.help) { console.info(`Usage: bun run bench:backend [--scenario ID ... | --all] [--rtt 0,3,5] --list List scenarios and reviewed SQL command budgets; no DB needed --samples N Measured, independently verified iterations (default 20, max 200) --warmup N Unreported, verified iterations per case (default 3, max 20) --rtt MS,... Requested additional loopback transport RTT (default 0, max 50) + --preparation MODE access (default) or all-custom; configures only disposable DB --output FILE.json Save a new report; never overwrite a baseline --compare FILE.json Compare a completed, compatible baseline with this run @@ -68,7 +73,7 @@ const environment = createTestEnvironment(process.env, databaseUrl) // must not silently change the deployment mode or invalidate that baseline. environment["INLINE_REALTIME_DISTRIBUTED"] = "0" environment["REALTIME_DISTRIBUTED"] = "0" -environment["INLINE_BACKEND_BENCH_CONFIG"] = JSON.stringify({ ids: selected.map((scenario) => scenario.id), samples, warmup, rtts, output: candidate }) +environment["INLINE_BACKEND_BENCH_CONFIG"] = JSON.stringify({ ids: selected.map((scenario) => scenario.id), samples, warmup, rtts, preparation, output: candidate }) let template: Awaited> | undefined let child: ReturnType | undefined diff --git a/server/src/__tests__/performance/report.ts b/server/src/__tests__/performance/report.ts index 846136e27..dfbbb7109 100644 --- a/server/src/__tests__/performance/report.ts +++ b/server/src/__tests__/performance/report.ts @@ -10,6 +10,15 @@ export type BenchmarkResult = { samples: Sample[] } +export type BenchmarkPreparation = { + mode: "access" | "all-custom" + planCacheMode: "auto" | "force_custom_plan" | "force_generic_plan" + // pg_prepared_statements is local to one sampled backend, not a pool total. + sampledBackendStatements: number + sampledBackendCustomPlans: number + sampledBackendGenericPlans: number +} + export type BenchmarkReport = { schemaVersion: 1 status: "passed" @@ -20,6 +29,7 @@ export type BenchmarkReport = { platform: string; arch: string; cpu: string; poolMax: number clientPrepareDefault: boolean; concurrency: 1; cache: "cold-application-warm-pool" } + preparation?: BenchmarkPreparation warmup: number sampleCount: number results: BenchmarkResult[] @@ -38,6 +48,9 @@ export function summarize(result: BenchmarkResult) { export function compareReports(before: BenchmarkReport, after: BenchmarkReport) { validateReport(before) validateReport(after) + if (Boolean(before.preparation) !== Boolean(after.preparation)) { + throw new Error("Preparation policy is unknown in one report; rerun the baseline with explicit preparation metadata") + } if (JSON.stringify(before.environment) !== JSON.stringify(after.environment) || before.warmup !== after.warmup || before.sampleCount !== after.sampleCount) { throw new Error("Incompatible benchmark environment, warmup, or sample count; rerun the baseline with matching settings") } @@ -52,6 +65,10 @@ export function compareReports(before: BenchmarkReport, after: BenchmarkReport) const a = summarize(previous), b = summarize(result) return { scenario: result.id, rttMs: result.rttMs, + preparationBefore: before.preparation?.mode ?? "legacy-unknown", + preparationAfter: after.preparation?.mode ?? "legacy-unknown", + planCacheModeBefore: before.preparation?.planCacheMode ?? "unknown", + planCacheModeAfter: after.preparation?.planCacheMode ?? "unknown", commandsSaved: a.commands.median - b.commands.median, exchangesSaved: a.exchanges.median - b.exchanges.median, operationMsSaved: a.operationMs.median - b.operationMs.median, @@ -77,6 +94,15 @@ export function validateReport(value: unknown): asserts value is BenchmarkReport report.environment.cache !== "cold-application-warm-pool") { throw new Error("Missing benchmark provenance or measurement settings") } + const preparation = report.preparation + if (preparation !== undefined && (!preparation || + !["access", "all-custom"].includes(preparation.mode) || + !["auto", "force_custom_plan", "force_generic_plan"].includes(preparation.planCacheMode) || + (preparation.mode === "all-custom" && preparation.planCacheMode !== "force_custom_plan") || + [preparation.sampledBackendStatements, preparation.sampledBackendCustomPlans, preparation.sampledBackendGenericPlans] + .some((count) => !Number.isSafeInteger(count) || count < 0))) { + throw new Error("Invalid benchmark preparation policy or counters") + } const keys = new Set() for (const result of report.results) { const key = `${result.id}@${result.rttMs}` diff --git a/server/src/__tests__/performance/wire.postgres.test.ts b/server/src/__tests__/performance/wire.postgres.test.ts index db4ad7cd8..c16bc76f8 100644 --- a/server/src/__tests__/performance/wire.postgres.test.ts +++ b/server/src/__tests__/performance/wire.postgres.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "bun:test" import { sql } from "drizzle-orm" -import { db, closeDb, initDb } from "@in/server/db" +import { makeDatabaseClients } from "@in/server/db/connectionPolicy" +import { drizzle } from "drizzle-orm/postgres-js" import { setupTestLifecycle } from "../database" import { createWireProxy } from "./wire" import { measureOperation } from "./measure" @@ -13,9 +14,14 @@ test("real Drizzle queries expose protocol exchanges independently of SQL comman const proxy = await createWireProxy({ host: url.hostname, port: Number(url.port || 5432) }, 0) url.hostname = "127.0.0.1" url.port = String(proxy.port) - await closeDb() - initDb(url.toString()) + // This dedicated direct fixture qualifies the opt-in explicitly; production + // uses a role default because PgBouncer rejects this startup parameter. + url.searchParams.set("plan_cache_mode", "force_custom_plan") + const clients = makeDatabaseClients(url.toString(), { INLINE_DATABASE_PREPARED_QUERIES: "all-custom" }) + const db = drizzle(clients.queryClient) try { + await clients.validateStartup() + await db.execute(sql`select ${0}::int as value`) await db.execute(sql`select ${1}::int as value`) const sample = await measureOperation(db.$client.options, async () => { const result = await db.execute(sql`select ${2}::int as value`) @@ -36,8 +42,7 @@ test("real Drizzle queries expose protocol exchanges independently of SQL comman expect(delayed.wire!.exchangeBoundaries).toBe(1) expect(delayed.operationMs).toBeGreaterThan(0) } finally { - await closeDb() + await clients.close() await proxy.close() - initDb(direct) } }) diff --git a/server/src/db/connectionPolicy.integration.test.ts b/server/src/db/connectionPolicy.integration.test.ts index f7f6a844e..5e64171ff 100644 --- a/server/src/db/connectionPolicy.integration.test.ts +++ b/server/src/db/connectionPolicy.integration.test.ts @@ -57,9 +57,14 @@ suite("real transaction pooling safety (requires local PostgreSQL and PgBouncer await admin`alter role fixture_app set lock_timeout = '5s'` await admin`alter role fixture_app set idle_in_transaction_session_timeout = '15s'` await admin`create role fixture_unsafe login` + await admin`create role fixture_custom login` + await admin`alter role fixture_custom set statement_timeout = '30s'` + await admin`alter role fixture_custom set lock_timeout = '5s'` + await admin`alter role fixture_custom set idle_in_transaction_session_timeout = '15s'` + await admin`alter role fixture_custom set plan_cache_mode = force_custom_plan` await admin`create table public.pooling_fixture (id int primary key)` - await admin`grant select, insert on public.pooling_fixture to fixture_app` - await writeFile(join(directory, "users.txt"), '"fixture_app" ""\n"fixture_unsafe" ""\n') + await admin`grant select, insert on public.pooling_fixture to fixture_app, fixture_custom` + await writeFile(join(directory, "users.txt"), '"fixture_app" ""\n"fixture_unsafe" ""\n"fixture_custom" ""\n') await writeFile(join(directory, "pgbouncer.ini"), `[databases]\npostgres = host=127.0.0.1 port=${port} dbname=postgres\n[pgbouncer]\nlisten_addr = 127.0.0.1\nlisten_port = ${pooledPort}\nunix_socket_dir = ${directory}\nauth_type = trust\nauth_file = ${join(directory, "users.txt")}\npool_mode = transaction\ndefault_pool_size = 2\nmax_client_conn = 30\nmax_prepared_statements = 200\nlogfile = ${join(directory, "pgbouncer.log")}\npidfile = ${join(directory, "pgbouncer.pid")}\n`) bouncer = Bun.spawn([pgbouncer!, join(directory, "pgbouncer.ini")], { stdout: "ignore", stderr: "ignore" }) const deadline = Date.now() + 5_000 @@ -93,6 +98,46 @@ suite("real transaction pooling safety (requires local PostgreSQL and PgBouncer finally { await unsafe.close() } }) + test("all-custom rejects the query role even when the direct health role is qualified", async () => { + const unqualified = makeDatabaseClients(pooledUrl, { + DATABASE_CONNECTION_MODE: "pgbouncer", + DATABASE_DIRECT_URL: directUrl.replace("fixture_app", "fixture_custom"), + INLINE_DATABASE_PREPARED_QUERIES: "all-custom", + }) + try { await expect(unqualified.validateStartup()).rejects.toThrow("force_custom_plan") } + finally { await unqualified.close() } + }) + + test("all-custom validates role defaults through PgBouncer and keeps custom plans", async () => { + const qualified = makeDatabaseClients(pooledUrl.replace("fixture_app", "fixture_custom"), { + DATABASE_CONNECTION_MODE: "pgbouncer", + DATABASE_DIRECT_URL: directUrl, + INLINE_DATABASE_PREPARED_QUERIES: "all-custom", + }) + try { + await qualified.validateStartup() + const db = drizzle(qualified.queryClient) + for (let value = 0; value < 24; value++) { + const rows = await db.execute(sql`select ${value}::int as qualified_value`) + expect(rows[0]?.["qualified_value"]).toBe(value) + } + const plans = await qualified.queryClient`select generic_plans, custom_plans from pg_prepared_statements + where statement like '%as qualified_value%' and statement not like '%pg_prepared_statements%'` + expect(plans.length).toBeGreaterThan(0) + expect(plans.every((row) => Number(row["generic_plans"]) === 0)).toBe(true) + expect(plans.some((row) => Number(row["custom_plans"]) > 5)).toBe(true) + await qualified.checkHealth() + // Deliberately change the isolated fixture backend to exercise readiness + // losing its qualification; application queries return to narrow mode. + await qualified.queryClient`set plan_cache_mode = auto` + try { + await expect(qualified.checkHealth()).rejects.toThrow("force_custom_plan") + } finally { + await qualified.queryClient`set plan_cache_mode = force_custom_plan` + } + } finally { await qualified.close() } + }) + test("prepared queries reuse two server connections without losing role defaults", async () => { const results = await Promise.all(Array.from({ length: 40 }, (_, value) => clients!.queryClient.begin(async (tx) => { const rows = await tx`select ${value}::int as value, pg_backend_pid() as pid, current_setting('statement_timeout') as timeout` diff --git a/server/src/db/connectionPolicy.test.ts b/server/src/db/connectionPolicy.test.ts index 397cc6daa..9caac8d9f 100644 --- a/server/src/db/connectionPolicy.test.ts +++ b/server/src/db/connectionPolicy.test.ts @@ -74,6 +74,11 @@ describe("database connection safety", () => { } }) + test("invalid prepared modes fail without echoing their value", () => { + expect(() => makeDatabaseClients(direct, { INLINE_DATABASE_PREPARED_QUERIES: "private-secret" })) + .toThrow("INLINE_DATABASE_PREPARED_QUERIES must be access or all-custom.") + }) + test("pooled mode and a separate direct URL must be explicit", () => { expect(() => databaseConnectionPolicy(pooled)).toThrow("explicit PgBouncer mode") expect(() => databaseConnectionPolicy(pooled, { DATABASE_CONNECTION_MODE: "pgbouncer" })).toThrow("DATABASE_DIRECT_URL") diff --git a/server/src/db/connectionPolicy.ts b/server/src/db/connectionPolicy.ts index 52103b6b8..950e437c8 100644 --- a/server/src/db/connectionPolicy.ts +++ b/server/src/db/connectionPolicy.ts @@ -1,5 +1,5 @@ import postgres from "postgres" -import { prepareQueries } from "./preparedQueries" +import { prepareQueries, type PreparedQueryPolicy } from "./preparedQueries" export const QUERY_TIMEOUTS = { statement_timeout: 30_000, @@ -19,6 +19,7 @@ type DatabaseEnvironment = { DATABASE_DIRECT_URL?: string DATABASE_CONNECTION_MODE?: string INLINE_DATABASE_QUERY_POOL_MAX?: string + INLINE_DATABASE_PREPARED_QUERIES?: string } const parseDatabaseUrl = (value: string): URL => { @@ -143,6 +144,11 @@ export const makeDatabaseClients = (databaseUrl: string, environment: DatabaseEn if (!/^(?:[1-9]|10)$/.test(queryPoolMax) || queryPoolMax.trim() !== queryPoolMax) { throw new Error("INLINE_DATABASE_QUERY_POOL_MAX must be an integer from 1 through 10.") } + const preparedMode = environment.INLINE_DATABASE_PREPARED_QUERIES ?? "access" + if (preparedMode !== "access" && preparedMode !== "all-custom") { + throw new Error("INLINE_DATABASE_PREPARED_QUERIES must be access or all-custom.") + } + const preparation: PreparedQueryPolicy = { mode: "access" } const policy = databaseConnectionPolicy(databaseUrl, environment) const queryClient = postgres(policy.queryUrl, { max: Number(queryPoolMax), connect_timeout: 5, idle_timeout: 30, @@ -155,29 +161,36 @@ export const makeDatabaseClients = (databaseUrl: string, environment: DatabaseEn max: 1, connect_timeout: 2, idle_timeout: 30, connection: { application_name: "inline-health", ...HEALTH_TIMEOUTS }, }) - prepareQueries(queryClient) + prepareQueries(queryClient, preparation) const close = () => Promise.all([queryClient.end({ timeout: 5 }), healthClient.end({ timeout: 5 })]).then(() => {}) const queryTimeoutSettings = () => queryClient` select name, setting, unit from pg_settings - where name in ('statement_timeout', 'lock_timeout', 'idle_in_transaction_session_timeout') + where name in ('statement_timeout', 'lock_timeout', 'idle_in_transaction_session_timeout', 'plan_cache_mode') `.execute() + const assertPreparation = (rows: readonly TimeoutRow[]) => { + if (preparedMode === "all-custom" && rows.find((row) => row.name === "plan_cache_mode")?.setting !== "force_custom_plan") { + preparation.mode = "access" + throw new Error("all-custom preparation requires force_custom_plan on the query role.") + } + } const checkHealth = () => { const direct = healthClient.unsafe( "SELECT (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::double precision AS database_time_millis", ).execute() - if (policy.mode === "direct") return direct + if (policy.mode === "direct" && preparedMode !== "all-custom") return direct // A healthy direct endpoint must not hide a broken/queued pooled endpoint. // The health controller's existing deadline cancels both pending queries. const pooled = queryTimeoutSettings() const probes = combineHealthProbes(direct, [pooled]) return Object.assign(probes.then(async (clock) => { const rows = await pooled - assertQueryTimeouts(rows) + if (policy.mode === "pgbouncer") assertQueryTimeouts(rows) + assertPreparation(rows) return clock }), { cancel: probes.cancel }) } const validateStartup = async (): Promise => { - if (policy.mode === "direct") return + if (policy.mode === "direct" && preparedMode === "access") return const query = queryTimeoutSettings() let timer: ReturnType | undefined try { @@ -185,23 +198,28 @@ export const makeDatabaseClients = (databaseUrl: string, environment: DatabaseEn query, new Promise((_, reject) => { timer = setTimeout(() => { - reject(new Error("PgBouncer timeout qualification exceeded 5 seconds.")) + reject(new Error("Database role qualification exceeded 5 seconds.")) try { query.cancel() } catch { /* The startup deadline remains authoritative. */ } }, 5_000) }), ]) - assertQueryTimeouts(rows) - await healthClient`select 1` + if (policy.mode === "pgbouncer") assertQueryTimeouts(rows) + // The query path can use a different role from the direct health client. + // Operators must also recycle existing backends after setting role defaults. + assertPreparation(rows) + if (policy.mode === "pgbouncer") await healthClient`select 1` + preparation.mode = preparedMode } catch { await close() // Driver errors can include connection metadata; emit a stable safe error. - throw new Error("PgBouncer startup qualification failed; verify role timeout defaults and the direct health endpoint.") + throw new Error(`${policy.mode === "pgbouncer" ? "PgBouncer" : "Database"} startup qualification failed; verify role timeout defaults, the direct health endpoint, and force_custom_plan when all-custom preparation is enabled.`) } finally { clearTimeout(timer) } } const diagnosticsConfiguration = { mode: policy.mode, + preparedQueries: preparedMode, queryPoolMax: queryClient.options.max, healthPoolMax: healthClient.options.max, statementTimeoutMs: QUERY_TIMEOUTS.statement_timeout, diff --git a/server/src/db/models/users.ts b/server/src/db/models/users.ts index 050c64556..0efc0ec84 100644 --- a/server/src/db/models/users.ts +++ b/server/src/db/models/users.ts @@ -22,10 +22,12 @@ export class UsersModel { return [] } + // One binding keeps the SQL shape stable across recipient counts. bigint + // also lets unknown protocol IDs beyond int4 simply miss the indexed key. const rows = await db .select({ id: users.id }) .from(users) - .where(and(inArray(users.id, uniqueUserIds), userNotDeleted())) + .where(and(sql`${users.id} = any(${sql.param(uniqueUserIds)}::bigint[])`, userNotDeleted())) const activeUserIds = new Set(rows.map((row) => row.id)) return uniqueUserIds.filter((id) => activeUserIds.has(id)) diff --git a/server/src/db/preparedQueries.postgres.test.ts b/server/src/db/preparedQueries.postgres.test.ts index 05c2f1f13..49982ff8e 100644 --- a/server/src/db/preparedQueries.postgres.test.ts +++ b/server/src/db/preparedQueries.postgres.test.ts @@ -2,19 +2,20 @@ import { expect, test } from "bun:test" import postgres from "postgres" import { drizzle } from "drizzle-orm/postgres-js" import { sql } from "drizzle-orm" +import { accessQueryComment } from "./databaseDiagnostics" import { setupTestLifecycle } from "../__tests__/database" import { createWireProxy } from "../__tests__/performance/wire" -import { MAX_PREPARED_QUERY_SHAPES, prepareQueries } from "./preparedQueries" +import { MAX_BROAD_PREPARED_QUERY_SHAPES, MAX_PREPARED_QUERY_SHAPES, prepareQueries, type PreparedQueryPolicy } from "./preparedQueries" setupTestLifecycle() -async function withClient(run: (client: postgres.Sql, proxy: Awaited>) => Promise) { +async function withClient(run: (client: postgres.Sql, proxy: Awaited>) => Promise, mode: PreparedQueryPolicy["mode"] = "all-custom") { const url = new URL(process.env["DATABASE_URL"]!) const proxy = await createWireProxy({ host: url.hostname, port: Number(url.port || 5432) }, 0) url.hostname = "127.0.0.1" url.port = String(proxy.port) - const client = postgres(url.toString(), { max: 1, ssl: false }) - prepareQueries(client) + const client = postgres(url.toString(), { max: 1, ssl: false, connection: { plan_cache_mode: "force_custom_plan" } }) + prepareQueries(client, { mode }) prepareQueries(client) // Installing twice must not stack wrappers or reset the shape budget. try { await client`select 1` @@ -32,6 +33,7 @@ test("Drizzle reuses plans with new values in the pool, transactions and savepoi const rows = await db.execute(sql`select ${value}::int as value`) expect(rows[0]?.["value"]).toBe(value) } + await read(0) await read(1) proxy.reset() await read(2) @@ -47,6 +49,7 @@ test("Drizzle reuses plans with new values in the pool, transactions and savepoi expect(inner[0]?.["value"]).toBe(value + 20) }) }, { isolationLevel: "repeatable read", accessMode: "read only" }) + await nestedRead(2) await nestedRead(3) proxy.reset() await nestedRead(4) @@ -55,6 +58,7 @@ test("Drizzle reuses plans with new values in the pool, transactions and savepoi const reserved = await client.reserve() try { + await reserved.unsafe("select $1::int + 30 as value", [4]) await reserved.unsafe("select $1::int + 30 as value", [5]) proxy.reset() const rows = await reserved.unsafe("select $1::int + 30 as value", [6]) @@ -88,11 +92,13 @@ test("prepared writes preserve savepoint and outer transaction rollback", async test("explicit opt-out and shape-budget overflow still execute correctly without retaining new plans", async () => { await withClient(async (client, proxy) => { const query = "select $1::int as value" + await client.unsafe(query, [0]) await client.unsafe(query, [1]) proxy.reset() expect((await client.unsafe(query, [2], { prepare: false }))[0]?.["value"]).toBe(2) expect(proxy.snapshot().frames["H"]).toBe(1) - for (let i = 1; i < MAX_PREPARED_QUERY_SHAPES; i++) { + for (let i = 1; i < MAX_BROAD_PREPARED_QUERY_SHAPES; i++) { + await client.unsafe(`select $1::int + ${i} as value`, [0]) await client.unsafe(`select $1::int + ${i} as value`, [1]) } const overflow = `select $1::int + ${MAX_PREPARED_QUERY_SHAPES} as value` @@ -100,6 +106,11 @@ test("explicit opt-out and shape-budget overflow still execute correctly without proxy.reset() expect((await client.unsafe(overflow, [2]))[0]?.["value"]).toBe(MAX_PREPARED_QUERY_SHAPES + 2) expect(proxy.snapshot().frames["H"]).toBe(1) + const access = `select $1::int as access_value ${accessQueryComment("sync.replay")}` + await client.unsafe(access, [1]) + proxy.reset() + expect((await client.unsafe(access, [2]))[0]?.["access_value"]).toBe(2) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) proxy.reset() expect((await client.unsafe(query, [3]))[0]?.["value"]).toBe(3) expect(proxy.snapshot().exchangeBoundaries).toBe(1) @@ -125,6 +136,7 @@ test("ordinary array bindings reuse plans with changed lengths and values", asyn // JSON readback avoids the driver's numeric-array NULL parser so this // checks the bound database values, including NULL and an empty array. const query = "select to_json($1::integer[]) as ids" + await client.unsafe(query, [[1]]) await client.unsafe(query, [[1, 2]]) for (const ids of [[3], [4, 5, 6], [], [null, 7]]) { proxy.reset() @@ -135,3 +147,65 @@ test("ordinary array bindings reuse plans with changed lengths and values", asyn } }) }) + + +test("default policy leaves unrelated partial-index predicates unnamed after repeated reads", async () => { + await withClient(async (client, proxy) => { + await client`set plan_cache_mode = auto` + await client`create temporary table prepared_partial_test (id int, stopped boolean)` + await client`insert into prepared_partial_test select id, id > 10 from generate_series(1, 10000) id` + await client`create index prepared_partial_due on prepared_partial_test (id) where stopped = false` + await client`analyze prepared_partial_test` + const query = "select id from prepared_partial_test where stopped = $1 order by id" + for (let run = 0; run < 12; run++) expect(await client.unsafe(query, [false])).toHaveLength(10) + proxy.reset() + expect(await client.unsafe(query, [false])).toHaveLength(10) + expect(proxy.snapshot().frames["H"]).toBe(1) + const retained = await client`select name from pg_prepared_statements where statement = ${query}` + expect(retained).toHaveLength(0) + const plan = await client.unsafe(`explain (format json) ${query}`, [false]) + expect(JSON.stringify(plan)).toContain("prepared_partial_due") + const access = `select $1::int as value ${accessQueryComment("sync.replay")}` + await client.unsafe(access, [1]) + proxy.reset() + expect((await client.unsafe(access, [2]))[0]?.["value"]).toBe(2) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + }, "access") +}) + +test("broad custom plans retain partial indexes and skip Describe after more than five executions", async () => { + await withClient(async (client, proxy) => { + await client`create temporary table prepared_custom_test (id int, stopped boolean)` + await client`insert into prepared_custom_test select id, id > 10 from generate_series(1, 10000) id` + await client`create index prepared_custom_due on prepared_custom_test (id) where stopped = false` + await client`analyze prepared_custom_test` + const query = "select id from prepared_custom_test where stopped = $1 order by id" + for (let run = 0; run < 12; run++) expect(await client.unsafe(query, [false])).toHaveLength(10) + proxy.reset() + expect(await client.unsafe(query, [false])).toHaveLength(10) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + const plans = await client`select name, generic_plans, custom_plans from pg_prepared_statements where statement = ${query}` + expect(plans).toHaveLength(1) + expect(Number(plans[0]?.["generic_plans"])).toBe(0) + expect(Number(plans[0]?.["custom_plans"])).toBeGreaterThan(5) + const name = String(plans[0]?.["name"]).replaceAll('"', '""') + const plan = await client.unsafe(`explain (format json) execute "${name}"(false)`) + expect(JSON.stringify(plan)).toContain("prepared_custom_due") + }) +}) + +test("one-off shapes do not exhaust broad admission and access keeps reserved capacity", async () => { + await withClient(async (client, proxy) => { + for (let i = 0; i < MAX_PREPARED_QUERY_SHAPES + 20; i++) { + await client.unsafe(`select $1::int + ${i} as one_off`, [1]) + } + const hot = "select $1::int as hot" + await client.unsafe(hot, [1]) + await client.unsafe(hot, [2]) + proxy.reset() + expect((await client.unsafe(hot, [3]))[0]?.["hot"]).toBe(3) + expect(proxy.snapshot().exchangeBoundaries).toBe(1) + const plans = await client`select count(*)::int as count from pg_prepared_statements where statement like '%as one_off'` + expect(plans[0]?.["count"]).toBe(0) + }) +}) diff --git a/server/src/db/preparedQueries.ts b/server/src/db/preparedQueries.ts index 8afe10f36..ceade51f9 100644 --- a/server/src/db/preparedQueries.ts +++ b/server/src/db/preparedQueries.ts @@ -1,8 +1,12 @@ import type postgres from "postgres" +import { accessQueryComment } from "./databaseDiagnostics" -// Bound automatically retained SQL/type signatures per pool. Once full, new shapes keep -// postgres.js's unprepared behavior; existing hot shapes still reuse their plans. -export const MAX_PREPARED_QUERY_SHAPES = 256 +// postgres.js retains admitted names until the physical connection closes. Do not +// pretend that evicting an application entry would deallocate those statements. +export const MAX_PREPARED_QUERY_SHAPES = 200 +export const MAX_BROAD_PREPARED_QUERY_SHAPES = 100 +export type PreparedQueryPolicy = { mode: "access" | "all-custom" } +const accessComments = [accessQueryComment("sync.replay"), accessQueryComment("sync.sidecars")] const MAX_QUERY_LENGTH = 32_768 const MAX_PARAMETERS = 301 const installed = new WeakSet() @@ -11,9 +15,14 @@ type TransactionCallback = (sql: postgres.TransactionSql) => T | Promise /** Drizzle uses unsafe(), whose call-level default disables preparation even * when the pool enables it. Apply the same bounded policy to borrowed/transaction * clients, including savepoints. Cache plans only, never rows or authority. + * Broad preparation requires force_custom_plan on every backend serving the + * application role. The connection policy activates it only after startup checks. * Requires protocol-level prepared statement support when using PgBouncer. */ -export function prepareQueries(client: postgres.Sql): void { +export function prepareQueries(client: postgres.Sql, policy: PreparedQueryPolicy = { mode: "access" }): void { const shapes = new Set() + const broadShapes = new Set() + // Only candidates are evicted: they have never created retained statements. + const candidates = new Set() const wrap = (scope: T): T => { if (installed.has(scope)) return scope installed.add(scope) @@ -23,15 +32,29 @@ export function prepareQueries(client: postgres.Sql): void { query.length > MAX_QUERY_LENGTH || !/^\s*(select|insert|update|delete|with)\b/i.test(query)) { return unsafe(query, parameters, options) } + const access = accessComments.some((comment) => query.trimEnd().endsWith(comment)) + if (!access && policy.mode !== "all-custom") return unsafe(query, parameters, options) // Match the driver's ordinary scalar/array type categories. Custom typed // parameters retain their original execution path rather than admitting // unbounded type variants under one SQL-text entry. const types = parameters.map(parameterType) if (types.some((type) => type === undefined)) return unsafe(query, parameters, options) const signature = `${types.join(",")}\0${query}` - const prepare = shapes.has(signature) || shapes.size < MAX_PREPARED_QUERY_SHAPES - if (prepare) shapes.add(signature) - return unsafe(query, parameters, { ...options, prepare }) + if (shapes.has(signature)) return unsafe(query, parameters, { ...options, prepare: true }) + // In broad mode leave half the budget for replay access, even when many + // variable-length IN queries arrive first. Default access mode has all 200. + const capacity = shapes.size < MAX_PREPARED_QUERY_SHAPES && + (access || broadShapes.size < MAX_BROAD_PREPARED_QUERY_SHAPES) + if (!capacity) return unsafe(query, parameters, options) + if (!access && !candidates.has(signature)) { + candidates.add(signature) + if (candidates.size > MAX_PREPARED_QUERY_SHAPES) candidates.delete(candidates.values().next().value!) + return unsafe(query, parameters, options) + } + candidates.delete(signature) + shapes.add(signature) + if (!access) broadShapes.add(signature) + return unsafe(query, parameters, { ...options, prepare: true }) } if (typeof scope.begin === "function") { const begin = scope.begin diff --git a/server/src/modules/dialogOpen.postgres.test.ts b/server/src/modules/dialogOpen.postgres.test.ts index 7b37742bd..ec8a7da29 100644 --- a/server/src/modules/dialogOpen.postgres.test.ts +++ b/server/src/modules/dialogOpen.postgres.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "bun:test" import { asc, eq } from "drizzle-orm" import { db } from "@in/server/db" import { dialogs, users } from "@in/server/db/schema" +import { UsersModel } from "@in/server/db/models/users" import { setupTestLifecycle, testUtils } from "../__tests__/setup" import { measureOperation } from "../__tests__/performance/measure" import { setDialogOpenForUsers } from "./dialogOpen" @@ -23,9 +24,26 @@ test("one hundred unchanged dialogs use a constant query budget and preserve the }, async () => {}) // Active owners, BEGIN, one sorted owner-lock batch, existing dialogs, COMMIT. expect(sample.sql.commands).toBe(5) + const smallBatch = await measureOperation(db.$client.options, async () => { + const result = await setDialogOpenForUsers({ chat, userIds: [actors[0]!.id], open: true }) + expect(result.changedDialogs).toEqual([]) + expect(result.dialogs).toHaveLength(1) + }, async () => {}) + expect(smallBatch.sql.shapes).toEqual(sample.sql.shapes) expect(await db.select().from(dialogs).where(eq(dialogs.chatId, chat.id)).orderBy(asc(dialogs.userId))).toEqual(before) }) +test("array-bound active owners preserve filtering and caller order", async () => { + const first = await testUtils.createUser("array-first@example.test") + const second = await testUtils.createUser("array-second@example.test") + const removed = await testUtils.createUser("array-removed@example.test") + await db.update(users).set({ deleted: true }).where(eq(users.id, removed.id)) + expect(await UsersModel.getActiveUserIds([ + second.id, removed.id, first.id, second.id, 2 ** 40, 0, -1, Number.NaN, + ])).toEqual([second.id, first.id]) + expect(await UsersModel.getActiveUserIds([])).toEqual([]) +}) + test("overlapping reversed owner batches keep independent chat allocations ordered without deadlock", async () => { const actors = await db.insert(users).values(Array.from({ length: 10 }, (_, i) => ({ email: `overlap-${i}@example.test` }))).returning() const first = await testUtils.createChat(null) diff --git a/server/src/modules/dialogOpen.ts b/server/src/modules/dialogOpen.ts index 6e16082c0..ccc99e6dd 100644 --- a/server/src/modules/dialogOpen.ts +++ b/server/src/modules/dialogOpen.ts @@ -2,7 +2,7 @@ import { db } from "@in/server/db" import { UsersModel } from "@in/server/db/models/users" import type { Transaction } from "@in/server/db/types" import { chats, dialogFolders, dialogs, users, type DbChat, type DbDialog, type DbNewDialog } from "@in/server/db/schema" -import { and, asc, desc, eq, inArray, isNotNull, isNull, or } from "drizzle-orm" +import { and, asc, desc, eq, isNotNull, isNull, or, sql } from "drizzle-orm" import { FractionalIndex } from "@in/server/modules/fractionalIndex" export type DialogOpenPlacement = "top" | "bottom" @@ -261,16 +261,17 @@ async function setDialogOpenForUsersInTransaction( // Acquire all user owners before touching any dialog rows. Projection // writers use the same users -> dialogs order, so overlapping batches do - // not deadlock or derive a stale fractional order. + // not deadlock or derive a stale fractional order. Bind each ID list as one + // array so varying recipient counts do not consume new prepared SQL shapes. await tx.select({ id: users.id }).from(users) - .where(inArray(users.id, userIds)) + .where(sql`${users.id} = any(${sql.param(userIds)}::bigint[])`) .orderBy(asc(users.id)) .for("no key update") const existingDialogs = await tx .select() .from(dialogs) - .where(and(eq(dialogs.chatId, input.chat.id), inArray(dialogs.userId, userIds))) + .where(and(eq(dialogs.chatId, input.chat.id), sql`${dialogs.userId} = any(${sql.param(userIds)}::bigint[])`)) const existingUserIds = new Set(existingDialogs.map((dialog) => dialog.userId)) const changedUserIds = new Set() @@ -363,7 +364,7 @@ async function setDialogOpenForUsersInTransaction( order: null, folderId: null, }) - .where(and(eq(dialogs.chatId, input.chat.id), inArray(dialogs.userId, persistCloseUserIds))) + .where(and(eq(dialogs.chatId, input.chat.id), sql`${dialogs.userId} = any(${sql.param(persistCloseUserIds)}::bigint[])`)) changedCloseUserIds.forEach((userId) => changedUserIds.add(userId)) } @@ -388,7 +389,7 @@ async function setDialogOpenForUsersInTransaction( const finalDialogs = await tx .select() .from(dialogs) - .where(and(eq(dialogs.chatId, input.chat.id), inArray(dialogs.userId, userIds))) + .where(and(eq(dialogs.chatId, input.chat.id), sql`${dialogs.userId} = any(${sql.param(userIds)}::bigint[])`)) return { dialogs: finalDialogs, From 2e22dc1b947e1e314e4c61725c5a288014dadb09 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 20:55:51 +0330 Subject: [PATCH 05/14] test: preserve delayed proxy bytes across half closes --- server/src/__tests__/performance/wire.test.ts | 72 +++++++++++++++++++ server/src/__tests__/performance/wire.ts | 59 +++++++++------ 2 files changed, 109 insertions(+), 22 deletions(-) create mode 100644 server/src/__tests__/performance/wire.test.ts diff --git a/server/src/__tests__/performance/wire.test.ts b/server/src/__tests__/performance/wire.test.ts new file mode 100644 index 000000000..9a0bebb7f --- /dev/null +++ b/server/src/__tests__/performance/wire.test.ts @@ -0,0 +1,72 @@ +import { expect, test } from "bun:test" +import { connect, createServer, type Socket } from "node:net" +import { createWireProxy } from "./wire" + +function requestBytes() { + const startup = Buffer.alloc(8) + startup.writeUInt32BE(8) + startup.writeUInt32BE(196608, 4) + const query = Buffer.alloc(5 + 96 * 1024) + query[0] = "Q".charCodeAt(0) + query.writeUInt32BE(query.length - 1, 1) + query.fill(97, 5) + return { startup, query } +} + +test.each(["client", "server"] as const)("delayed proxy preserves queued bytes when %s half-closes first", async (first) => { + const { startup, query } = requestBytes() + const request = Buffer.concat([startup, query]) + const response = Buffer.alloc(256 * 1024, 87) + const received = Promise.withResolvers() + const peers = new Set() + const server = createServer({ allowHalfOpen: true }, (peer) => { + peers.add(peer) + const chunks: Buffer[] = [] + let replied = false + peer.on("data", (chunk: Buffer) => { + chunks.push(chunk) + if (first === "server" && !replied) { replied = true; peer.end(response) } + }) + peer.on("end", () => { + received.resolve(Buffer.concat(chunks)) + if (first === "client") peer.end(response) + }) + peer.on("error", received.reject) + peer.on("close", () => peers.delete(peer)) + }) + await new Promise((resolve, reject) => { + server.once("error", reject) + server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve() }) + }) + const address = server.address() + if (!address || typeof address === "string") throw new Error("Missing fixture port") + const proxy = await createWireProxy({ host: "127.0.0.1", port: address.port }, 50) + const client = connect({ host: "127.0.0.1", port: proxy.port, allowHalfOpen: true }) + const reply = new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + client.on("data", (chunk: Buffer) => chunks.push(chunk)) + client.on("error", reject) + client.on("end", () => { + // Even after receiving the server's EOF, the request direction remains + // writable until the client explicitly finishes its remaining bytes. + if (first === "server") client.end(query) + resolve(Buffer.concat(chunks)) + }) + client.on("close", () => { + if (!client.readableEnded) reject(new Error("Proxy closed before response EOF")) + }) + }) + try { + if (first === "client") client.end(request) + else client.write(startup) + const [actualRequest, actualResponse] = await Promise.all([received.promise, reply]) + expect(actualRequest.equals(request)).toBe(true) + expect(actualResponse.equals(response)).toBe(true) + expect(proxy.snapshot()).toMatchObject({ clientBytes: request.length, serverBytes: response.length, frames: { Q: 1 } }) + } finally { + client.destroy() + await proxy.close() + for (const peer of peers) peer.destroy() + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())) + } +}) diff --git a/server/src/__tests__/performance/wire.ts b/server/src/__tests__/performance/wire.ts index d4014cf10..111e0b1fd 100644 --- a/server/src/__tests__/performance/wire.ts +++ b/server/src/__tests__/performance/wire.ts @@ -51,37 +51,52 @@ export async function createWireProxy(target: { host: string; port: number }, rt let counts: WireCounts const reset = () => { counts = { frames: {}, exchangeBoundaries: 0, clientBytes: 0, serverBytes: 0 } } reset() - const server = createServer((client) => { - const upstream = connect({ host: target.host, port: target.port }) + const server = createServer({ allowHalfOpen: true }, (client) => { + const upstream = connect({ host: target.host, port: target.port, allowHalfOpen: true }) for (const socket of [client, upstream]) { sockets.add(socket); socket.setNoDelay(true) } const frames = new FrontendFrames((kind) => { counts.frames[kind] = (counts.frames[kind] ?? 0) + 1 if (kind === "Q" || kind === "H" || kind === "S") counts.exchangeBoundaries++ }) - const forward = (source: Socket, destination: Socket, bytes: Buffer) => { - source.pause() - const write = () => { - if (destination.destroyed) return - destination.write(bytes, () => { if (!source.destroyed) source.resume() }) - } - if (!delayMs) write() - else { - const timer = setTimeout(() => { timers.delete(timer); write() }, delayMs / 2) - timers.add(timer) + const forward = (source: Socket, destination: Socket, observe: (bytes: Buffer) => void) => { + let pending = 0 + let ended = false + const finish = () => { + // Reading EOF must not end the destination before its delayed writes. + if (ended && !pending && !destination.destroyed) destination.end() } + source.on("data", (bytes: Buffer) => { + try { observe(bytes) } catch (error) { failures.push(error as Error); source.destroy(); return } + source.pause() + pending++ + const write = () => { + if (destination.destroyed) { pending--; return } + destination.write(bytes, () => { + pending-- + finish() + if (!source.destroyed) source.resume() + }) + } + if (!delayMs) write() + else { + const timer = setTimeout(() => { timers.delete(timer); write() }, delayMs / 2) + timers.add(timer) + } + }) + source.on("end", () => { ended = true; finish() }) } - client.on("data", (bytes: Buffer) => { - try { frames.accept(bytes) } catch (error) { failures.push(error as Error); client.destroy(); return } - counts.clientBytes += bytes.length - forward(client, upstream, bytes) - }) - upstream.on("data", (bytes: Buffer) => { counts.serverBytes += bytes.length; forward(upstream, client, bytes) }) + forward(client, upstream, (bytes) => { frames.accept(bytes); counts.clientBytes += bytes.length }) + forward(upstream, client, (bytes) => { counts.serverBytes += bytes.length }) client.on("error", (error) => { failures.push(error); upstream.destroy() }) upstream.on("error", (error) => { failures.push(error); client.destroy() }) - client.on("end", () => upstream.end()) - upstream.on("end", () => client.end()) - client.on("close", () => { sockets.delete(client); upstream.destroy() }) - upstream.on("close", () => { sockets.delete(upstream); client.destroy() }) + for (const [socket, peer] of [[client, upstream], [upstream, client]] as const) { + socket.on("close", () => { + sockets.delete(socket) + // A cleanly closed source can still have delayed bytes headed to its + // peer. Abort the pair only for an incomplete/failed connection. + if (!socket.readableEnded || !socket.writableFinished) peer.destroy() + }) + } }) await new Promise((resolve, reject) => { server.once("error", reject) From f2951b20518135e7792bc8acfedea82498c9f8f6 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 21:12:19 +0330 Subject: [PATCH 06/14] server: harden cache recovery and webhook claim ownership --- server/src/db/models/botUpdates.test.ts | 83 +++++++++++++++++++ server/src/db/models/botUpdates.ts | 26 ++++++ .../src/modules/botUpdates/delivery.test.ts | 22 +++++ server/src/modules/botUpdates/delivery.ts | 74 +++++++++++------ server/src/modules/cache/cluster.test.ts | 71 +++++++++++++++- server/src/modules/cache/cluster.ts | 5 +- .../modules/internalMessaging/repair.test.ts | 25 ++++++ .../src/modules/internalMessaging/repair.ts | 7 ++ 8 files changed, 285 insertions(+), 28 deletions(-) diff --git a/server/src/db/models/botUpdates.test.ts b/server/src/db/models/botUpdates.test.ts index 00938da4f..97152dd91 100644 --- a/server/src/db/models/botUpdates.test.ts +++ b/server/src/db/models/botUpdates.test.ts @@ -6,6 +6,7 @@ import { db } from "@in/server/db" import { botUpdates, botUpdateStreams, chatParticipants } from "@in/server/db/schema" import { markServerShuttingDown, resetServerShutdownStateForTests } from "@in/server/lifecycle/shutdownState" import { botUpdateWaiters } from "./botUpdateWaiters" +import { makeBotWebhookTransport, runBotWebhookDeliveryOnce } from "@in/server/modules/botUpdates/delivery" import { BOT_UPDATE_QUEUE_LIMITS, BotUpdatesModel, @@ -127,8 +128,10 @@ describe("BotUpdatesModel", () => { const update = await queueParticipation(botUserId, chatId, "replace") const [stale] = await BotUpdatesModel.claimWebhookDeliveries(1) expect(stale?.update.update_id).toBe(update?.update_id) + expect(await BotUpdatesModel.isCurrentWebhookClaim(stale!)).toBeTrue() await BotUpdatesModel.setWebhook(botUserId, { url: "https://example.com/replacement" }) + expect(await BotUpdatesModel.isCurrentWebhookClaim(stale!)).toBeFalse() expect(await BotUpdatesModel.markWebhookDelivered(stale!)).toBeFalse() const [replacement] = await BotUpdatesModel.claimWebhookDeliveries(1) @@ -149,6 +152,86 @@ describe("BotUpdatesModel", () => { const [recovered] = await BotUpdatesModel.claimWebhookDeliveries(1) expect(recovered?.update.update_id).toBe(update?.update_id) expect(recovered?.claimToken).not.toBe(abandoned?.claimToken) + expect(await BotUpdatesModel.isCurrentWebhookClaim(abandoned!)).toBeFalse() + expect(await BotUpdatesModel.isCurrentWebhookClaim(recovered!)).toBeTrue() + }) + + for (const change of ["replaced", "deleted", "expired lease", "reclaimed", "expired update", "access removed"] as const) { + test(`does not open a webhook POST when ${change} during DNS resolution`, async () => { + await BotUpdatesModel.setWebhook(botUserId, { url: "https://receiver.test/old", secret_token: "old-secret" }) + const update = await queueParticipation(botUserId, chatId, "paused DNS") + const resolving = Promise.withResolvers() + const releaseDns = Promise.withResolvers() + let posts = 0 + const delivery = runBotWebhookDeliveryOnce(1, makeBotWebhookTransport({ + resolve: async (rawUrl) => { + resolving.resolve() + await releaseDns.promise + return { url: new URL(rawUrl), address: "8.8.8.8", family: 4 } + }, + request: () => { + posts++ + throw new Error("A stale webhook must not open a connection") + }, + })) + try { + await resolving.promise + const [row] = await db.select().from(botUpdates).where(eq(botUpdates.botUserId, botUserId)) + if (!row?.claimToken) throw new Error("Expected an active webhook claim before DNS") + if (change === "replaced") { + await BotUpdatesModel.setWebhook(botUserId, { url: "https://receiver.test/new", secret_token: "new-secret" }) + } else if (change === "deleted") { + await BotUpdatesModel.deleteWebhook(botUserId) + } else if (change === "expired lease" || change === "reclaimed") { + await db.update(botUpdates).set({ claimExpiresAt: new Date(Date.now() - 1_000) }).where(eq(botUpdates.id, row.id)) + if (change === "reclaimed") { + const [replacement] = await BotUpdatesModel.claimWebhookDeliveries(1) + expect(replacement?.claimToken).not.toBe(row.claimToken) + } + } else if (change === "expired update") { + await db.update(botUpdates).set({ expiresAt: new Date(Date.now() - 1_000) }).where(eq(botUpdates.id, row.id)) + } else { + await db.delete(chatParticipants).where(and(eq(chatParticipants.chatId, chatId), eq(chatParticipants.userId, botUserId))) + } + releaseDns.resolve() + expect(await delivery).toBe(1) + expect(posts).toBe(0) + const pending = await db.select().from(botUpdates).where(eq(botUpdates.botUserId, botUserId)) + if (change === "access removed") { + expect(pending).toHaveLength(0) + expect((await streamFor(botUserId)).droppedUpdateCount).toBe(1) + } else { + expect(pending).toHaveLength(1) + expect(pending[0]?.updateId).toBe(update?.update_id) + expect(pending[0]?.attemptCount).toBe(0) + expect((await streamFor(botUserId)).pendingUpdateCount).toBe(1) + } + } finally { + releaseDns.resolve() + await delivery + } + }) + } + + test("delivers a current claim and fences its ACK if the webhook changes after connection", async () => { + await BotUpdatesModel.setWebhook(botUserId, { url: "https://receiver.test/old", secret_token: "old-secret" }) + const update = await queueParticipation(botUserId, chatId, "in flight") + const posts: { url: string; updateId?: string; secret?: string }[] = [] + expect(await runBotWebhookDeliveryOnce(1, async (input) => { + if (!await input.beforeConnect()) return undefined + posts.push({ url: input.rawUrl, updateId: input.headers["x-inline-update-id"], secret: input.headers["x-inline-bot-api-secret-token"] }) + await BotUpdatesModel.setWebhook(botUserId, { url: "https://receiver.test/new", secret_token: "new-secret" }) + return { status: 200 } + })).toBe(1) + expect(posts).toEqual([{ url: "https://receiver.test/old", updateId: String(update?.update_id), secret: "old-secret" }]) + expect((await streamFor(botUserId)).pendingUpdateCount).toBe(1) + expect(await runBotWebhookDeliveryOnce(1, async (input) => { + if (!await input.beforeConnect()) return undefined + posts.push({ url: input.rawUrl, updateId: input.headers["x-inline-update-id"], secret: input.headers["x-inline-bot-api-secret-token"] }) + return { status: 200 } + })).toBe(1) + expect(posts.at(-1)).toEqual({ url: "https://receiver.test/new", updateId: String(update?.update_id), secret: "new-secret" }) + expect((await streamFor(botUserId)).pendingUpdateCount).toBe(0) }) test("lets a replacement poll proceed and cancels the older long poll", async () => { diff --git a/server/src/db/models/botUpdates.ts b/server/src/db/models/botUpdates.ts index f4fa971fa..bf335baa6 100644 --- a/server/src/db/models/botUpdates.ts +++ b/server/src/db/models/botUpdates.ts @@ -531,6 +531,31 @@ async function claimWebhookDeliveries(limit: number): Promise { + if (!claim.stream.webhookUrl) return false + const [current] = await db + .select({ id: botUpdates.id }) + .from(botUpdates) + .innerJoin(botUpdateStreams, eq(botUpdateStreams.botUserId, botUpdates.botUserId)) + .where(and( + eq(botUpdates.id, claim.updateRowId), + eq(botUpdates.botUserId, claim.stream.botUserId), + eq(botUpdates.updateId, claim.update.update_id), + eq(botUpdates.claimToken, claim.claimToken), + eq(botUpdates.claimGeneration, claim.claimGeneration), + // These columns store UTC without a time zone; comparing directly to + // timestamptz would reinterpret them in the database session's zone. + gt(botUpdates.claimExpiresAt, sql`clock_timestamp() at time zone 'UTC'`), + gt(botUpdates.expiresAt, sql`clock_timestamp() at time zone 'UTC'`), + gt(botUpdates.updateId, botUpdateStreams.acknowledgedUpdateId), + eq(botUpdateStreams.configGeneration, claim.claimGeneration), + eq(botUpdateStreams.webhookUrl, claim.stream.webhookUrl), + )) + .limit(1) + return current !== undefined +} + async function finishWebhookDelivery( claim: WebhookDeliveryClaim, options: { dropped: boolean }, @@ -905,6 +930,7 @@ export const BotUpdatesModel = { getWebhookInfo, claimWebhookDeliveries, decryptWebhookSecret, + isCurrentWebhookClaim, markWebhookDelivered, markWebhookFailed, queue, diff --git a/server/src/modules/botUpdates/delivery.test.ts b/server/src/modules/botUpdates/delivery.test.ts index 249483a1d..bcf5c7723 100644 --- a/server/src/modules/botUpdates/delivery.test.ts +++ b/server/src/modules/botUpdates/delivery.test.ts @@ -1,10 +1,32 @@ import { describe, expect, it } from "bun:test" import { BotWebhookDeliveryWorker, + makeBotWebhookTransport, webhookRetryDelayMs, } from "./delivery" describe("bot webhook delivery", () => { + it("checks delivery authority after DNS and before opening an HTTPS request", async () => { + const steps: string[] = [] + const transport = makeBotWebhookTransport({ + resolve: async (rawUrl) => { + steps.push("dns") + return { url: new URL(rawUrl), address: "8.8.8.8", family: 4 } + }, + request: () => { + steps.push("request") + throw new Error("synthetic request boundary") + }, + }) + const input = { rawUrl: "https://receiver.test/webhook", body: "{}", headers: {} } + expect(await transport({ ...input, beforeConnect: async () => { steps.push("authority"); return false } })).toBeUndefined() + expect(steps).toEqual(["dns", "authority"]) + steps.length = 0 + await expect(transport({ ...input, beforeConnect: async () => { steps.push("authority"); return true } })) + .rejects.toThrow("synthetic request boundary") + expect(steps).toEqual(["dns", "authority", "request"]) + }) + it("uses bounded increasing retry delays", () => { const original = Math.random Math.random = () => 0 diff --git a/server/src/modules/botUpdates/delivery.ts b/server/src/modules/botUpdates/delivery.ts index 9954b9825..d2b1fb3a9 100644 --- a/server/src/modules/botUpdates/delivery.ts +++ b/server/src/modules/botUpdates/delivery.ts @@ -2,7 +2,8 @@ import { BotUpdatesModel } from "@in/server/db/models/botUpdates" import { isTest } from "@in/server/env" import { Log } from "@in/server/utils/log" import { resolveWebhookUrl } from "./webhookSecurity" -import { request as httpsRequest } from "node:https" +import { request as httpsRequest, type RequestOptions } from "node:https" +import type { ClientRequest, IncomingMessage } from "node:http" const log = new Log("botUpdates.delivery") const requestTimeoutMs = 10_000 @@ -21,35 +22,46 @@ const retryAfterMs = (status: number, header: string | string[] | undefined): nu return Math.min(3_600_000, Math.floor(seconds * 1_000)) } -const postPinned = async (input: { +export type BotWebhookTransport = (input: { rawUrl: string body: string headers: Record -}): Promise<{ status: number; retryAfter?: string | string[] }> => { - const target = await resolveWebhookUrl(input.rawUrl) - return new Promise((resolve, reject) => { - const request = httpsRequest(target.url, { - method: "POST", - headers: { ...input.headers, "content-length": Buffer.byteLength(input.body).toString() }, - servername: target.url.hostname, - lookup: ((_hostname: string, _options: unknown, callback: (error: Error | null, address: string, family: number) => void) => - callback(null, target.address, target.family)) as any, - }, (response) => { - let bytes = 0 - response.on("data", (chunk: Buffer) => { - bytes += chunk.byteLength - if (bytes > 4_096) response.destroy() + beforeConnect: () => Promise +}) => Promise<{ status: number; retryAfter?: string | string[] } | undefined> + +export function makeBotWebhookTransport(dependencies: { + resolve?: typeof resolveWebhookUrl + request?: (url: URL, options: RequestOptions, callback: (response: IncomingMessage) => void) => ClientRequest +} = {}): BotWebhookTransport { + return async (input) => { + const target = await (dependencies.resolve ?? resolveWebhookUrl)(input.rawUrl) + if (!await input.beforeConnect()) return undefined + return new Promise((resolve, reject) => { + const request = (dependencies.request ?? httpsRequest)(target.url, { + method: "POST", + headers: { ...input.headers, "content-length": Buffer.byteLength(input.body).toString() }, + servername: target.url.hostname, + lookup: ((_hostname: string, _options: unknown, callback: (error: Error | null, address: string, family: number) => void) => + callback(null, target.address, target.family)) as NonNullable, + }, (response) => { + let bytes = 0 + response.on("data", (chunk: Buffer) => { + bytes += chunk.byteLength + if (bytes > 4_096) response.destroy() + }) + response.on("end", () => resolve({ status: response.statusCode ?? 0, retryAfter: response.headers["retry-after"] })) + response.on("error", reject) }) - response.on("end", () => resolve({ status: response.statusCode ?? 0, retryAfter: response.headers["retry-after"] })) - response.on("error", reject) + request.setTimeout(requestTimeoutMs, () => request.destroy(new Error("Webhook request timed out"))) + request.on("error", reject) + request.end(input.body) }) - request.setTimeout(requestTimeoutMs, () => request.destroy(new Error("Webhook request timed out"))) - request.on("error", reject) - request.end(input.body) - }) + } } -async function deliver(claim: Awaited>[number]) { +const postPinned = makeBotWebhookTransport() + +async function deliver(claim: Awaited>[number], transport: BotWebhookTransport) { const url = claim.stream.webhookUrl if (!url) return try { @@ -70,7 +82,17 @@ async function deliver(claim: Awaited { + if (!await BotUpdatesModel.isCurrentWebhookClaim(claim)) return false + if (!await BotUpdatesModel.canBotAccessUpdate(claim.stream.botUserId, claim.update)) { + await BotUpdatesModel.discardWebhookDelivery(claim) + return false + } + // Access reads can cross replacement or lease expiry. Fence once more + // after those reads, immediately before opening the external connection. + return BotUpdatesModel.isCurrentWebhookClaim(claim) + } }) + if (!response) return if (response.status >= 200 && response.status < 300) { await BotUpdatesModel.markWebhookDelivered(claim) return @@ -90,13 +112,13 @@ async function deliver(claim: Awaited { +export async function runBotWebhookDeliveryOnce(limit = 25, transport: BotWebhookTransport = postPinned): Promise { await Promise.all([ BotUpdatesModel.cleanupBotUpdateRows(), BotUpdatesModel.cleanupBotMessageRoutes(), ]) const claims = await BotUpdatesModel.claimWebhookDeliveries(limit) - await Promise.all(claims.map(deliver)) + await Promise.all(claims.map((claim) => deliver(claim, transport))) return claims.length } diff --git a/server/src/modules/cache/cluster.test.ts b/server/src/modules/cache/cluster.test.ts index e15209f57..82b010da6 100644 --- a/server/src/modules/cache/cluster.test.ts +++ b/server/src/modules/cache/cluster.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it, spyOn } from "bun:test" +import { UserSettingsModel } from "@in/server/db/models/userSettings" +import { UserSettingsGeneralSchema } from "@in/server/db/models/userSettings/types" +import { internalMessaging } from "@in/server/modules/internalMessaging/service" import { connectionManager } from "@in/server/ws/connections" -import { refreshSpaceMemberships } from "./cluster" +import { clearClusterCaches, refreshSpaceMemberships, subscribeClusterCaches } from "./cluster" +import { getCachedUserSettings } from "./userSettings" describe("refreshSpaceMemberships", () => { it("keeps one AccessChanged event at sixteen membership refreshes", async () => { @@ -22,3 +26,68 @@ describe("refreshSpaceMemberships", () => { } }) }) + +describe("cluster cache recovery", () => { + for (const inFlight of [false, true]) { + it(`refreshes a ${inFlight ? "pending" : "completed"} outage refill after broker recovery`, async () => { + const readyListeners = new Set<() => void>() + const continuityListeners = new Set<() => void>() + const ready = spyOn(internalMessaging, "onReady").mockImplementation((listener) => { + readyListeners.add(listener) + return () => { readyListeners.delete(listener) } + }) + const continuity = spyOn(internalMessaging, "onContinuityLost").mockImplementation((listener) => { + continuityListeners.add(listener) + return () => { continuityListeners.delete(listener) } + }) + const before = UserSettingsGeneralSchema.parse({ privacy: { shareTimeZone: true } }) + const after = UserSettingsGeneralSchema.parse({ privacy: { shareTimeZone: false } }) + const refillStarted = Promise.withResolvers() + const releaseRefill = Promise.withResolvers() + let current = before + let reads = 0 + const getGeneral = spyOn(UserSettingsModel, "getGeneral").mockImplementation(async () => { + if (++reads === 2 && inFlight) { + refillStarted.resolve() + return releaseRefill.promise + } + return current + }) + clearClusterCaches() + const unsubscribe = subscribeClusterCaches() + try { + const userId = 90420 + await expect(getCachedUserSettings(userId)).resolves.toEqual(before) + + // The lost connection clears the old value, but reads during the gap + // can refill it before a remote mutation's invalidation is missed. + for (const listener of continuityListeners) listener() + const refill = getCachedUserSettings(userId) + if (inFlight) await refillStarted.promise + else await expect(refill).resolves.toEqual(before) + current = after + if (!inFlight) { + await expect(getCachedUserSettings(userId)).resolves.toEqual(before) + } + + for (const listener of readyListeners) listener() + releaseRefill.resolve(before) + if (inFlight) await expect(refill).resolves.toEqual(after) + await expect(getCachedUserSettings(userId)).resolves.toEqual(after) + await expect(getCachedUserSettings(userId)).resolves.toEqual(after) + expect(getGeneral).toHaveBeenCalledTimes(3) + + unsubscribe() + expect(readyListeners.size).toBe(0) + expect(continuityListeners.size).toBe(0) + } finally { + releaseRefill.resolve(before) + unsubscribe() + ready.mockRestore() + continuity.mockRestore() + getGeneral.mockRestore() + clearClusterCaches() + } + }) + } +}) diff --git a/server/src/modules/cache/cluster.ts b/server/src/modules/cache/cluster.ts index f98bf7400..4f69fef1f 100644 --- a/server/src/modules/cache/cluster.ts +++ b/server/src/modules/cache/cluster.ts @@ -128,5 +128,8 @@ export function subscribeClusterCaches(): () => void { } }) const removeContinuity = internalMessaging.onContinuityLost(clearClusterCaches) - return () => { removeCache(); removeAccess(); removeContinuity() } + // Reads can refill a cache during an outage and then miss another process's + // invalidation. Clear again when subscriptions recover, fencing those fills. + const removeReady = internalMessaging.onReady(clearClusterCaches) + return () => { removeCache(); removeAccess(); removeContinuity(); removeReady() } } diff --git a/server/src/modules/internalMessaging/repair.test.ts b/server/src/modules/internalMessaging/repair.test.ts index 0dab595c8..e2eca354a 100644 --- a/server/src/modules/internalMessaging/repair.test.ts +++ b/server/src/modules/internalMessaging/repair.test.ts @@ -85,6 +85,31 @@ const scan = async (repair: ConnectedUserRepair, userId = 71) => { } describe("ConnectedUserRepair", () => { + test("idle sweeps avoid database fences and a following admission still repairs", async () => { + const { runtime, hinted, requestedDates } = createRuntime({ states: [{ date: 1n, seq: 9 }] }) + let connected = false + runtime.connectedUserIds = () => connected ? [71] : [] + runtime.hasConnections = () => connected + const capture = spyOn(runtime.discovery, "captureWatermark") + const repair = new ConnectedUserRepair(runtime) + repairs.add(repair) + await repair.start() + expect(capture).toHaveBeenCalledTimes(1) + + repair.observeConnectedUsers() + await repair.waitForIdle() + repair.observeConnectedUsers() + // Admission can arrive before the skipped sweep's completion is observed. + connected = true + repair.observeConnection(71) + await repair.waitForIdle() + + expect(capture).toHaveBeenCalledTimes(2) + expect(requestedDates).toHaveLength(1) + expect(hinted).toEqual([9]) + capture.mockRestore() + }) + test("joins preparation on stop and never discovers after it finishes", async () => { const started = Promise.withResolvers() const release = Promise.withResolvers() diff --git a/server/src/modules/internalMessaging/repair.ts b/server/src/modules/internalMessaging/repair.ts index ecea17229..c2e2033a9 100644 --- a/server/src/modules/internalMessaging/repair.ts +++ b/server/src/modules/internalMessaging/repair.ts @@ -190,6 +190,13 @@ export class ConnectedUserRepair { } private async runPeriodicSweep(generation: number): Promise { + // An idle standby has nothing to repair. Socket admissions independently + // capture a watermark, so a connection arriving after this check is covered. + this.pruneDisconnectedObservations() + if (this.repairRuntime.connectedUserIds().length === 0) { + this.connectedUserCursor = 0 + return + } const watermark = await this.repairRuntime.discovery.captureWatermark() assertDiscoveryWatermark(watermark) if (!this.isActive(generation)) return From ba66a87f2c29805934a9caddb6716bcbfccb8824 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 21:12:19 +0330 Subject: [PATCH 07/14] server: reduce history reads and qualify cross-node recovery --- server/BENCHMARKING.md | 41 ++- .../functions/historySnapshots.test.ts | 24 +- server/src/__tests__/performance/catalog.ts | 14 +- server/src/__tests__/performance/scenarios.ts | 2 + .../__tests__/performance/userReplay.test.ts | 53 ++++ .../performance/userReplayScenario.ts | 77 ++++++ .../v3OriginRecovery.test.ts | 249 ++++++++++++++++++ .../v3OriginRecovery.worker.ts | 54 ++++ .../message/historySnapshot.postgres.test.ts | 74 ++++++ server/src/modules/message/historySnapshot.ts | 36 ++- 10 files changed, 602 insertions(+), 22 deletions(-) create mode 100644 server/src/__tests__/performance/userReplay.test.ts create mode 100644 server/src/__tests__/performance/userReplayScenario.ts create mode 100644 server/src/modules/internalMessaging/v3OriginRecovery.test.ts create mode 100644 server/src/modules/internalMessaging/v3OriginRecovery.worker.ts create mode 100644 server/src/modules/message/historySnapshot.postgres.test.ts diff --git a/server/BENCHMARKING.md b/server/BENCHMARKING.md index 7ef0e8561..e8a3476eb 100644 --- a/server/BENCHMARKING.md +++ b/server/BENCHMARKING.md @@ -125,12 +125,14 @@ cardinality and multi-server delivery need separate experiments. ## Coverage and budgets -The initial catalog has 24 scenarios: +The catalog has 26 scenarios: - DM send with open/closed dialogs and an idempotent retry. - Public-thread and reply-thread sends with 1, 10 and 100 recipients. - Chat lists with 1, 10 and 100 DMs, and a 50-message history page. - Empty/nonempty chat replay and a fresh discovery checkpoint. +- User-bucket replay of 100 private-chat snapshots, including a warmed-grant + revocation case with 99 visible chats and an accounted skipped sequence. - User update batches and unchanged dialog batches at 1, 10 and 100 users. - Read-position advancement and its unchanged case. @@ -258,3 +260,40 @@ shapes; inspect `pg_prepared_statements` counters to verify prepared reuse and p Role-wide custom planning also applies to the previously tuned access queries. Qualify representative replay latency as well as simple point reads before enabling broad mode; faster protocol exchanges alone do not prove a net gain. + + +## User replay planning comparison + +The older `updates.replay.empty` and `updates.replay.message` cases replay a +**chat** bucket. They do not qualify the access planning policy for user replay. +Use the dedicated 100-chat user-page cases to exercise both `sync.replay` and +`sync.sidecars` with their existing parameterized recursive access projection: + +```sh +bun run bench:backend --preparation access --scenario updates.user-replay.100 --scenario updates.user-replay.revoked.100 --rtt 0,6 --warmup 3 --samples 20 --output .test-results/user-access.json +bun run bench:backend --preparation all-custom --scenario updates.user-replay.100 --scenario updates.user-replay.revoked.100 --rtt 0,6 --warmup 3 --samples 20 --compare .test-results/user-access.json --output .test-results/user-custom.json +``` + +Each iteration seeds a valid encrypted user journal, current Space membership, +private-thread grants, and retained dialogs. The full case performs two 100-chat +access batches (301 bindings each). The revocation case first reads the complete +page, removes one direct grant, and measures replay with all 100 historical +references; sidecar access receives the 99 surviving chats (298 bindings). +Assertions cover the complete response, including the embedded snapshot title, +sidecars, cursor, skipped sequence, and retained obsolete dialog. A separate +regression removes Space membership after a warm read and checks that every +historical snapshot is filtered while all 100 sequences remain accounted. + +These new scenario IDs retain fixture version 2 and do not change existing +scenario meanings or baselines. They qualify this synthetic top-level private- +thread distribution only. Production graph depth, grant skew, large membership +sets, and planning/execution cost still need representative read-only evidence +before changing a role-wide plan policy. The local fixture's database-specific +custom-plan default does not change the provisioning role or production settings. + + +Explicit-chat history requests read the chat once inside their existing +repeatable-read authorization snapshot. The `history.50` fixture now costs ten +SQL commands, down from eleven; user/self peer resolution and DM creation keep +their existing path. Snapshot tests cover concurrent edits and deletion, current +membership, outsider rejection, and unchanged missing-chat errors. diff --git a/server/src/__tests__/functions/historySnapshots.test.ts b/server/src/__tests__/functions/historySnapshots.test.ts index 75192754e..b4a7412b9 100644 --- a/server/src/__tests__/functions/historySnapshots.test.ts +++ b/server/src/__tests__/functions/historySnapshots.test.ts @@ -29,7 +29,7 @@ import { AccessGuards } from "@in/server/modules/authorization/accessGuards" import { encryptMessage, encryptMessageEntities } from "@in/server/modules/encryption/encryptMessage" import { UpdatesModel } from "@in/server/db/models/updates" import { UpdateBucket } from "@in/server/db/schema/updates" -import { MessageEntities, MessageEntity_Type, SearchMessagesFilter } from "@inline-chat/protocol/core" +import { MessageEntities, MessageEntity_Type, SearchMessagesFilter, RpcError_Code } from "@inline-chat/protocol/core" import { and, asc, eq, sql } from "drizzle-orm" setupTestLifecycle() @@ -113,6 +113,28 @@ for (const readKind of ["history", "resources", "lookup", "text", "chat"] as con }) } +test("an explicit-chat history snapshot survives a concurrent chat deletion and the next read rejects", async () => { + const { user, chat, peerId, context } = await fixture() + await testUtils.createTestMessage({ chatId: chat.id, fromId: user.id, messageId: 1, text: "snapshot before deletion" }) + await db.update(chats).set({ updateSeq: 7, lastMsgId: 1, messageIdCounter: 1 }).where(eq(chats.id, chat.id)) + const ensureAccess = AccessGuards.ensureChatAccess + let deleted = false + const guard = spyOn(AccessGuards, "ensureChatAccess").mockImplementation(async (row, userId, tx) => { + await ensureAccess(row, userId, tx) + if (tx && row.id === chat.id && !deleted) { + deleted = true + await db.delete(chats).where(eq(chats.id, chat.id)) + } + }) + try { + const result = await getChatHistory({ peerId }, context) + expect(deleted).toBe(true) + expect(result.seq).toBe(7n) + expect(result.messages.map((row) => row.message)).toEqual(["snapshot before deletion"]) + await expect(getChatHistory({ peerId }, context)).rejects.toMatchObject({ code: RpcError_Code.CHAT_ID_INVALID }) + } finally { guard.mockRestore() } +}) + test("unsupported historical file rows stay readable and are omitted before resource paging", async () => { const { user, chat, peerId, context } = await fixture() await db.insert(files).values([ diff --git a/server/src/__tests__/performance/catalog.ts b/server/src/__tests__/performance/catalog.ts index 3807a0c8b..eb13bae96 100644 --- a/server/src/__tests__/performance/catalog.ts +++ b/server/src/__tests__/performance/catalog.ts @@ -1,8 +1,8 @@ export type ScenarioSpec = { id: string - kind: "sendDm" | "sendThread" | "getChats" | "history" | "replay" | "checkpoint" | "enqueue" | "dialog" | "read" + kind: "sendDm" | "sendThread" | "getChats" | "history" | "replay" | "userReplay" | "checkpoint" | "enqueue" | "dialog" | "read" size: number - variant?: "closed" | "retry" | "reply" | "empty" | "noop" + variant?: "closed" | "retry" | "reply" | "empty" | "noop" | "revoked" maxCommands: number } @@ -12,11 +12,10 @@ export type ScenarioSpec = { export const scenarioVersion = 2 // Coherent reads submit BEGIN, SET TRANSACTION and COMMIT. Catalog discovery -// additionally prepares missing dialogs and checks candidate access; history -// rereads its chat inside the snapshot. These fixed costs do not scale with rows. +// additionally prepares missing dialogs and checks candidate access. History +// reads its explicit chat once inside the snapshot. Fixed costs do not scale with rows. const snapshotCommands = 3 const catalogAdmissionCommands = 2 -const historySnapshotChatCommands = 1 export const scenarios: readonly ScenarioSpec[] = [ { id: "send.dm.open", kind: "sendDm", size: 1, maxCommands: 25 }, { id: "send.dm.closed", kind: "sendDm", size: 1, variant: "closed", maxCommands: 42 }, @@ -24,9 +23,12 @@ export const scenarios: readonly ScenarioSpec[] = [ ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `send.public.${size}`, kind: "sendThread", size, maxCommands: 32 })), ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `send.reply.${size}`, kind: "sendThread", size, variant: "reply", maxCommands: 66 + 7 * size })), ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `getChats.${size}`, kind: "getChats", size, maxCommands: 11 + snapshotCommands + catalogAdmissionCommands })), - { id: "history.50", kind: "history", size: 50, maxCommands: 7 + snapshotCommands + historySnapshotChatCommands }, + { id: "history.50", kind: "history", size: 50, maxCommands: 7 + snapshotCommands }, { id: "updates.replay.empty", kind: "replay", size: 0, variant: "empty", maxCommands: 7 }, { id: "updates.replay.message", kind: "replay", size: 1, maxCommands: 23 }, + // New IDs preserve the existing fixture version and prior scenario meanings. + { id: "updates.user-replay.100", kind: "userReplay", size: 100, maxCommands: 17 }, + { id: "updates.user-replay.revoked.100", kind: "userReplay", size: 100, variant: "revoked", maxCommands: 17 }, { id: "updates.checkpoint", kind: "checkpoint", size: 1, maxCommands: 6 }, ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `updates.enqueue.${size}`, kind: "enqueue", size, maxCommands: 4 * size + 2 })), ...[1, 10, 100].map((size): ScenarioSpec => ({ id: `dialogs.noop.${size}`, kind: "dialog", size, maxCommands: 5 })), diff --git a/server/src/__tests__/performance/scenarios.ts b/server/src/__tests__/performance/scenarios.ts index e341539df..7602fd9d2 100644 --- a/server/src/__tests__/performance/scenarios.ts +++ b/server/src/__tests__/performance/scenarios.ts @@ -18,6 +18,7 @@ import { decryptMessage, encryptMessage } from "@in/server/modules/encryption/en import { desktopPushSuppressionTracker } from "@in/server/modules/notifications/desktopPushSuppression" import { cleanDatabase, testUtils } from "../setup" import type { ScenarioSpec } from "./catalog" +import { prepareUserReplayScenario } from "./userReplayScenario" export type PreparedOperation = { run(): Promise; verify(): Promise } @@ -176,6 +177,7 @@ export async function prepareScenario(spec: ScenarioSpec): Promise getUpdatesState({}, a.context), async (result) => { diff --git a/server/src/__tests__/performance/userReplay.test.ts b/server/src/__tests__/performance/userReplay.test.ts new file mode 100644 index 000000000..46893ac6a --- /dev/null +++ b/server/src/__tests__/performance/userReplay.test.ts @@ -0,0 +1,53 @@ +import { expect, test } from "bun:test" +import { eq } from "drizzle-orm" +import { GetUpdatesResult as ResultMessage } from "@inline-chat/protocol/core" +import { db } from "@in/server/db" +import { members } from "@in/server/db/schema" +import { accessQueryComment } from "@in/server/db/databaseDiagnostics" +import { getUpdates } from "@in/server/functions/updates.getUpdates" +import { setupTestLifecycle, testUtils } from "../setup" +import { prepareScenario } from "./scenarios" +import { selectScenarios } from "./catalog" +import { assertQueryBudget, measureOperation } from "./measure" + +setupTestLifecycle() + +test.each(["updates.user-replay.100", "updates.user-replay.revoked.100"])("%s exercises both access batches and complete page accounting", async (id) => { + const spec = selectScenarios([id])[0]! + const operation = await prepareScenario(spec) + const reads: { source: string; parameters: number }[] = [] + const sample = await measureOperation(db.$client.options, async () => { + const debug = db.$client.options.debug + db.$client.options.debug = (connection, query, parameters, types) => { + for (const source of ["sync.replay", "sync.sidecars"] as const) { + if (query.trimEnd().endsWith(accessQueryComment(source))) reads.push({ source, parameters: parameters.length }) + } + if (typeof debug === "function") debug(connection, query, parameters, types) + } + try { await operation.run() } finally { db.$client.options.debug = debug } + }, async () => {}) + assertQueryBudget(id, sample, spec.maxCommands) + await operation.verify() + expect(reads).toEqual([ + { source: "sync.replay", parameters: 301 }, + { source: "sync.sidecars", parameters: id.includes("revoked") ? 298 : 301 }, + ]) +}) + +test("a warmed 100-chat user page cannot disclose snapshots after Space membership revocation", async () => { + const operation = await prepareScenario(selectScenarios(["updates.user-replay.100"])[0]!) + await operation.run() + await operation.verify() + const viewer = await db.query.users.findFirst({ where: { email: "bench-replay-viewer@example.test" } }) + if (!viewer) throw new Error("Missing replay viewer") + await db.delete(members).where(eq(members.userId, viewer.id)) + const result = await getUpdates({ + bucket: { type: { oneofKind: "user", user: {} } }, startSeq: 0n, seqEnd: 0n, totalLimit: 0, limit: 100, + }, testUtils.functionContext({ userId: viewer.id })) + expect(result.seq).toBe(100n) + expect(result.final).toBe(true) + expect(result.updates).toEqual([]) + expect(result.skippedSequences.map((entry) => entry.seq)).toEqual(Array.from({ length: 100 }, (_, index) => BigInt(index + 1))) + expect(result.sidecars).toBeUndefined() + expect(Buffer.from(ResultMessage.toBinary(result)).includes(Buffer.from("Replay private snapshot"))).toBe(false) +}) diff --git a/server/src/__tests__/performance/userReplayScenario.ts b/server/src/__tests__/performance/userReplayScenario.ts new file mode 100644 index 000000000..6d8dc9025 --- /dev/null +++ b/server/src/__tests__/performance/userReplayScenario.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict" +import { and, eq } from "drizzle-orm" +import { GetUpdatesResult as ResultMessage, SyncSkippedSequence_Reason } from "@inline-chat/protocol/core" +import { db } from "@in/server/db" +import * as S from "@in/server/db/schema" +import { UpdatesModel } from "@in/server/db/models/updates" +import { getUpdates } from "@in/server/functions/updates.getUpdates" +import { Encoders } from "@in/server/realtime/encoders/encoders" +import { encodeDateStrict } from "@in/server/realtime/encoders/helpers" +import { testUtils } from "../setup" +import type { PreparedOperation } from "./scenarios" + +/** A full user page, not a chat-bucket replay: both access sources see batches. */ +export async function prepareUserReplayScenario(size: number, revokeOne: boolean): Promise { + const users = await db.insert(S.users).values([ + { email: "bench-replay-viewer@example.test" }, { email: "bench-replay-owner@example.test" }, + ]).returning() + const viewer = users[0]!, owner = users[1]! + const { session } = await testUtils.createSessionForUser(viewer.id, { clientType: "web" }) + const context = testUtils.functionContext({ userId: viewer.id, sessionId: session.id }) + const [space] = await db.insert(S.spaces).values({ name: "Replay benchmark space" }).returning() + assert(space) + await db.insert(S.members).values(users.map((user) => ({ userId: user.id, spaceId: space.id, role: "member" as const }))) + const chats = await db.insert(S.chats).values(Array.from({ length: size }, (_, index) => ({ + type: "thread" as const, spaceId: space.id, publicThread: false, createdBy: owner.id, + title: `Replay private snapshot ${index}`, isUntitled: false, + }))).returning() + await db.insert(S.chatParticipants).values(chats.flatMap((chat) => users.map((user) => ({ chatId: chat.id, userId: user.id })))) + const dialogs = await db.insert(S.dialogs).values(chats.map((chat) => ({ + userId: viewer.id, chatId: chat.id, spaceId: space.id, open: true, order: "a0", + }))).returning() + const dialogByChat = new Map(dialogs.map((dialog) => [dialog.chatId, dialog])) + const date = new Date() + // Seed a valid encrypted historic page in one write. Setup is outside timing; + // there is no bypass or replacement of the real authenticated replay reader. + await db.insert(S.updates).values(chats.map((chat, index) => ({ + bucket: S.UpdateBucket.User, entityId: viewer.id, seq: index + 1, date, + payload: UpdatesModel.build({ seq: index + 1, date: encodeDateStrict(date), update: { + oneofKind: "userChatOpen", userChatOpen: { + chat: Encoders.chat(chat, { encodingForUserId: viewer.id }), + dialog: Encoders.dialog(dialogByChat.get(chat.id)!, { unreadCount: 0 }), + }, + } }).encrypted, + }))) + await db.update(S.users).set({ updateSeq: size, lastUpdateDate: date }).where(eq(S.users.id, viewer.id)) + const input = { bucket: { type: { oneofKind: "user" as const, user: {} } }, startSeq: 0n, seqEnd: 0n, totalLimit: 0, limit: 100 } + const hidden = revokeOne ? chats.at(-1)! : undefined + if (hidden) { + // Warm both access queries before revocation. Retained dialogs and embedded + // journal snapshots must not turn a warmed plan into cached authority. + const warm = await getUpdates(input, context) + assert.equal(warm.updates.length, size) + await db.delete(S.chatParticipants).where(and(eq(S.chatParticipants.chatId, hidden.id), eq(S.chatParticipants.userId, viewer.id))) + } + const visible = chats.filter((chat) => chat.id !== hidden?.id).map((chat) => BigInt(chat.id)) + let result: Awaited> + return { + async run() { result = await getUpdates(input, context) }, + async verify() { + assert.equal(result.seq, BigInt(size)) + assert.equal(result.final, true) + assert.deepEqual(result.updates.map((update) => { + assert.equal(update.update.oneofKind, "chatOpen") + return update.update.oneofKind === "chatOpen" ? update.update.chatOpen.chat?.id : undefined + }), visible) + assert.deepEqual(result.sidecars?.chats.map((chat) => chat.id).sort(), [...visible].sort()) + assert.deepEqual(result.sidecars?.dialogs.map((dialog) => dialog.chatId).sort(), [...visible].sort()) + assert.deepEqual(result.skippedSequences, hidden ? [{ seq: BigInt(size), reason: SyncSkippedSequence_Reason.IRRELEVANT_TO_BUCKET }] : []) + if (hidden) { + // Assert the complete encoded response, including embedded historical + // metadata, not only the filtered chat sidecar list. + assert(!Buffer.from(ResultMessage.toBinary(result)).includes(Buffer.from(hidden.title!))) + assert(await db.query.dialogs.findFirst({ where: { chatId: hidden.id, userId: viewer.id } })) + } + }, + } +} diff --git a/server/src/modules/internalMessaging/v3OriginRecovery.test.ts b/server/src/modules/internalMessaging/v3OriginRecovery.test.ts new file mode 100644 index 000000000..36c000556 --- /dev/null +++ b/server/src/modules/internalMessaging/v3OriginRecovery.test.ts @@ -0,0 +1,249 @@ +import { expect, test } from "bun:test" +import { generateKeyPairSync } from "node:crypto" +import { createInterface } from "node:readline" +import { Readable } from "node:stream" +import { + GetUpdatesResult_ResultType, Method, + type RealtimeV3Update, type RpcCall, type RpcResult, +} from "@inline-chat/protocol/core" +import { setupTestLifecycle, testUtils } from "@in/server/__tests__/setup" +import { db } from "@in/server/db" +import { PermanentAuthorizationKeyRepository } from "@in/server/db/models/inlineProtocol" +import { makeAuthorizationKeyCipher } from "@in/server/modules/inlineProtocol/keyCipher" +import { makeInlineProtocolRsaSigner } from "@in/server/modules/inlineProtocol/rsaSigner" +import { sendMessage } from "@in/server/functions/messages.sendMessage" +import { revokeSession } from "@in/server/modules/sessions/revokeSession" +import { sql } from "drizzle-orm" + +setupTestLifecycle() + +type Authorization = { + key: Uint8Array; keyId: Uint8Array; serverSalt: bigint; temporary: boolean; expiresAt?: number +} +type Credentials = { permanent: Authorization; temporary?: Authorization } +type PublicKey = { modulus: string; exponent: string; fingerprint: string } +type ClientOptions = { + url: string; rsaPublicKeys: readonly PublicKey[]; connectTimeoutMs: number; requestTimeoutMs: number + onCredentials?: (credentials: Credentials) => void + onUpdate?: (update: RealtimeV3Update) => void +} +type AuthenticatedClient = { + readonly credentials: Credentials | undefined + connect(credentials: Credentials): Promise + callRpc(call: RpcCall): Promise + close(): Promise +} +type ProtocolClient = { + readonly authorization: Authorization + probeTemporaryAuthorization(): Promise + close(): Promise +} + +// Exercise the actual SDK implementation without introducing a production SDK +// dependency into the server. This fixture boundary only describes what it uses; +// the SDK's own project remains responsible for compiling its implementation. +const loadClients = async () => { + const base = new URL("../../../../packages/sdk/src/realtime/", import.meta.url) + const client = await import(new URL("v3-client.ts", base).href) as { + InlineRealtimeV3Client: new (options: ClientOptions) => AuthenticatedClient + } + const connection = await import(new URL("v3-connection.ts", base).href) as { + InlineProtocolV3Connection: { connect(options: ClientOptions & { authorization?: Authorization }): Promise } + } + return { ...client, ...connection } +} + +const bounded = async (operation: PromiseLike, label: string, milliseconds = 10_000): Promise => { + let timer: ReturnType | undefined + try { + return await Promise.race([operation, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Timed out: ${label}`)), milliseconds) + })]) + } finally { clearTimeout(timer) } +} + +async function startOrigin(databaseUrl: string, rsaPrivateKeysJson: string) { + const child = Bun.spawn({ + cmd: [process.execPath, "--no-env-file", "src/modules/internalMessaging/v3OriginRecovery.worker.ts"], + cwd: import.meta.dir + "/../../..", + env: { + ...process.env, DATABASE_URL: databaseUrl, TEST_DATABASE_URL: databaseUrl, + REDIS_URL: "", VALKEY_URL: "", INLINE_REALTIME_DISTRIBUTED: "1", + INLINE_TEST_V3_RSA_KEYS: rsaPrivateKeysJson, + SESSION_AUTHORITY_RECONCILIATION_INTERVAL_MS: "1000", SESSION_AUTHORITY_MAX_AGE_MS: "10000", + }, + stdin: "pipe", stdout: "pipe", stderr: "pipe", + }) + let output = "" + const signals = new Map>>() + const signal = (key: string) => { + let value = signals.get(key) + if (!value) { value = Promise.withResolvers(); signals.set(key, value) } + return value + } + const readers = Promise.all([child.stdout, child.stderr].map(async (stream) => { + for await (const line of createInterface({ input: Readable.fromWeb(stream as never) })) { + if (line.startsWith("V3_ORIGIN:PORT:")) signal("PORT").resolve(line.slice("V3_ORIGIN:PORT:".length)) + else if (line.startsWith("V3_ORIGIN:")) signal(line.slice("V3_ORIGIN:".length)).resolve("") + else output = (output + line + "\n").slice(-8_000) + } + })) + const wait = (name: string) => bounded(Promise.race([ + signal(name).promise, + child.exited.then((code): never => { throw new Error(`Origin exited before ${name}: ${code}; ${output}`) }), + ]), `origin ${name}; ${output}`) + let stopPromise: Promise | undefined + const stop = (): Promise => stopPromise ??= (async () => { + try { + child.stdin.write("STOP\n") + child.stdin.end() + const code = await bounded(child.exited, `graceful origin exit; ${output}`) + expect(code).toBe(0) + } catch (error) { + child.kill() + await child.exited + throw error + } finally { await readers } + })() + try { + const port = Number(await wait("PORT")) + return { + url: `ws://127.0.0.1:${port}/realtime/v3`, httpUrl: `http://127.0.0.1:${port}`, + command: async (command: string, response: string) => { child.stdin.write(command + "\n"); await wait(response) }, + stop, + } + } catch (error) { + child.kill() + await child.exited + await readers + throw error + } +} + +test("V3 recovers cached authorization across origins, catches up without Redis, and revokes both live origins", async () => { + const previousDistributed = process.env["INLINE_REALTIME_DISTRIBUTED"] + process.env["INLINE_REALTIME_DISTRIBUTED"] = "1" + const clients: Array<{ close(): Promise }> = [] + const origins: Awaited>[] = [] + const failures: unknown[] = [] + try { + const { InlineRealtimeV3Client, InlineProtocolV3Connection } = await loadClients() + const rsaPrivateKeysJson = JSON.stringify(Array.from({ length: 2 }, () => ({ + privateKeyPem: generateKeyPairSync("rsa", { modulusLength: 2048 }).privateKey.export({ type: "pkcs8", format: "pem" }), + }))) + const rsaPublicKeys = makeInlineProtocolRsaSigner(rsaPrivateKeysJson).publicKeyRing + const [database] = await db.execute<{ name: string }>(sql`select current_database() as name`) + if (!database) throw new Error("Missing isolated database") + const databaseUrl = new URL(process.env["TEST_DATABASE_URL"]!) + databaseUrl.pathname = "/" + database.name + const first = await startOrigin(databaseUrl.toString(), rsaPrivateKeysJson) + origins.push(first) + const second = await startOrigin(databaseUrl.toString(), rsaPrivateKeysJson) + origins.push(second) + const options = (url: string): ClientOptions => ({ url, rsaPublicKeys, connectTimeoutMs: 5_000, requestTimeoutMs: 5_000 }) + const recipient = await testUtils.createUser("v3-origin-recipient@test.invalid") + const targetSession = await testUtils.createSessionForUser(recipient.id) + const observerSession = await testUtils.createSessionForUser(recipient.id) + const sender = await testUtils.createUser("v3-origin-sender@test.invalid") + const chat = await testUtils.createPrivateChat(sender, recipient) + if (!chat) throw new Error("Missing private chat") + const repository = new PermanentAuthorizationKeyRepository(makeAuthorizationKeyCipher({ + activeId: "fixture", keys: new Map([["fixture", new Uint8Array(32).fill(0x71)]]), + })) + const permanentFor = async (sessionId: number) => { + const login = await InlineProtocolV3Connection.connect(options(first.url)) + clients.push(login) + const permanent = login.authorization + expect(await repository.authorize(permanent.keyId, recipient.id, sessionId)).toBe(true) + await login.close() + return permanent + } + const permanent = await permanentFor(targetSession.session.id) + const firstClient = new InlineRealtimeV3Client(options(first.url)) + clients.push(firstClient) + await firstClient.connect({ permanent }) + const original = firstClient.credentials + if (!original?.temporary) throw new Error("Origin did not create temporary authorization") + const getMe: RpcCall = { method: Method.GET_ME, input: { oneofKind: "getMe", getMe: {} } } + expect((await firstClient.callRpc(getMe)).oneofKind).toBe("getMe") + + // B shares the durable permanent key, never A's process-local temporary key. + const forgotten = await InlineProtocolV3Connection.connect({ ...options(second.url), authorization: original.temporary }) + clients.push(forgotten) + await expect(forgotten.probeTemporaryAuthorization()).rejects.toMatchObject({ code: "unauthorized" }) + await forgotten.close() + expect(await repository.getActive(permanent.keyId)).toBeDefined() + + const updates: RealtimeV3Update[] = [] + let updateArrived = Promise.withResolvers() + const replacements: Credentials[] = [] + const secondClient = new InlineRealtimeV3Client({ + ...options(second.url), + onCredentials: (credentials) => { replacements.push(credentials) }, + onUpdate: (update) => { updates.push(update); updateArrived.resolve(); updateArrived = Promise.withResolvers() }, + }) + clients.push(secondClient) + await secondClient.connect(original) + expect(replacements).toHaveLength(1) + expect(secondClient.credentials?.permanent.keyId).toEqual(permanent.keyId) + expect(secondClient.credentials?.temporary?.keyId).not.toEqual(original.temporary.keyId) + const me = await secondClient.callRpc(getMe) + if (me.oneofKind !== "getMe") throw new Error("Unexpected getMe response") + expect(me.getMe.user?.id).toBe(BigInt(recipient.id)) + await Promise.all(origins.map((origin) => origin.command("ADMISSION_IDLE", "ADMISSION_IDLE"))) + + // A different account session on B must survive revoking the shared target. + const observer = new InlineRealtimeV3Client(options(second.url)) + clients.push(observer) + await observer.connect({ permanent: await permanentFor(observerSession.session.id) }) + expect((await observer.callRpc(getMe)).oneofKind).toBe("getMe") + updates.length = 0 + const sent = await sendMessage({ + peerId: { type: { oneofKind: "chat", chat: { chatId: BigInt(chat.id) } } }, + message: "Durable message while both V3 origins are active", randomId: 712345n, + }, testUtils.functionContext({ userId: sender.id })) + const message = sent.updates.find((update) => update.update.oneofKind === "newMessage") + if (message?.update.oneofKind !== "newMessage" || !message.update.newMessage.message) throw new Error("Missing committed message") + const hasHint = () => updates.some((update) => update.message?.payload.oneofKind === "update" && + update.message.payload.update.updates.some((item) => item.update.oneofKind === "chatHasNewUpdates" && + item.update.chatHasNewUpdates.chatId === BigInt(chat.id) && item.update.chatHasNewUpdates.updateSeq >= 1)) + await bounded((async () => { while (!hasHint()) await updateArrived.promise })(), "automatic brokerless V3 durable hint", 5_000) + const recovered = await secondClient.callRpc({ + method: Method.GET_UPDATES, input: { oneofKind: "getUpdates", getUpdates: { + bucket: { type: { oneofKind: "chat", chat: { peerId: { type: { oneofKind: "chat", chat: { chatId: BigInt(chat.id) } } } } } }, + startSeq: 0n, seqEnd: 0n, limit: 100, totalLimit: 0, + } }, + }) + if (recovered.oneofKind !== "getUpdates") throw new Error("Unexpected catch-up response") + expect(recovered.getUpdates.resultType).toBe(GetUpdatesResult_ResultType.SLICE) + expect(recovered.getUpdates.updates.some((update) => update.update.oneofKind === "newMessage" && + update.update.newMessage.message?.message === "Durable message while both V3 origins are active")).toBe(true) + expect(recovered.getUpdates.sidecars?.users.some((user) => user.id === BigInt(sender.id))).toBe(true) + + expect((await revokeSession({ actor: "system", targetUserId: recipient.id, sessionId: targetSession.session.id })).revoked).toBe(true) + // Neither child has Redis. Real periodic DB authority reconciliation closes + // both sockets; no manual reconciliation or injected broker event is used. + await Promise.all(origins.map((origin) => origin.command(`WAIT_REVOKED:${targetSession.session.id}`, "REVOKED"))) + // Registry removal precedes the remote close event. If that event arrives + // during this RPC the SDK reports unauthorized; if already received it + // reports closed. Neither origin may successfully execute the RPC. + const revokedOrClosed = { code: expect.stringMatching(/^(unauthorized|closed)$/) } + await expect(firstClient.callRpc(getMe)).rejects.toMatchObject(revokedOrClosed) + await expect(secondClient.callRpc(getMe)).rejects.toMatchObject(revokedOrClosed) + expect((await observer.callRpc(getMe)).oneofKind).toBe("getMe") + + await first.stop() + expect((await fetch(second.httpUrl + "/readyz")).status).toBe(200) + expect((await observer.callRpc(getMe)).oneofKind).toBe("getMe") + } catch (error) { failures.push(error) } + finally { + const closed = await Promise.allSettled(clients.map((client) => client.close())) + const stopped = await Promise.allSettled(origins.map((origin) => origin.stop())) + for (const result of [...closed, ...stopped]) { + if (result.status === "rejected") failures.push(result.reason) + } + if (previousDistributed === undefined) Reflect.deleteProperty(process.env, "INLINE_REALTIME_DISTRIBUTED") + else process.env["INLINE_REALTIME_DISTRIBUTED"] = previousDistributed + } + if (failures.length) throw new AggregateError(failures, "V3 origin recovery failed") +}, 45_000) diff --git a/server/src/modules/internalMessaging/v3OriginRecovery.worker.ts b/server/src/modules/internalMessaging/v3OriginRecovery.worker.ts new file mode 100644 index 000000000..40ca9af3e --- /dev/null +++ b/server/src/modules/internalMessaging/v3OriginRecovery.worker.ts @@ -0,0 +1,54 @@ +import { createInterface } from "node:readline" +import { Readable } from "node:stream" +import { makeCandidateHttpApplication } from "@in/server/core/http/candidateApplication" +import { startCoreProductionServer } from "@in/server/core/http/productionHost" +import { closeDb, validateDatabaseStartup } from "@in/server/db" +import { connectionManager } from "@in/server/ws/connections" +import { connectionBackgroundWork } from "@in/server/ws/backgroundWork" +import { connectedUserRepair } from "./repair" + +// This process uses the real API, protocol repositories and lifecycle. Only the +// RSA/key-encryption material is synthetic; no auth provider or job is started. +const rsaPrivateKeysJson = process.env["INLINE_TEST_V3_RSA_KEYS"] +if (!rsaPrivateKeysJson) throw new Error("Missing synthetic V3 key ring") +const ring = { activeId: "fixture", keys: new Map([["fixture", new Uint8Array(32).fill(0x71)]]) } +await validateDatabaseStartup() +const host = await startCoreProductionServer({ + application: makeCandidateHttpApplication({ apiBaseUrl: "http://127.0.0.1", middleware: { isProduction: false } }), + hostname: "127.0.0.1", + port: 0, + installSignalHandlers: false, + startClusterServices: true, + inlineProtocolConfiguration: { + enabled: true, + requireCanonicalPublicRing: false, + rsaPrivateKeysJson, + authKeyKekRing: ring, + authCodePepperRing: ring, + encryptReplayResults: false, + }, +}) +try { + console.log(`V3_ORIGIN:PORT:${host.port}`) + for await (const line of createInterface({ input: Readable.fromWeb(Bun.stdin.stream() as never) })) { + if (line === "STOP") break + if (line === "ADMISSION_IDLE") { + await new Promise((resolve) => setTimeout(resolve, 0)) + await connectionBackgroundWork.waitForIdle() + await connectedUserRepair.waitForIdle() + console.log("V3_ORIGIN:ADMISSION_IDLE") + } + if (line.startsWith("WAIT_REVOKED:")) { + const sessionId = Number(line.slice("WAIT_REVOKED:".length)) + const deadline = performance.now() + 5_000 + while (connectionManager.getAuthenticatedSessionIdentities().some((session) => session.sessionId === sessionId)) { + if (performance.now() >= deadline) throw new Error("Revoked session remained admitted on this origin") + await new Promise((resolve) => setTimeout(resolve, 10)) + } + console.log("V3_ORIGIN:REVOKED") + } + } +} finally { + await host.shutdown() + await closeDb() +} diff --git a/server/src/modules/message/historySnapshot.postgres.test.ts b/server/src/modules/message/historySnapshot.postgres.test.ts new file mode 100644 index 000000000..85704b621 --- /dev/null +++ b/server/src/modules/message/historySnapshot.postgres.test.ts @@ -0,0 +1,74 @@ +import { expect, test } from "bun:test" +import { eq } from "drizzle-orm" +import { RpcError_Code } from "@inline-chat/protocol/core" +import { db } from "@in/server/db" +import { chatParticipants, members } from "@in/server/db/schema" +import { setupTestLifecycle, testUtils } from "@in/server/__tests__/setup" +import { getChatHistory } from "@in/server/functions/messages.getChatHistory" +import { measureOperation } from "@in/server/__tests__/performance/measure" +import { prepareScenario } from "@in/server/__tests__/performance/scenarios" +import { selectScenarios } from "@in/server/__tests__/performance/catalog" +import { withHistorySnapshot } from "./historySnapshot" + +setupTestLifecycle() + +const peer = (id: number) => ({ type: { oneofKind: "chat" as const, chat: { chatId: BigInt(id) } } }) + +async function fixture() { + const owner = await testUtils.createUser("snapshot-owner@example.test") + const other = await testUtils.createUser("snapshot-peer@example.test") + const chat = await testUtils.createPrivateChat(owner, other) + if (!chat) throw new Error("Missing private chat") + return { owner, chat } +} + +test("an explicit chat peer needs one SELECT inside its read-only snapshot", async () => { + const { owner, chat } = await fixture() + let returnedId: number | undefined + const sample = await measureOperation(db.$client.options, async () => { + returnedId = await withHistorySnapshot(peer(chat.id), owner.id, async (_tx, row) => row.id) + }, async () => {}) + expect(returnedId).toBe(chat.id) + // BEGIN, SET TRANSACTION, one chat SELECT, COMMIT. Authorization for a DM + // checks this row directly; no outside-snapshot lookup is needed. + expect(sample.sql.commands).toBe(4) + expect(Object.entries(sample.sql.shapes).filter(([shape]) => shape.startsWith("SELECT:")).reduce((sum, [, count]) => sum + count, 0)).toBe(1) +}) + +test("a complete explicit-chat history page saves the redundant lookup", async () => { + const operation = await prepareScenario(selectScenarios(["history.50"])[0]!) + const sample = await measureOperation(db.$client.options, operation.run, async () => {}) + await operation.verify() + expect(sample.sql.commands).toBe(10) +}) + +test("a missing explicit chat preserves CHAT_ID_INVALID and never enters the response reader", async () => { + const { owner } = await fixture() + let entered = false + await expect(withHistorySnapshot(peer(2_147_483_647), owner.id, async () => { entered = true })).rejects.toMatchObject({ code: RpcError_Code.CHAT_ID_INVALID }) + expect(entered).toBe(false) +}) + +test("an explicit private-chat ID gives an outsider no history or response metadata", async () => { + const { owner, chat } = await fixture() + const outsider = await testUtils.createUser("snapshot-outsider@example.test") + await testUtils.createTestMessage({ chatId: chat.id, fromId: owner.id, messageId: 1, text: "private snapshot body" }) + let entered = false + await expect(withHistorySnapshot(peer(chat.id), outsider.id, async () => { entered = true })).rejects.toMatchObject({ code: RpcError_Code.PEER_ID_INVALID }) + expect(entered).toBe(false) + await expect(getChatHistory({ peerId: peer(chat.id) }, testUtils.functionContext({ userId: outsider.id }))).rejects.toMatchObject({ code: RpcError_Code.PEER_ID_INVALID }) +}) + +test("a warmed explicit-chat history read rechecks Space membership despite a retained grant", async () => { + const { space, users } = await testUtils.createSpaceWithMembers("Snapshot authority", ["snapshot-member@example.test"]) + const viewer = users[0]! + const chat = await testUtils.createChat(space.id, "Private snapshot title", "thread", false) + if (!chat) throw new Error("Missing private thread") + await testUtils.addParticipant(chat.id, viewer.id) + await testUtils.createTestMessage({ chatId: chat.id, fromId: viewer.id, messageId: 1, text: "Private snapshot body" }) + const context = testUtils.functionContext({ userId: viewer.id }) + expect((await getChatHistory({ peerId: peer(chat.id) }, context)).messages[0]?.message).toBe("Private snapshot body") + await db.delete(members).where(eq(members.userId, viewer.id)) + expect(await db.select().from(chatParticipants).where(eq(chatParticipants.chatId, chat.id))).toHaveLength(1) + await expect(getChatHistory({ peerId: peer(chat.id) }, context)).rejects.toMatchObject({ code: RpcError_Code.SPACE_ID_INVALID }) +}) diff --git a/server/src/modules/message/historySnapshot.ts b/server/src/modules/message/historySnapshot.ts index 3f91df017..70ac40742 100644 --- a/server/src/modules/message/historySnapshot.ts +++ b/server/src/modules/message/historySnapshot.ts @@ -50,23 +50,31 @@ export async function withHistorySnapshot( read: (tx: Transaction, chat: DbChat) => Promise, ): Promise { validateHistoryPeer(peer) - let chat: DbChat - try { - chat = await ChatModel.getChatFromInputPeer(peer, { currentUserId }) - } catch (error) { - if (!(error instanceof ModelError && error.code === ModelError.Codes.CHAT_INVALID)) throw error - if (peer.type.oneofKind !== "user" && peer.type.oneofKind !== "self") { - throw RealtimeRpcError.ChatIdInvalid() + let chatId: number + if (peer.type.oneofKind === "chat") { + // The validated ID is enough to enter the snapshot. Read the chat and + // authorize it there rather than loading the same row twice. + chatId = Number(peer.type.chat.chatId) + } else { + let chat: DbChat + try { + chat = await ChatModel.getChatFromInputPeer(peer, { currentUserId }) + } catch (error) { + if (!(error instanceof ModelError && error.code === ModelError.Codes.CHAT_INVALID)) throw error + if (peer.type.oneofKind !== "user" && peer.type.oneofKind !== "self") { + throw RealtimeRpcError.ChatIdInvalid() + } + const peerUserId = peer.type.oneofKind === "self" ? currentUserId : Number(peer.type.user.userId) + const user = await UsersModel.getUserById(peerUserId) + if (!user || UsersModel.isDeleted(user)) throw RealtimeRpcError.UserIdInvalid() + await ChatModel.createUserChatAndDialog({ peerUserId, currentUserId }) + await ChatModel.createUserChatAndDialog({ peerUserId: currentUserId, currentUserId: peerUserId }) + chat = await ChatModel.getChatFromInputPeer(peer, { currentUserId }) } - const peerUserId = peer.type.oneofKind === "self" ? currentUserId : Number(peer.type.user.userId) - const user = await UsersModel.getUserById(peerUserId) - if (!user || UsersModel.isDeleted(user)) throw RealtimeRpcError.UserIdInvalid() - await ChatModel.createUserChatAndDialog({ peerUserId, currentUserId }) - await ChatModel.createUserChatAndDialog({ peerUserId: currentUserId, currentUserId: peerUserId }) - chat = await ChatModel.getChatFromInputPeer(peer, { currentUserId }) + chatId = chat.id } return withHistoryReadSnapshot(async (tx) => { - const [snapshotChat] = await tx.select().from(chats).where(eq(chats.id, chat.id)).limit(1) + const [snapshotChat] = await tx.select().from(chats).where(eq(chats.id, chatId)).limit(1) if (!snapshotChat) throw RealtimeRpcError.ChatIdInvalid() await AccessGuards.ensureChatAccess(snapshotChat, currentUserId, tx) return read(tx, snapshotChat) From 8e3adbd351a5e783c9760e742cc6abedeece1c58 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 21:12:19 +0330 Subject: [PATCH 08/14] infra: prepare ordered three-origin api failover --- server/deploy/ha-node-profile.json | 71 +++++ server/docs/hetzner-standby.md | 270 ++++++++++++------ .../ha/cloudflare-three-origin.example.json | 196 +++++++++++++ server/scripts/qualify-ha-host.test.ts | 81 ++++++ server/scripts/qualify-ha-host.ts | 150 ++++++++++ 5 files changed, 681 insertions(+), 87 deletions(-) create mode 100644 server/deploy/ha-node-profile.json create mode 100644 server/ha/cloudflare-three-origin.example.json create mode 100644 server/scripts/qualify-ha-host.test.ts create mode 100644 server/scripts/qualify-ha-host.ts diff --git a/server/deploy/ha-node-profile.json b/server/deploy/ha-node-profile.json new file mode 100644 index 000000000..e708237c3 --- /dev/null +++ b/server/deploy/ha-node-profile.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "purpose": "Preparation only. Bind qualified release digests before applying through the existing Coolify controller. This file contains no deployment action or credentials.", + "release": { + "arm64Image": null, + "sourceRevision": null, + "rollbackArm64Image": null, + "rollbackSourceRevision": null + }, + "common": { + "architecture": "arm64", + "internalPort": 8000, + "publishedHostPorts": [], + "privateBackendNetwork": true, + "fixedContainerName": false, + "restartPolicy": "unless-stopped", + "stopGraceSeconds": 45, + "logMaxSize": "2m", + "logMaxFiles": 2, + "health": { + "path": "/readyz", + "intervalSeconds": 5, + "timeoutSeconds": 3, + "retries": 3, + "startPeriodSeconds": 15, + "expectedStatus": 200 + }, + "environment": { + "NODE_ENV": "production", + "PORT": "8000", + "INLINE_PROCESS_ROLE": "all", + "INLINE_REALTIME_DISTRIBUTED": "1", + "INLINE_DATABASE_QUERY_POOL_MAX": "3", + "INLINE_INGRESS_MODE": "cloudflare-optional", + "INLINE_INGRESS_HOST": "api.inline.chat", + "INLINE_TRUSTED_CLIENT_IP_HEADER": "cf-connecting-ip", + "INLINE_INGRESS_DIRECT_IP_SOURCE": "x-real-ip", + "SHUTDOWN_TIMEOUT_MS": "40000" + }, + "canonicalTlsHost": "api.inline.chat", + "nativeProtocolEnabled": true, + "clusterServicesEnabled": true, + "migrationOnBoot": false + }, + "nodes": { + "nbg": { + "priority": 2, + "cpuLimit": 2, + "memoryBytes": 1610612736, + "memoryPlusSwapBytes": 1610612736, + "description": "First fallback after Fly. Reserve memory for two containers during replacement on the existing 4 GB host." + }, + "finland": { + "priority": 3, + "cpuLimit": 2, + "memoryBytes": 2147483648, + "memoryPlusSwapBytes": 2147483648, + "description": "Second fallback after Nuremberg. Preserve existing shared applications and backup jobs; reserve two containers for replacement." + } + }, + "operation": { + "deploymentMethod": "Coolify Docker Image application on the existing controller", + "automaticGitDeploy": false, + "start": "Keep origin out of customer routing; pull the selected digest, verify source/architecture, start and qualify before enabling its pool.", + "stop": "Disable this origin in routing and verify another healthy origin; allow routing propagation, then SIGTERM and wait for completion within 45 seconds.", + "rollback": "Deploy the recorded compatible rollback digest through the same resource; no automatic database rollback.", + "fixtureQualification": "Use a separate synthetic database and no production credentials or public routes. The existing artifact smoke does not qualify native protocol, distributed delivery, or workers.", + "workerActivation": "The all-role expectation is conditional on the concurrent-worker review and isolated qualification. Do not start real workers merely to satisfy this profile.", + "databaseBudget": "Each process uses up to three query plus one health connections. Count every running origin and replacement; verify actual database capacity before starting overlap." + } +} diff --git a/server/docs/hetzner-standby.md b/server/docs/hetzner-standby.md index 6e50e48c9..9f82b2789 100644 --- a/server/docs/hetzner-standby.md +++ b/server/docs/hetzner-standby.md @@ -1,41 +1,52 @@ # Hetzner standby and Cloudflare proxy bypass -This is the deployment contract, not a statement that the standby is already -qualified. Keep Cloudflare authoritative DNS. Start with one Fly API and one -warm API on the existing Hetzner host. A second Fly Machine can later join the -same app without changing clients or adding another load-balancer pool. +This is a prepared deployment option, not a statement that the origins or routing +are active or qualified. Keep Cloudflare authoritative DNS. The preferred order is +**Fly → Nuremberg → Finland**: Fly serves normally, Nuremberg is the first +fallback, and the existing Finland host is the second fallback. A second Fly +Machine can later join the same app without another Cloudflare pool. ```text api.inline.chat — Cloudflare DNS + optional proxy normal: Fly app → one healthy API Machine (temporary deployment overlap) - manual fallback: existing Hetzner → Traefik → private API container - all APIs → same PostgreSQL, object storage and key rings + first fallback: Nuremberg → private API container behind its qualified TLS proxy + second fallback: existing Finland → Traefik → private API container + all APIs → same PostgreSQL, object storage and key rings Cloudflare proxy incident: same hostname, DNS-only A/AAAA → qualified origin ``` -The minimum is a running container on the existing host and tested manual -cutover. Automatic Cloudflare Load Balancing is optional after this gate passes; -confirm billing/entitlement before enabling it. No new VM, Redis cluster, -database replica, ingress daemon or failover controller is required. +Qualify running fallback containers and direct authenticated access before +adding automatic routing. A stopped container is manual recovery capacity. +The [three-origin proposal](../ha/cloudflare-three-origin.example.json) contains +non-secret release intent and Cloudflare request templates with unresolved +identities. It is local data, not an apply script or a live configuration. +Confirm the actual three-origin subscription quote before enabling it. No new +VM, Redis cluster, database replica or failover controller is required for this +option; check the existing hosts' capacity independently. ## Shared-host boundary -The existing Hetzner host also serves unrelated production applications. Scope +The existing Finland host also serves unrelated production applications. Scope changes to the Inline API resource and its uniquely named Traefik routers, service and middleware. Preserve shared entrypoints, networks, certificate storage and other applications. A shared proxy restart or host-level change requires a separate reviewed operation; it is not part of this rollout. -Build and qualify the ARM64 Coolify image on the native GitHub Actions runner, -then deploy the qualified image by digest. Do not build on the shared host or -run host-wide Docker cleanup. Confirm pull access before changing the resource +Build and qualify the appropriate architecture in GitHub Actions, then deploy +the tested image by digest. The current ARM64 Nuremberg and Finland candidates +use the native ARM64 Coolify artifact; verify each host's actual architecture. +Fly uses the amd64 runtime artifact from the same source commit. Do not build +on the shared host or run host-wide Docker cleanup. Confirm pull access before changing the resource from its historical source-build configuration. -Initially cap the API container at two CPUs and 2 GiB RAM, with total -memory-plus-swap also capped at 2 GiB. These are isolation limits, not measured +Use the prepared [host profile](../deploy/ha-node-profile.json): initially cap +Nuremberg at two CPUs/1536 MiB and Finland at two CPUs/2 GiB per container, with +matching total memory-plus-swap caps. These are isolation limits, not measured capacity. Qualify actual failover load within them; do not automatically raise -them when the host is busy. Budget two such containers for rolling overlap. +them when the host is busy. Budget two containers on each host for its rolling +overlap. The host qualifier consumes sanitized metadata; passing it does not +establish application, TLS, broker or worker acceptance. Keep Docker log retention bounded for this resource. Capture the current API configuration and a baseline of the other public applications before starting, then check their availability during qualification. Roll back only the API @@ -59,12 +70,16 @@ make a database-healthy API unready. For a complete standby use `INLINE_PROCESS_ROLE=all`. `api` is suitable for isolated qualification but omits background workers and is not complete failover. +An `all` fallback starts workers immediately, even before user traffic reaches +it. User ingress is ordered failover; background execution is concurrent. Concurrent workers rely on existing PostgreSQL claims, leases and fencing. -External side effects retain their existing delivery/idempotency contracts. +Qualify those mechanisms and broker/recovery behavior across all intended +nodes; routing does not fence workers. External side effects retain their +existing delivery/idempotency contracts. ## Hetzner ingress -The source supports direct profiles on both origins without a provider admission +The source supports direct profiles on all origins without a provider admission secret. Keep application authentication and canonical Host enforcement. The optional Cloudflare secret authenticates client-IP attribution only. @@ -107,74 +122,150 @@ must continue while normal DNS points at Cloudflare/Fly; qualify DNS validation or another renewal arrangement rather than relying on HTTP challenges always reaching Hetzner. Never use insecure TLS for the load-balancer monitor or clients. +The monitor's `allow_insecure=false` protects only health probes. Before cutover, +verify the **effective customer-to-origin path** uses HTTPS/443 with Full (strict) +(`ssl=strict`) or a reviewed equivalent certificate-validation policy for +`api.inline.chat`. Full without strict validation does not validate the origin +certificate. Inspect matching Origin Rules, Configuration Rules and legacy Page +Rules for SSL-mode, scheme, port, Host or SNI overrides; a zone default alone is +not enough. Do not change zone-wide SSL settings blindly on a shared zone. +See [Full (strict)](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes/full-strict/) +and [Full](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes/full/). + ## Release and qualification -1. Qualify the exact source/image in both the amd64 Fly and native ARM64 Coolify - Linux CI lanes. Apply additive migration - `0152_recent-realtime-buckets` with the migration role, then verify the ledger. - API startup checks schema compatibility; it never performs migrations. - A manual publish-only release records `runtime_image` and `standby_image` - digests in its summary without deploying either provider. The ARM container - smoke proves packaging, native dependencies, migrations, HTTP, a legacy socket - and shutdown in API-only mode. It does not qualify distributed delivery, V3 - authentication or the `all` worker role; retain the later runtime gates. -2. Keep Coolify automatic deploys disabled and pin the qualified release. Copy - runtime application credentials using managed authentication and stdin/in-memory - transport. Verify byte parity for encryption/session/native-protocol key rings - and semantic parity for PostgreSQL/R2/provider endpoints without printing values. - Use the DML-only database role; do not copy a migration/owner credential. -3. Confirm independent Hetzner access to PostgreSQL, object storage, keys and - required providers. Set `INLINE_DATABASE_QUERY_POOL_MAX=3` on each Fly and Hetzner - process; the separate direct health pool remains capped at one connection. - Three overlapping processes therefore use at most 12 direct connections. - With 22 usable database slots, reserve at least four for administration, - exports, migrations and other clients, and inspect actual usage before - starting overlap. During the first upgrade, keep Hetzner stopped while the - legacy Fly process with 11 slots and new Fly process with four slots coexist. - Start standby only after the old process is stopped. Query clients may use the direct - endpoint within this budget; PgBouncer requires explicit mode and successful - role timeout qualification. The omitted query cap retains the historical - default 10; invalid values fail startup rather than silently changing it. -4. Set Coolify's health path to `/readyz` (not `/healthz`), five-second probes, - a bounded timeout, and enough startup grace. Configure a stop grace longer - than the API drain deadline. Verify source/image revision and worker role. -5. Before directing user traffic, test the real Hetzner route with public TLS/SNI and - the canonical Host. Prove authenticated HTTP, V2/V3, upload/read-back, - cross-origin message recovery, revocation, drain/reconnect and worker handoff. - Send forged CF/X-Real-IP/Fly-Client-IP/XFF headers to prove the real proxy - boundary. A `200 /readyz` is insufficient evidence. -6. Rehearse Fly origin removal from routing while keeping shared PostgreSQL. - Measure detection, routing, socket reconnect/catch-up and job recovery. Never - restore a database or stop the old writer just because a health check failed. +1. Qualify one committed source revision in the amd64 runtime and native ARM64 + Coolify CI lanes. After all jobs pass, retain the `runtime_image` and + `standby_image` immutable digests, source revision and workflow run URL. + These are **different architecture-specific digests from the same source + commit**, not interchangeable images. The manual `publish_only` workflow + publishes this pair without production preflight, migrations or deployment; + publication itself is a separate authorized action. Never select a digest + from a partly failed run. Do not rebuild on the target host. +2. Run the explicit packaged migration and verification commands once, using + the migration role, when the selected release requires schema changes. + Startup checks compatibility but never migrates. Use tested additive + migrations during overlap, and qualify the retained rollback release on the + resulting schema. Image smoke proves packaging, native dependencies, HTTP, + a legacy socket and shutdown in API-only mode; it does not qualify V3, + multi-node delivery or concurrent `all` workers. +3. Keep automatic deploys disabled on fallback resources and pin the qualified + digest. Transfer runtime credentials through managed authentication and + stdin/in-memory transport. Verify key-ring byte parity and semantic parity + for PostgreSQL/R2/provider endpoints without printing values. Use the DML-only + database role, never a migration/owner credential. Retain pullable current + and compatible rollback artifacts independently of a failed primary provider. +4. Confirm each origin has independent access to PostgreSQL, storage and required + providers. Set query cap 3 plus the separate health cap 1 per process. Three + steady APIs budget 12 direct connections; replacing one at a time budgets + 16 during overlap, before administration, exports, migrations and other + clients. Verify actual provider limits/usage and reserve headroom rather than + treating this arithmetic as capacity proof. Keep deployments sequential; do + not overlap replacements across the three locations. PgBouncer needs explicit mode + and successful role-timeout qualification. An omitted query cap retains the + historical default 10; invalid values fail startup. +5. Configure each supervisor's `/readyz` probe, bounded timeout/startup grace, + SIGTERM and stop grace longer than the 40-second application drain (at least + Fly's 45 seconds). Check effective image revision, role, resource limits and + proxy isolation. Capture unrelated shared-host service health before and + after the scoped candidate; do not restart the shared proxy. +6. Before customer routing, qualify canonical public TLS/SNI and direct + authenticated HTTP, V2/V3, exact upload readback, cross-origin sync/revocation, + drain/reconnect and worker handoff. Verify forged CF/X-Real-IP/Fly-Client-IP/XFF + headers through the real proxy boundary. A `200 /readyz` alone proves none + of those user paths. Perform test mutations only in an authorized account/chat. +7. Rehearse the ordered selection on a nonproduction hostname only after its + separate purchase/routing authorization. Remove the test Fly origin, verify + Nuremberg; remove test Nuremberg too, verify Finland; recover Nuremberg and + Fly in order and verify failback. Measure detection, routing, first + authenticated success, socket catch-up and job recovery separately. Use + isolated fixtures for DB-only network loss while the process remains alive; + do not interrupt production dependencies to prepare the rollout. + +### Rolling upgrades and removing a node + +Qualify the fallback candidates before changing the preferred Fly origin. For +an existing three-origin fleet, update Finland, then Nuremberg, then Fly, one +host at a time. On each host start the new immutable container on a separate +private backend, qualify it, switch only that API's proxy upstream, then SIGTERM +the old container and wait for its real exit/drain. Keep the other origins ready. +Source changes, image publication and local examples alone change no routing. + +For Fly, retain the existing exact-fleet preflight, migration and blue-green +workflow; `--ha=false` preserves the selected public Machine count. It does not +update either Hetzner origin. Record all three effective release identities +before calling a fleet release complete. Mixed releases are acceptable only +for deliberately qualified additive overlap, not indefinitely stale fallbacks. + +Before a planned node stop, verify a ready alternative, withdraw that node from +new ingress (its pool/endpoint or scoped reverse-proxy routing), then send +SIGTERM and wait for the supervisor to observe exit. Disabling a Cloudflare +pool does **not** stop its workers or prove its WebSockets drained. Existing +sockets reconnect or close through application shutdown; the optional CF +session-affinity drain feature is not part of this no-affinity topology. +Stopping a node also removes one readiness/worker participant. Keep the shared +database and remaining writers intact; provider failure alone is no reason to +restore or promote a database. + +To restore a node, verify retained image/config/keys and schema compatibility, +start it, prove direct readiness and authenticated operation, then return it to +its original priority after stable health. Existing sockets on other nodes need +not be forcibly disconnected. If a fallback is intentionally cold/stopped, +document manual startup recovery instead of advertising automatic third-origin +availability. An unhealthy last-resort fallback is still a possible routing +target, so do not treat pool disablement alone as an admission fence. ## Optional automatic Cloudflare routing -Use two ordered pools, one direct origin address per provider, `minimum_origins=1`, -`steering_policy=off`, and no geographic overrides or session affinity. The Fly -pool targets its Fly Proxy address, which can route to two healthy Machines. -Origin addresses must not resolve back to Cloudflare's proxy. Preserve -`Host: api.inline.chat`, verified SNI/TLS and the existing origin-secret rule. - -Start with this HTTPS monitor, subject to the purchased plan: +Use three ordered pools, one direct origin address per location: -```json -{ - "type": "https", "method": "GET", "path": "/readyz", - "header": { "Host": ["api.inline.chat"] }, - "expected_codes": "200", "allow_insecure": false, - "follow_redirects": false, "interval": 60, "timeout": 5, - "retries": 1, "consecutive_down": 2, "consecutive_up": 3 -} -``` - -Use Fly then Hetzner in `default_pools`; use Hetzner as `fallback_pool`. -The fallback pool is used even if every pool is unhealthy, so it is a last -attempt, not evidence of health. Monitor serving behavior independently. -These settings target recovery in a few minutes; only a measured drill can -establish the actual recovery time. Traffic switching affects new requests and -connections. Existing WebSockets must reconnect or drain naturally. Health -recovery may automatically return new traffic to Fly; it does not stop either -API or its workers. +| Priority | Pool | Normal destination | +| --- | --- | --- | +| 1 | Fly | Fly Proxy → healthy public API Machine(s) | +| 2 | Nuremberg | Qualified Nuremberg TLS proxy → API container | +| 3 | Finland | Existing shared-host Traefik → API container | + +Use `minimum_origins=1`, `steering_policy=off`, `session_affinity=none`, no +geographic/rule overrides, and ordered `default_pools=[fly, nuremberg, finland]`. +Set Finland as `fallback_pool`. Origin addresses must resolve directly to the +origin, never back to the Cloudflare-proxied hostname. Preserve canonical Host, +verified SNI/TLS and the existing attribution-proof handling. + +The [example](../ha/cloudflare-three-origin.example.json) separates release +intent, monitor/pool create templates and the production LB create template. +Replace every sentinel from actual inventory/qualified receipts and review the +resolved plan. No tool in this preparation applies it. **Creating a load +balancer named `api.inline.chat` takes precedence over the existing A record and +is a production cutover.** The current Create LB schema has no `enabled` field; +do not create it assuming `enabled=false` is a staging fence. Disabled pools +are not a safe substitute. Keep payloads local until routing is authorized; +use a separate test hostname for a later authorized rehearsal. + +The initial monitor is HTTPS `/readyz`, canonical Host, expected HTTP 200, +verified TLS, no redirects, interval 60s, timeout 5s, one retry, two consecutive +failures and three successes. Check plan entitlement; faster intervals are not +assumed. Broker degradation with HTTP 200 is acceptable, so do not require the +JSON status to be `ok`. DB/schema, clock and lifecycle failures must remove an +origin. Cloudflare sends probes from multiple locations and combines health; +keep health polling bounded and coalesced. These settings target a few minutes +of recovery, not instantaneous failover or a measured guarantee. + +When Fly is unhealthy, new connections use Nuremberg; when both are unhealthy, +new connections use Finland. Standard ordered steering automatically returns +new traffic to the highest-priority healthy pool. Recovery hysteresis reduces +flapping. Existing WebSockets stay with their current process until reconnect +or drain. V3 temporary keys can require a fresh handshake at a different node; +permanent authority remains shared. Health recovery does not stop any API or +its workers. If all pools fail, Cloudflare still uses `fallback_pool` without +regard to health; independent user-path monitoring must report the failure. + +Cloudflare's public non-Enterprise price page lists $5/month for two origins +and $5/month per additional origin: **three origins budget $10/month base**, +plus DNS-query usage, optional features and tax. First 500,000 DNS queries are +included, then $0.50 per additional 500,000. A proxied WebSocket frame is not a +DNS query. The live public rate card was retrieved October 15, 2026; verify the +actual account quote and 60s monitor entitlement before purchase. No geo-steering +add-on is needed for this order. See the [published rate card][cf-pricing]. ## Bypass Cloudflare's proxy @@ -191,16 +282,17 @@ If an optional load balancer exists, use the following LB-specific procedure. The load balancer takes precedence over an A/AAAA record with the same name. Editing only the old A record or its orange-cloud state does not bypass the LB. -For this simple two-pool configuration, first verify direct Hetzner TLS and -authenticated service, then PATCH the **load balancer**: +For this three-pool proposal, first choose and verify a healthy direct origin +(for example Nuremberg, then Finland), capture its pool identity, and deliberately +PATCH the **load balancer** to that one verified pool: ```json { "proxied": false, "ttl": 60, "steering_policy": "off", - "default_pools": [""], - "fallback_pool": "" + "default_pools": [""], + "fallback_pool": "" } ``` @@ -209,7 +301,7 @@ steering rules/region overrides that could reintroduce Fly. Check authoritative and recursive answers, then authenticate through the public hostname. Previously cached Cloudflare addresses may remain for their old TTL (normally 300 seconds) and existing sockets need recovery. Restore the saved proxied ordered-pool -configuration only after Cloudflare and Fly have stable health. +configuration only after Cloudflare and the intended origins have stable health. This emergency operation requires Cloudflare's management API/dashboard and DNS updates to work. Healthy authoritative DNS alone does not guarantee that. The @@ -221,7 +313,9 @@ authentication, rate limits and resource bounds must remain effective. The shared PostgreSQL service, object storage, credentials and application release remain common dependencies. An API failover cannot repair a database -outage, R2 outage or a bad release on both origins. Database HA and restore +outage, R2 outage or a bad release on all origins. Nuremberg and Finland still +share the Hetzner provider, while Fly adds provider diversity; Finland also +shares execution capacity with other applications and backups. Database HA and restore drills are separate changes. Optional Redis does not make PostgreSQL optional. References: [Cloudflare DNS precedence](https://developers.cloudflare.com/load-balancing/load-balancers/dns-records/), @@ -229,3 +323,5 @@ References: [Cloudflare DNS precedence](https://developers.cloudflare.com/load-b [standard steering](https://developers.cloudflare.com/load-balancing/understand-basics/traffic-steering/steering-policies/standard-options/), [Origin CA limitations](https://developers.cloudflare.com/ssl/origin-configuration/origin-ca/), [Traefik 2.10 forwarding source](https://github.com/traefik/traefik/blob/v2.10/pkg/middlewares/forwardedheaders/forwarded_header.go). + +[cf-pricing]: https://www.cloudflare.com/sv-se/application-services/products/website-optimization/ diff --git a/server/ha/cloudflare-three-origin.example.json b/server/ha/cloudflare-three-origin.example.json new file mode 100644 index 000000000..8d1028df7 --- /dev/null +++ b/server/ha/cloudflare-three-origin.example.json @@ -0,0 +1,196 @@ +{ + "status": "local-proposal-only", + "warnings": [ + "This document is not a Cloudflare API request and has no apply/network code.", + "Nested templates are incomplete; replace every sentinel from verified inventory.", + "Creating the production hostname load balancer is a routing cutover, even if existing DNS is unchanged.", + "Create Load Balancer has no enabled field; disabled pools do not make production staging safe.", + "Do not run templates or purchase resources without separate authorization." + ], + "hostProfile": "../deploy/ha-node-profile.json", + "release": { + "sourceCommit": null, + "successfulWorkflowRun": null, + "runtime_image": null, + "standby_image": null, + "requirements": [ + "Both artifacts must be immutable registry references with @sha256 digests.", + "runtime_image is qualified linux/amd64; standby_image is qualified linux/arm64.", + "Both image revision labels must equal sourceCommit; digests are architecture-specific.", + "All CI jobs must pass; image publication alone is not qualification.", + "Retain compatible rollback artifacts and configuration for each architecture." + ] + }, + "preferredOrder": [ + "fly", + "nuremberg", + "finland" + ], + "origins": [ + { + "name": "fly", + "architecture": "amd64", + "imageArtifact": "runtime_image", + "directAddress": null, + "poolId": null, + "qualificationReceipt": null + }, + { + "name": "nuremberg", + "architecture": "arm64", + "imageArtifact": "standby_image", + "directAddress": null, + "poolId": null, + "qualificationReceipt": null + }, + { + "name": "finland", + "architecture": "arm64", + "imageArtifact": "standby_image", + "directAddress": null, + "poolId": null, + "qualificationReceipt": null + } + ], + "cloudflareCreateTemplates": { + "monitor": { + "type": "https", + "method": "GET", + "port": 443, + "path": "/readyz", + "header": { + "Host": [ + "api.inline.chat" + ] + }, + "expected_codes": "200", + "allow_insecure": false, + "follow_redirects": false, + "interval": 60, + "timeout": 5, + "retries": 1, + "consecutive_down": 2, + "consecutive_up": 3 + }, + "pools": [ + { + "name": "inline-api-fly", + "enabled": true, + "minimum_origins": 1, + "monitor": "REPLACE_WITH_QUALIFIED_MONITOR_ID", + "check_regions": [ + "WEU" + ], + "origins": [ + { + "name": "fly", + "address": "REPLACE_WITH_VERIFIED_FLY_DIRECT_ADDRESS", + "enabled": true, + "weight": 1, + "header": { + "Host": [ + "api.inline.chat" + ] + }, + "port": 443 + } + ] + }, + { + "name": "inline-api-nuremberg", + "enabled": true, + "minimum_origins": 1, + "monitor": "REPLACE_WITH_QUALIFIED_MONITOR_ID", + "check_regions": [ + "WEU" + ], + "origins": [ + { + "name": "nuremberg", + "address": "REPLACE_WITH_VERIFIED_NUREMBERG_DIRECT_ADDRESS", + "enabled": true, + "weight": 1, + "header": { + "Host": [ + "api.inline.chat" + ] + }, + "port": 443 + } + ] + }, + { + "name": "inline-api-finland", + "enabled": true, + "minimum_origins": 1, + "monitor": "REPLACE_WITH_QUALIFIED_MONITOR_ID", + "check_regions": [ + "WEU" + ], + "origins": [ + { + "name": "finland", + "address": "REPLACE_WITH_VERIFIED_FINLAND_DIRECT_ADDRESS", + "enabled": true, + "weight": 1, + "header": { + "Host": [ + "api.inline.chat" + ] + }, + "port": 443 + } + ] + } + ], + "productionLoadBalancer": { + "name": "api.inline.chat", + "description": "Fly preferred; Nuremberg first fallback; Finland second fallback", + "proxied": true, + "steering_policy": "off", + "session_affinity": "none", + "default_pools": [ + "REPLACE_WITH_FLY_POOL_ID", + "REPLACE_WITH_NUREMBERG_POOL_ID", + "REPLACE_WITH_FINLAND_POOL_ID" + ], + "fallback_pool": "REPLACE_WITH_FINLAND_POOL_ID", + "adaptive_routing": { + "failover_across_pools": false + }, + "country_pools": {}, + "region_pools": {}, + "pop_pools": {}, + "rules": [] + } + }, + "qualification": { + "status": "not-qualified", + "separateTestHostname": null, + "required": [ + "Current source/image/config/key compatibility at every origin", + "Independent PostgreSQL/storage/provider access and host capacity", + "Canonical public TLS/SNI with sustainable renewal", + "Trusted proxy IP handling and unpublished backend port", + "Effective customer-origin Full(strict) certificate validation and matching rule overrides verified", + "Cross-node authorization and realtime recovery contract", + "Concurrent worker claims/fencing/provider effects contract", + "Nonproduction Fly-to-Nuremberg-to-Finland selection and failback rehearsal", + "Measured authenticated reconnect/catch-up and graceful drain", + "Account quote and monitor entitlement", + "Separate authorization before public hostname LB creation" + ] + }, + "customerOriginTlsIntent": { + "qualificationStatus": "not-verified", + "scheme": "https", + "port": 443, + "requiredEffectivePolicy": "Full (strict), ssl=strict, or reviewed equivalent validating origin certificates for api.inline.chat", + "requiredRuleReview": [ + "Origin Rules", + "Configuration Rules", + "legacy Page Rules" + ], + "warning": "Monitor allow_insecure=false validates probes only; inspect effective customer-origin mode, scheme, port, Host and SNI overrides. Do not change zone-wide SSL blindly." + } +} diff --git a/server/scripts/qualify-ha-host.test.ts b/server/scripts/qualify-ha-host.test.ts new file mode 100644 index 000000000..74dd7c79e --- /dev/null +++ b/server/scripts/qualify-ha-host.test.ts @@ -0,0 +1,81 @@ +import { expect, test } from "bun:test" +import { parseHostMetadata, qualifyHaHost, type ReleaseBinding } from "./qualify-ha-host" + +const release: ReleaseBinding = { + arm64Image: `registry.example/inline-api@sha256:${"a".repeat(64)}`, + sourceRevision: "b".repeat(40), + rollbackArm64Image: `registry.example/inline-api@sha256:${"c".repeat(64)}`, + rollbackSourceRevision: "d".repeat(40), +} + +const metadata = () => ({ + image: { reference: release.arm64Image, sourceRevision: release.sourceRevision, architecture: "arm64" }, + runtime: { + running: true, readyzStatus: 200, role: "all", distributedMode: true, + queryPoolMax: 3, shutdownTimeoutMs: 40_000, ingressHost: "api.inline.chat", + ingressMode: "cloudflare-optional", directIpSource: "x-real-ip", + nativeProtocolEnabled: true, clusterServicesEnabled: true, migrationOnBoot: false, + }, + container: { + cpuLimit: 2, memoryBytes: 1_610_612_736, memoryPlusSwapBytes: 1_610_612_736, + restartPolicy: "unless-stopped", stopGraceSeconds: 45, publishedHostPorts: [], + privateBackendNetwork: true, fixedContainerName: false, logMaxSize: "2m", logMaxFiles: 2, + }, + health: { path: "/readyz", intervalSeconds: 5, timeoutSeconds: 3, retries: 3, startPeriodSeconds: 15 }, + tls: { host: "api.inline.chat", verified: true, expiresAt: "2030-01-01T00:00:00Z" }, + isolation: { productionCredentials: false, publicRouting: false, syntheticDatabase: true }, +}) + +test("matching fixture metadata never claims production acceptance", () => { + const result = qualifyHaHost("nbg", metadata(), release, 0) + expect(result).toMatchObject({ state: "matches-runtime-profile", fixtureOnly: true, productionQualified: false, missing: [], failed: [] }) + expect(result.remaining).toContain("Cross-node delivery and worker ownership") +}) + +test("readiness from the existing reduced artifact harness does not qualify full API behavior", () => { + const input = metadata() + Object.assign(input.runtime, { role: "api", nativeProtocolEnabled: false, clusterServicesEnabled: false }) + const result = qualifyHaHost("nbg", input, release, 0) + expect(result.state).toBe("mismatch") + expect(result.failed).toEqual(["runtime.role", "runtime.nativeProtocolEnabled", "runtime.clusterServicesEnabled"]) +}) + +test("retained old source or a mutable image cannot satisfy release identity", () => { + const input = metadata() + input.image.sourceRevision = "e".repeat(40) + expect(qualifyHaHost("nbg", input, release, 0).failed).toContain("image.sourceRevision") + expect(qualifyHaHost("nbg", metadata(), { ...release, arm64Image: "registry.example/inline-api:latest" }, 0).failed) + .toContain("release.arm64Image") +}) + +test("unbound release placeholders and unknown observations stay incomplete", () => { + const result = qualifyHaHost("finland", {}) + expect(result.state).toBe("incomplete") + expect(result.missing).toContain("release.arm64Image") + expect(result.missing).toContain("runtime.role") + expect(result.productionQualified).toBe(false) +}) + +test("Finland caps, supervisor drain and certificate expiration are checked", () => { + const input = metadata() + input.container.stopGraceSeconds = 30 + input.tls.expiresAt = "2026-10-15T00:00:00Z" + const result = qualifyHaHost("finland", input, release, Date.parse("2026-10-15T01:00:00Z")) + expect(result.failed).toEqual(["container.memoryBytes", "container.memoryPlusSwapBytes", "container.stopGraceSeconds", "tls.expiresAt"]) +}) + +test("raw provider or environment payloads fail closed without echoing secrets", () => { + for (const input of [ + { Config: { Env: ["SECRET=synthetic-credential"] } }, + { runtime: { role: "all", DATABASE_URL: "postgres://synthetic-credential" } }, + { constructor: "synthetic-credential" }, + { container: { publishedHostPorts: [8000] } }, + ]) { + let output = "" + try { output = JSON.stringify(parseHostMetadata(input)) } + catch (cause) { output = cause instanceof Error ? cause.message : "rejected" } + expect(output).not.toContain("synthetic-credential") + expect(output).not.toContain("DATABASE_URL") + expect(output).not.toContain("8000") + } +}) diff --git a/server/scripts/qualify-ha-host.ts b/server/scripts/qualify-ha-host.ts new file mode 100644 index 000000000..b9d4b977f --- /dev/null +++ b/server/scripts/qualify-ha-host.ts @@ -0,0 +1,150 @@ +import profile from "../deploy/ha-node-profile.json" + +type RecordValue = Record +export type NodeName = keyof typeof profile.nodes +export type ReleaseBinding = { + arm64Image: string | null + sourceRevision: string | null + rollbackArm64Image: string | null + rollbackSourceRevision: string | null +} + +const imagePattern = /^[a-z0-9][a-z0-9.:-]*(?:\/[a-z0-9][a-z0-9._-]*)+@sha256:[a-f0-9]{64}$/ +const sourcePattern = /^[a-f0-9]{40}$/ +const sections = { + image: ["reference", "sourceRevision", "architecture"], + runtime: ["running", "readyzStatus", "role", "distributedMode", "queryPoolMax", "shutdownTimeoutMs", "ingressHost", "ingressMode", "directIpSource", "nativeProtocolEnabled", "clusterServicesEnabled", "migrationOnBoot"], + container: ["cpuLimit", "memoryBytes", "memoryPlusSwapBytes", "restartPolicy", "stopGraceSeconds", "publishedHostPorts", "privateBackendNetwork", "fixedContainerName", "logMaxSize", "logMaxFiles"], + health: ["path", "intervalSeconds", "timeoutSeconds", "retries", "startPeriodSeconds"], + tls: ["host", "verified", "expiresAt"], + isolation: ["productionCredentials", "publicRouting", "syntheticDatabase"], +} as const + +const isRecord = (value: unknown): value is RecordValue => + typeof value === "object" && value !== null && !Array.isArray(value) + +// Consume deliberately sanitized metadata, never docker inspect, environment +// dumps, raw Coolify responses, logs or resolved Compose output. Unknown keys +// are rejected without echoing their names or values. +export const parseHostMetadata = (input: unknown): RecordValue => { + if (!isRecord(input) || Object.keys(input).some((key) => !Object.hasOwn(sections, key))) { + throw new Error("Expected sanitized host metadata sections.") + } + for (const [key, allowed] of Object.entries(sections)) { + const value = input[key] + if (value === undefined) continue + if (!isRecord(value) || Object.keys(value).some((field) => !new Set(allowed).has(field))) { + throw new Error("Unexpected host metadata fields; provide an explicit safe projection.") + } + if (Object.values(value).some((field) => + field !== null && typeof field === "object" && + (!Array.isArray(field) || field.length !== 0))) { + throw new Error("Host metadata values must be scalar or an empty ports list.") + } + } + return input +} + +export const qualifyHaHost = ( + node: NodeName, + metadata: unknown, + release: ReleaseBinding = profile.release, + now = Date.now(), +) => { + const input = parseHostMetadata(metadata) + const missing: string[] = [] + const failed: string[] = [] + const matched: string[] = [] + const check = (section: string, field: string, expected: unknown) => { + const values = input[section] + const actual = isRecord(values) ? values[field] : undefined + const key = `${section}.${field}` + if (actual === undefined || actual === null) missing.push(key) + else if (Array.isArray(expected) ? Array.isArray(actual) && actual.length === 0 : actual === expected) matched.push(key) + else failed.push(key) + } + for (const [key, pattern] of [ + ["arm64Image", imagePattern], ["sourceRevision", sourcePattern], + ["rollbackArm64Image", imagePattern], ["rollbackSourceRevision", sourcePattern], + ] as const) { + const value = release[key] + if (value === null) missing.push(`release.${key}`) + else if (!pattern.test(value)) failed.push(`release.${key}`) + } + if (release.arm64Image !== null) check("image", "reference", release.arm64Image) + if (release.sourceRevision !== null) check("image", "sourceRevision", release.sourceRevision) + check("image", "architecture", profile.common.architecture) + const expectedRuntime = { + running: true, + readyzStatus: profile.common.health.expectedStatus, + role: profile.common.environment.INLINE_PROCESS_ROLE, + distributedMode: true, + queryPoolMax: Number(profile.common.environment.INLINE_DATABASE_QUERY_POOL_MAX), + shutdownTimeoutMs: Number(profile.common.environment.SHUTDOWN_TIMEOUT_MS), + ingressHost: profile.common.environment.INLINE_INGRESS_HOST, + ingressMode: profile.common.environment.INLINE_INGRESS_MODE, + directIpSource: profile.common.environment.INLINE_INGRESS_DIRECT_IP_SOURCE, + nativeProtocolEnabled: profile.common.nativeProtocolEnabled, + clusterServicesEnabled: profile.common.clusterServicesEnabled, + migrationOnBoot: profile.common.migrationOnBoot, + } + for (const [key, expected] of Object.entries(expectedRuntime)) check("runtime", key, expected) + const expectedContainer = { + cpuLimit: profile.nodes[node].cpuLimit, + memoryBytes: profile.nodes[node].memoryBytes, + memoryPlusSwapBytes: profile.nodes[node].memoryPlusSwapBytes, + restartPolicy: profile.common.restartPolicy, + stopGraceSeconds: profile.common.stopGraceSeconds, + publishedHostPorts: profile.common.publishedHostPorts, + privateBackendNetwork: profile.common.privateBackendNetwork, + fixedContainerName: profile.common.fixedContainerName, + logMaxSize: profile.common.logMaxSize, + logMaxFiles: profile.common.logMaxFiles, + } + for (const [key, expected] of Object.entries(expectedContainer)) check("container", key, expected) + for (const [key, expected] of Object.entries(profile.common.health)) { + if (key !== "expectedStatus") check("health", key, expected) + } + check("tls", "host", profile.common.canonicalTlsHost) + check("tls", "verified", true) + const tls = input["tls"] + const expiry = isRecord(tls) ? tls["expiresAt"] : undefined + if (expiry === undefined || expiry === null) missing.push("tls.expiresAt") + else if (typeof expiry !== "string" || !Number.isFinite(Date.parse(expiry)) || Date.parse(expiry) <= now) failed.push("tls.expiresAt") + else matched.push("tls.expiresAt") + const isolation = input["isolation"] + const fixtureOnly = isRecord(isolation) && isolation["syntheticDatabase"] === true && + isolation["productionCredentials"] === false && isolation["publicRouting"] === false + return { + node, + state: failed.length ? "mismatch" : missing.length ? "incomplete" : "matches-runtime-profile", + scope: "supplied sanitized metadata only", + fixtureOnly, + productionQualified: false, + matched, + missing, + failed, + remaining: ["Live authenticated V2/V3 and catch-up", "Cross-node delivery and worker ownership", "Real proxy header and TLS renewal boundary", "Measured drain, rollback and routing transition"], + } +} + +const main = async () => { + const args = process.argv.slice(2) + if (args.length !== 4 || args[0] !== "--node" || !["nbg", "finland"].includes(args[1] ?? "") || args[2] !== "--metadata") { + throw new Error("Usage: bun --no-env-file server/scripts/qualify-ha-host.ts --node nbg|finland --metadata SANITIZED_JSON") + } + if (!args[3]?.endsWith(".json")) throw new Error("Expected a sanitized JSON metadata file.") + const file = Bun.file(args[3]) + if (file.size > 65_536) throw new Error("Sanitized metadata exceeds the 64 KiB bound.") + const result = qualifyHaHost(args[1] as NodeName, await file.json()) + console.log(JSON.stringify(result, null, 2)) + if (result.state !== "matches-runtime-profile") process.exitCode = 1 +} + +if (import.meta.main) { + main().catch(() => { + // Parser/provider inputs can contain secrets; do not echo them or errors. + console.error("HA host qualification failed. Supply a selected node and bounded sanitized metadata; no input values were emitted.") + process.exitCode = 1 + }) +} From 8c8f90a74a75c954fc84ab9c281f1b3ab9d39280 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Thu, 15 Oct 2026 21:19:08 +0330 Subject: [PATCH 09/14] infra: reconcile host log caps and record replay planning costs --- server/BENCHMARKING.md | 22 ++++++++++++++++++++++ server/deploy/ha-node-profile.json | 13 ++++++++----- server/docs/hetzner-standby.md | 2 +- server/scripts/qualify-ha-host.test.ts | 4 +++- server/scripts/qualify-ha-host.ts | 4 ++-- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/server/BENCHMARKING.md b/server/BENCHMARKING.md index e8a3476eb..545b43434 100644 --- a/server/BENCHMARKING.md +++ b/server/BENCHMARKING.md @@ -297,3 +297,25 @@ repeatable-read authorization snapshot. The `history.50` fixture now costs ten SQL commands, down from eleven; user/self peer resolution and DM creation keep their existing path. Snapshot tests cover concurrent edits and deletion, current membership, outsider rejection, and unchanged missing-chat errors. + + +### Recorded planning tradeoff + +A local PostgreSQL14.24/Bun1.4/M1 run on source `8e3adbd351` used three warmups, +20 samples, and identical fixtures under access/auto and all-custom modes. +At nominal added RTT6, measured SELECT1 calibration was7.2–7.7ms: + +| Scenario | SQL | Exchanges access/custom | Median ms access/custom | +| --- | ---: | ---: | ---: | +| history.50 |10|17/10|162.81/99.63| +| updates.user-replay.100 |17|29/17|253.67/162.73| +| updates.user-replay.revoked.100 |17|29/17|242.59/162.33| + +Separate ten-replay, single-connection probes observed5custom/5generic plans +per access statement under auto, versus10custom/0generic under all-custom. +EXPLAIN EXECUTE planning observations were0.05–0.08ms with generic reuse and +2.3–2.6ms with custom planning. Those are planning observations, not execution +latency distributions. Native-RTT full-handler differences were only0.65–1.20ms +and do not establish a production improvement. Small tables, simple grants and +sequential A/B sampling leave production cardinality/skew and tail behavior +unqualified. Keep broad preparation opt-in despite the transport-sensitive gain. diff --git a/server/deploy/ha-node-profile.json b/server/deploy/ha-node-profile.json index e708237c3..0093df952 100644 --- a/server/deploy/ha-node-profile.json +++ b/server/deploy/ha-node-profile.json @@ -15,8 +15,6 @@ "fixedContainerName": false, "restartPolicy": "unless-stopped", "stopGraceSeconds": 45, - "logMaxSize": "2m", - "logMaxFiles": 2, "health": { "path": "/readyz", "intervalSeconds": 5, @@ -48,14 +46,18 @@ "cpuLimit": 2, "memoryBytes": 1610612736, "memoryPlusSwapBytes": 1610612736, - "description": "First fallback after Fly. Reserve memory for two containers during replacement on the existing 4 GB host." + "description": "First fallback after Fly. Reserve memory for two containers during replacement on the existing 4 GB host.", + "logMaxSize": "10m", + "logMaxFiles": 3 }, "finland": { "priority": 3, "cpuLimit": 2, "memoryBytes": 2147483648, "memoryPlusSwapBytes": 2147483648, - "description": "Second fallback after Nuremberg. Preserve existing shared applications and backup jobs; reserve two containers for replacement." + "description": "Second fallback after Nuremberg. Preserve existing shared applications and backup jobs; reserve two containers for replacement.", + "logMaxSize": "2m", + "logMaxFiles": 2 } }, "operation": { @@ -66,6 +68,7 @@ "rollback": "Deploy the recorded compatible rollback digest through the same resource; no automatic database rollback.", "fixtureQualification": "Use a separate synthetic database and no production credentials or public routes. The existing artifact smoke does not qualify native protocol, distributed delivery, or workers.", "workerActivation": "The all-role expectation is conditional on the concurrent-worker review and isolated qualification. Do not start real workers merely to satisfy this profile.", - "databaseBudget": "Each process uses up to three query plus one health connections. Count every running origin and replacement; verify actual database capacity before starting overlap." + "databaseBudget": "Each process uses up to three query plus one health connections. Count every running origin and replacement; verify actual database capacity before starting overlap.", + "logging": "Retain verified bounded per-node logs. The installed Coolify single-container path does not support per-app JSON log options; do not change the shared Docker daemon to force a smaller cap." } } diff --git a/server/docs/hetzner-standby.md b/server/docs/hetzner-standby.md index 9f82b2789..2cbdaf72e 100644 --- a/server/docs/hetzner-standby.md +++ b/server/docs/hetzner-standby.md @@ -47,7 +47,7 @@ capacity. Qualify actual failover load within them; do not automatically raise them when the host is busy. Budget two containers on each host for its rolling overlap. The host qualifier consumes sanitized metadata; passing it does not establish application, TLS, broker or worker acceptance. -Keep Docker log retention bounded for this resource. Capture the current API +Retain verified bounded Docker logging: the Nuremberg canary uses 10 MiB × 3 files; the retained Finland resource uses 2 MiB × 2 files. The installed Coolify single-container path does not support per-app JSON log limits through custom Docker options. Do not change a shared Docker daemon or switch deployment methods merely to reduce these small caps. Capture the current API configuration and a baseline of the other public applications before starting, then check their availability during qualification. Roll back only the API resource if it harms host headroom or unrelated services. diff --git a/server/scripts/qualify-ha-host.test.ts b/server/scripts/qualify-ha-host.test.ts index 74dd7c79e..d15334735 100644 --- a/server/scripts/qualify-ha-host.test.ts +++ b/server/scripts/qualify-ha-host.test.ts @@ -19,7 +19,7 @@ const metadata = () => ({ container: { cpuLimit: 2, memoryBytes: 1_610_612_736, memoryPlusSwapBytes: 1_610_612_736, restartPolicy: "unless-stopped", stopGraceSeconds: 45, publishedHostPorts: [], - privateBackendNetwork: true, fixedContainerName: false, logMaxSize: "2m", logMaxFiles: 2, + privateBackendNetwork: true, fixedContainerName: false, logMaxSize: "10m", logMaxFiles: 3, }, health: { path: "/readyz", intervalSeconds: 5, timeoutSeconds: 3, retries: 3, startPeriodSeconds: 15 }, tls: { host: "api.inline.chat", verified: true, expiresAt: "2030-01-01T00:00:00Z" }, @@ -58,6 +58,8 @@ test("unbound release placeholders and unknown observations stay incomplete", () test("Finland caps, supervisor drain and certificate expiration are checked", () => { const input = metadata() + input.container.logMaxSize = "2m" + input.container.logMaxFiles = 2 input.container.stopGraceSeconds = 30 input.tls.expiresAt = "2026-10-15T00:00:00Z" const result = qualifyHaHost("finland", input, release, Date.parse("2026-10-15T01:00:00Z")) diff --git a/server/scripts/qualify-ha-host.ts b/server/scripts/qualify-ha-host.ts index b9d4b977f..ff508531a 100644 --- a/server/scripts/qualify-ha-host.ts +++ b/server/scripts/qualify-ha-host.ts @@ -98,8 +98,8 @@ export const qualifyHaHost = ( publishedHostPorts: profile.common.publishedHostPorts, privateBackendNetwork: profile.common.privateBackendNetwork, fixedContainerName: profile.common.fixedContainerName, - logMaxSize: profile.common.logMaxSize, - logMaxFiles: profile.common.logMaxFiles, + logMaxSize: profile.nodes[node].logMaxSize, + logMaxFiles: profile.nodes[node].logMaxFiles, } for (const [key, expected] of Object.entries(expectedContainer)) check("container", key, expected) for (const [key, expected] of Object.entries(profile.common.health)) { From 930eee40fb26853d12677abdfdd5cf8d11225358 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Fri, 16 Oct 2026 17:27:59 +0330 Subject: [PATCH 10/14] ci: allow manual branch image qualification without deployment --- .github/workflows/server-deploy.yml | 28 ++++--- .github/workflows/server-test.yml | 30 ++++++- scripts/ci/fly-release-machines.test.ts | 107 ++++++++++++++++++------ server/docs/fly-deployment.md | 21 +++-- 4 files changed, 139 insertions(+), 47 deletions(-) diff --git a/.github/workflows/server-deploy.yml b/.github/workflows/server-deploy.yml index ea2017792..6c843f4df 100644 --- a/.github/workflows/server-deploy.yml +++ b/.github/workflows/server-deploy.yml @@ -1,12 +1,13 @@ name: Deploy Server -# Release only the SHA selected by dispatching main; no arbitrary ref input or -# push trigger. Configure the production environment to allow main only. +# Manual publication may select a branch in this repository. Production releases +# remain main-only; no arbitrary ref input or push trigger. Restrict production +# environments to main as an independent deployment fence. on: workflow_dispatch: inputs: publish_only: - description: Qualify and publish the image for rehearsal or initial bootstrap; do not deploy + description: Qualify and publish this branch's images for rehearsal; do not deploy type: boolean default: false bootstrap: @@ -23,7 +24,7 @@ concurrency: jobs: validate: - name: Require a manual main release + name: Validate manual publication or main release runs-on: ubuntu-latest steps: - env: @@ -31,8 +32,14 @@ jobs: BOOTSTRAP: ${{ inputs.bootstrap }} run: | test "$GITHUB_EVENT_NAME" = workflow_dispatch - test "$GITHUB_REF" = refs/heads/main - test "$PUBLISH_ONLY:$BOOTSTRAP" != true:true + test "$GITHUB_REPOSITORY" = inline-chat/inline + test "$GITHUB_REF_TYPE" = branch + [[ "$GITHUB_REF" == refs/heads/* ]] || exit 1 + case "$PUBLISH_ONLY:$BOOTSTRAP" in + true:false) ;; + false:false|false:true) test "$GITHUB_REF" = refs/heads/main ;; + *) exit 1 ;; + esac qualify: name: Qualify and publish the exact release image @@ -40,6 +47,7 @@ jobs: uses: ./.github/workflows/server-test.yml with: publish_image: true + publish_only: ${{ inputs.publish_only }} secrets: # Repository/org secret: environment secrets are unavailable to this call. FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} @@ -64,7 +72,7 @@ jobs: preflight: name: Record a healthy public Machine baseline before DDL needs: qualify - if: ${{ !inputs.publish_only }} + if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} runs-on: ubuntu-latest timeout-minutes: 5 environment: production @@ -93,7 +101,7 @@ jobs: migrate: name: Migrate and verify with the qualified image needs: [qualify, preflight] - if: ${{ !inputs.publish_only }} + if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} runs-on: ubuntu-latest timeout-minutes: 15 environment: production @@ -126,7 +134,7 @@ jobs: deploy: name: Replace the public Machines and verify readiness needs: [qualify, preflight, migrate] - if: ${{ !inputs.publish_only }} + if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} runs-on: ubuntu-latest timeout-minutes: 20 environment: production @@ -197,7 +205,7 @@ jobs: activate: name: Enable workers on the same qualified bootstrap image needs: [qualify, deploy] - if: ${{ inputs.bootstrap }} + if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only && inputs.bootstrap }} runs-on: ubuntu-latest timeout-minutes: 20 # Required-reviewer protection must be configured before bootstrap dispatch. diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 10c8dad84..5baa062e3 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -7,6 +7,10 @@ on: description: Publish the qualified runtime image for a manual release type: boolean default: false + publish_only: + description: Allow a manual same-repository branch publication without production release + type: boolean + default: false secrets: FLY_API_TOKEN: required: false @@ -57,11 +61,20 @@ jobs: --health-timeout 5s --health-retries 10 steps: - - name: Require manual main context when publishing + - name: Require manual branch publication or main release if: ${{ inputs.publish_image }} + env: + PUBLISH_ONLY: ${{ inputs.publish_only }} run: | test "$GITHUB_EVENT_NAME" = workflow_dispatch - test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_REPOSITORY" = inline-chat/inline + test "$GITHUB_REF_TYPE" = branch + [[ "$GITHUB_REF" == refs/heads/* ]] || exit 1 + case "$PUBLISH_ONLY" in + true) ;; + false) test "$GITHUB_REF" = refs/heads/main ;; + *) exit 1 ;; + esac - uses: actions/checkout@v6 - uses: oven-sh/setup-bun@v2 with: @@ -141,11 +154,20 @@ jobs: --health-timeout 5s --health-retries 10 steps: - - name: Require manual main context when publishing + - name: Require manual branch publication or main release if: ${{ inputs.publish_image }} + env: + PUBLISH_ONLY: ${{ inputs.publish_only }} run: | test "$GITHUB_EVENT_NAME" = workflow_dispatch - test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_REPOSITORY" = inline-chat/inline + test "$GITHUB_REF_TYPE" = branch + [[ "$GITHUB_REF" == refs/heads/* ]] || exit 1 + case "$PUBLISH_ONLY" in + true) ;; + false) test "$GITHUB_REF" = refs/heads/main ;; + *) exit 1 ;; + esac - uses: actions/checkout@v6 - uses: oven-sh/setup-bun@v2 with: diff --git a/scripts/ci/fly-release-machines.test.ts b/scripts/ci/fly-release-machines.test.ts index a942e6477..1c52b0c8d 100644 --- a/scripts/ci/fly-release-machines.test.ts +++ b/scripts/ci/fly-release-machines.test.ts @@ -63,23 +63,38 @@ test("an empty fleet fails, and changed IDs, count or image cannot match the rec expect(summary([{ ...machine(), image_ref: { ...machine().image_ref, digest: "sha256:" + "2".repeat(64) } }])).not.toEqual(before) }) -test("the reusable workflow publishing guard only accepts manual main dispatch", async () => { +test("both image publishing guards require manual same-repository branches and explicit publish-only off main", async () => { const workflow = Bun.YAML.parse(await Bun.file(resolve(import.meta.dir, "../../.github/workflows/server-test.yml")).text()) as { - jobs: Record + on: { workflow_call: { inputs: Record } } + jobs: Record }[] }> } + expect(workflow.on.workflow_call.inputs.publish_only).toMatchObject({ type: "boolean", default: false }) for (const job of ["container", "container-arm"]) { - const guard = workflow.jobs[job]?.steps.find((step) => step.name === "Require manual main context when publishing") + const steps = workflow.jobs[job]!.steps + const guard = steps.find((step) => step.name === "Require manual branch publication or main release") expect(guard?.run).toBeDefined() - for (const [event, ref, allowed] of [ - ["workflow_dispatch", "refs/heads/main", true], - ["push", "refs/heads/main", false], - ["pull_request", "refs/heads/main", false], - ["workflow_dispatch", "refs/heads/other", false], - ] as const) { - const result = Bun.spawnSync(["bash", "-e", "-c", guard!.run!], { - env: { PATH: process.env.PATH, GITHUB_EVENT_NAME: event, GITHUB_REF: ref }, - }) - expect(result.exitCode === 0, `${job} ${event} ${ref}`).toBe(allowed) + expect(guard?.if).toBe("${{ inputs.publish_image }}") + expect(guard?.env?.PUBLISH_ONLY).toBe("${{ inputs.publish_only }}") + // Reject before checkout/build or registry authentication can expose secrets. + expect(steps.indexOf(guard!)).toBe(0) + for (const event of ["workflow_dispatch", "push", "pull_request", "pull_request_target"]) { + for (const repository of ["inline-chat/inline", "outsider/inline"]) { + for (const [ref, refType] of [ + ["refs/heads/main", "branch"], ["refs/heads/codex/multi-node-ha", "branch"], + ["refs/tags/main", "tag"], ["refs/pull/124/merge", "branch"], + ] as const) { + for (const publishOnly of ["true", "false", ""]) { + const allowed = event === "workflow_dispatch" && repository === "inline-chat/inline" + && refType === "branch" && ref.startsWith("refs/heads/") + && (publishOnly === "true" || (publishOnly === "false" && ref === "refs/heads/main")) + const result = Bun.spawnSync(["bash", "-e", "-c", guard!.run!], { + env: { PATH: process.env.PATH, GITHUB_EVENT_NAME: event, GITHUB_REF: ref, + GITHUB_REF_TYPE: refType, GITHUB_REPOSITORY: repository, PUBLISH_ONLY: publishOnly }, + }) + expect(result.exitCode === 0, `${job} ${event} ${repository} ${ref} publish-only=${publishOnly}`).toBe(allowed) + } + } + } } } }) @@ -103,19 +118,33 @@ test("publish-only keeps full CI and excludes every production job; normal and f const workflow = await releaseWorkflow() expect(workflow.on.workflow_dispatch.inputs.publish_only).toMatchObject({ type: "boolean", default: false }) expect(workflow.on.workflow_dispatch.inputs.bootstrap).toMatchObject({ type: "boolean", default: false }) - expect(workflow.jobs.qualify).toMatchObject({ needs: "validate", uses: "./.github/workflows/server-test.yml", with: { publish_image: true } }) + expect(workflow.jobs.qualify).toMatchObject({ needs: "validate", uses: "./.github/workflows/server-test.yml", + with: { publish_image: true, publish_only: "${{ inputs.publish_only }}" } }) + expect(workflow.jobs.validate?.steps?.[0]?.env).toEqual({ + PUBLISH_ONLY: "${{ inputs.publish_only }}", BOOTSTRAP: "${{ inputs.bootstrap }}", + }) + for (const id of ["preflight", "migrate", "deploy", "activate"]) { + expect(workflow.jobs[id]?.if).toContain("github.ref == 'refs/heads/main'") + expect(workflow.jobs[id]?.if).toContain("!inputs.publish_only") + } // Interpret the workflow's deliberately simple boolean guards and dependency // success requirements, so changing either changes the jobs exercised here. - const runnable = (id: string, publishOnly: boolean, failed?: string, bootstrap = false): boolean => { + const runnable = (id: string, publishOnly: boolean, failed?: string, bootstrap = false, ref = "refs/heads/main"): boolean => { const job = workflow.jobs[id]! if (job.if !== undefined) { - const guard = job.if.match(/^\$\{\{\s*(!?)inputs\.(publish_only|bootstrap)\s*\}\}$/) - if (!guard) throw new Error("Unexpected release job condition") - const input = guard[2] === "bootstrap" ? bootstrap : publishOnly - if (!(guard[1] ? !input : input)) return false + const expression = job.if.match(/^\$\{\{\s*(.*?)\s*\}\}$/)?.[1] + if (!expression) throw new Error("Unexpected release job condition") + const clauses: Record = { + "github.ref == 'refs/heads/main'": ref === "refs/heads/main", + "inputs.publish_only": publishOnly, "!inputs.publish_only": !publishOnly, "inputs.bootstrap": bootstrap, + } + if (!expression.split(" && ").every((clause) => { + if (!(clause in clauses)) throw new Error(`Unexpected release condition: ${clause}`) + return clauses[clause] + })) return false } const needs = typeof job.needs === "string" ? [job.needs] : job.needs ?? [] - return needs.every((dependency) => dependency !== failed && runnable(dependency, publishOnly, failed, bootstrap)) + return needs.every((dependency) => dependency !== failed && runnable(dependency, publishOnly, failed, bootstrap, ref)) } for (const [publishOnly, expected] of [ [true, ["validate", "qualify", "prepared"]], @@ -139,6 +168,18 @@ test("publish-only keeps full CI and excludes every production job; normal and f expect(Object.keys(workflow.jobs).filter((job) => runnable(job, false, undefined, true)).sort()) .toEqual(["activate", "deploy", "migrate", "preflight", "qualify", "validate"]) expect(runnable("activate", false, "deploy", true)).toBe(false) + for (const ref of ["refs/heads/main", "refs/heads/codex/multi-node-ha", "refs/tags/main"]) { + for (const publishOnly of [true, false]) { + for (const bootstrap of [true, false]) { + for (const id of ["preflight", "migrate", "deploy", "activate"]) { + // Test every production job's own fence even if upstream validation + // were to accept a branch or conflicting inputs accidentally. + expect(runnable(id, publishOnly, undefined, bootstrap, ref)) + .toBe(ref === "refs/heads/main" && !publishOnly && (id !== "activate" || bootstrap)) + } + } + } + } expect(workflow.jobs.activate?.environment).toBe("production-cutover") const steps = workflow.jobs.activate?.steps ?? [] const fence = steps.findIndex((step) => step.name === "Verify the predecessor is durably stopped") @@ -147,11 +188,27 @@ test("publish-only keeps full CI and excludes every production job; normal and f expect(steps[fence + 1]?.name).toBe("Activate the same qualified digest with workers") const validate = workflow.jobs.validate?.steps?.[0]?.run if (!validate) throw new Error("Missing dispatch guard") - for (const [publishOnly, bootstrap, permitted] of [[false, false, true], [true, false, true], [false, true, true], [true, true, false]]) { - expect(Bun.spawnSync(["bash", "-e", "-c", validate], { env: { - PATH: process.env.PATH, GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REF: "refs/heads/main", - PUBLISH_ONLY: String(publishOnly), BOOTSTRAP: String(bootstrap), - } }).exitCode === 0).toBe(permitted) + for (const event of ["workflow_dispatch", "push", "pull_request", "pull_request_target"]) { + for (const repository of ["inline-chat/inline", "outsider/inline"]) { + for (const [ref, refType] of [ + ["refs/heads/main", "branch"], ["refs/heads/codex/multi-node-ha", "branch"], + ["refs/tags/main", "tag"], ["refs/pull/124/merge", "branch"], + ] as const) { + for (const publishOnly of ["false", "true", ""]) { + for (const bootstrap of ["false", "true", ""]) { + const permitted = event === "workflow_dispatch" && repository === "inline-chat/inline" + && refType === "branch" && ref.startsWith("refs/heads/") + && ((publishOnly === "true" && bootstrap === "false") + || (publishOnly === "false" && ["true", "false"].includes(bootstrap) && ref === "refs/heads/main")) + expect(Bun.spawnSync(["bash", "-e", "-c", validate], { env: { + PATH: process.env.PATH, GITHUB_EVENT_NAME: event, GITHUB_REF: ref, + GITHUB_REF_TYPE: refType, GITHUB_REPOSITORY: repository, + PUBLISH_ONLY: publishOnly, BOOTSTRAP: bootstrap, + } }).exitCode === 0, `${event} ${repository} ${ref} publish-only=${publishOnly} bootstrap=${bootstrap}`).toBe(permitted) + } + } + } + } } }) diff --git a/server/docs/fly-deployment.md b/server/docs/fly-deployment.md index f32a87705..ceee75e60 100644 --- a/server/docs/fly-deployment.md +++ b/server/docs/fly-deployment.md @@ -79,22 +79,27 @@ does not establish these effective settings. ### Prepare an image for rehearsal or initial bootstrap -When no healthy public fleet exists yet, or an exact-image rehearsal is needed, -dispatch from `main` with `publish_only` enabled: +For an exact-image rehearsal, manually dispatch from the selected branch in +`inline-chat/inline` with `publish_only` enabled. For example, to qualify the HA +candidate without merging it or changing production: ```sh -gh workflow run server-deploy.yml --ref main -f publish_only=true +gh workflow run server-deploy.yml --ref codex/multi-node-ha -f publish_only=true ``` -This runs the entire reusable CI workflow and publishes its tested runtime image -without rebuilding. After all CI succeeds, the run summary records the selected -source SHA and immutable digest. It skips the production environment jobs, -fleet preflight, migrations, and deployment; it never receives the production +This runs the entire reusable CI workflow and publishes its tested amd64 runtime +and native ARM64 Coolify images without rebuilding. After all CI succeeds, the +run summary records the selected source SHA and both immutable digests. +Tags, pull-request refs and fork repositories are rejected. It skips the +production environment jobs, fleet preflight, migrations, and deployment; it +never receives the production database credential. Registry publishing still requires the repository/org Fly token. Use the recorded digest for the separately authorized clone rehearsal and initial Machine bootstrap. This option does not bootstrap or migrate an app. -`publish_only` defaults to `false`. Routine releases retain the healthy-fleet +`publish_only` defaults to `false`; a branch dispatch without it fails before +publication. Production and bootstrap dispatches still require `main` and cannot +run in publish-only mode. Routine releases retain the healthy-fleet preflight before DDL and the immediate baseline recheck before replacement. ### Bootstrap the first API and finish cutover in the same run From 8d7d4f8e6524d2573f001b23e4977d84545553f1 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Fri, 16 Oct 2026 18:31:51 +0330 Subject: [PATCH 11/14] server: add retained release escrow verification --- scripts/docker/server-release-escrow.test.ts | 170 +++++++++++ scripts/docker/server-release-escrow.ts | 279 +++++++++++++++++++ server/docs/server-release-escrow.md | 95 +++++++ 3 files changed, 544 insertions(+) create mode 100644 scripts/docker/server-release-escrow.test.ts create mode 100644 scripts/docker/server-release-escrow.ts create mode 100644 server/docs/server-release-escrow.md diff --git a/scripts/docker/server-release-escrow.test.ts b/scripts/docker/server-release-escrow.test.ts new file mode 100644 index 000000000..f5eaac60b --- /dev/null +++ b/scripts/docker/server-release-escrow.test.ts @@ -0,0 +1,170 @@ +import { expect, test } from "bun:test" +import { createHash } from "node:crypto" +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { resolve } from "node:path" +import { captureEscrow, verifyEscrow } from "./server-release-escrow" + +const hash = (content: string | Buffer) => createHash("sha256").update(content).digest("hex") + +async function run(args: string[], cwd: string): Promise { + const child = Bun.spawn(args, { + cwd, + env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null" }, + stdout: "pipe", stderr: "pipe", + }) + const [output, error] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]) + expect(await child.exited, error).toBe(0) + return output.trim() +} + +async function fixture() { + const temp = await mkdtemp(resolve(tmpdir(), "inline-escrow-")) + const root = resolve(temp, "repo") + await mkdir(root) + const files: Record = { + "package.json": JSON.stringify({ workspaces: ["server"] }), + "bun.lock": "synthetic lock", "bunfig.toml": "", + "server/package.json": JSON.stringify({ name: "@inline-chat/server" }), + "server/src/index.ts": "export const committed = true", + "server/Dockerfile": "FROM scratch", "server/Dockerfile.dockerignore": "**/.env*", + "scripts/docker/prune-workspace.ts": "", "scripts/docker/smoke-artifact.ts": "", + } + for (const [path, content] of Object.entries(files)) { + await mkdir(resolve(root, path, ".."), { recursive: true }) + await writeFile(resolve(root, path), content) + } + await run(["git", "init"], root) + await run(["git", "add", "."], root) + await run(["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "commit", "-m", "fixture"], root) + const revision = await run(["git", "rev-parse", "HEAD"], root) + const archive = async (architecture: string, imageRevision = revision, modern = false) => { + const folder = await mkdtemp(resolve(temp, "image-")) + // Intentionally not runnable: escrow identity is distinct from runtime qualification. + const config = JSON.stringify({ os: "linux", architecture, config: { Labels: { "org.opencontainers.image.revision": imageRevision } } }) + const configPath = modern ? `blobs/sha256/${hash(config)}` : `${hash(config)}.json` + const layer = Buffer.alloc(2 * 1024 * 1024 + 19, 65) + const layerPath = modern ? `blobs/sha256/${hash(layer)}` : "layer.tar" + await mkdir(resolve(folder, configPath, ".."), { recursive: true }) + await writeFile(resolve(folder, configPath), config) + await writeFile(resolve(folder, layerPath), layer) + await writeFile(resolve(folder, "manifest.json"), JSON.stringify([{ Config: configPath, Layers: [layerPath], RepoTags: ["fixture:synthetic"] }])) + const path = `${folder}.tar` + await run(["tar", "-cf", path, "manifest.json", configPath, layerPath], folder) + return path + } + const amd64Archive = await archive("amd64") + const arm64Archive = await archive("arm64", revision, true) + const destination = resolve(temp, "escrow") + return { temp, root, revision, amd64Archive, arm64Archive, destination, archive } +} + +test("retains committed source and two platform identities; verifies offline from a moved directory", async () => { + const f = await fixture() + try { + await writeFile(resolve(f.root, "server/src/index.ts"), "dirty source is excluded") + const result = await captureEscrow(f) + expect(result.sourceRevision).toBe(f.revision) + expect(await readFile(resolve(f.destination, "source/server/src/index.ts"), "utf8")).toBe("export const committed = true") + const moved = resolve(f.temp, "independent-copy") + await run(["cp", "-R", f.destination, moved], f.temp) + await rm(f.root, { recursive: true }) + await rm(f.destination, { recursive: true }) + await rm(f.amd64Archive) + await rm(f.arm64Archive) + const manifest = await verifyEscrow(moved, result.manifestSha256) + expect(manifest.images.map((image) => [image.platform, image.target])).toEqual([["linux/amd64", "runtime"], ["linux/arm64", "coolify"]]) + expect(manifest.verification).toBe("archive-bytes-and-image-config-only") + const output = await run([process.execPath, "--no-env-file", resolve(moved, "tools/server-release-escrow.ts"), "verify", moved, result.manifestSha256], f.temp) + expect(JSON.parse(output).sourceRevision).toBe(f.revision) + } finally { await rm(f.temp, { recursive: true, force: true }) } +}) + +test("rejects archive and source corruption, manifest replacement and unexpected files", async () => { + const f = await fixture() + try { + const result = await captureEscrow(f) + for (const path of ["source/server/src/index.ts", "images/runtime-amd64.tar", "escrow.json"]) { + const file = resolve(f.destination, path) + const original = await readFile(file) + await writeFile(file, Buffer.concat([original, Buffer.from("corruption")])) + await expect(verifyEscrow(f.destination, result.manifestSha256)).rejects.toThrow("checksum mismatch") + await writeFile(file, original) + } + await writeFile(resolve(f.destination, "source/private.key"), "synthetic forbidden file") + await expect(verifyEscrow(f.destination, result.manifestSha256)).rejects.toThrow("Unexpected escrow file") + await rm(resolve(f.destination, "source/private.key")) + await writeFile(resolve(f.destination, "source/extra.ts"), "unrecorded") + await expect(verifyEscrow(f.destination, result.manifestSha256)).rejects.toThrow("inventory mismatch") + } finally { await rm(f.temp, { recursive: true, force: true }) } +}) + +test("refuses symlinked inputs, output reuse and symlinks in a recovered escrow", async () => { + const f = await fixture() + try { + const result = await captureEscrow(f) + await expect(captureEscrow(f)).rejects.toThrow() + const archived = resolve(f.destination, "images/runtime-amd64.tar") + await rm(archived) + await symlink(f.amd64Archive, archived) + await expect(verifyEscrow(f.destination, result.manifestSha256)).rejects.toThrow("Symlinks") + const link = resolve(f.temp, "input-link.tar") + await symlink(f.amd64Archive, link) + await expect(captureEscrow({ ...f, amd64Archive: link, destination: resolve(f.temp, "rejected") })).rejects.toThrow() + expect(await Bun.file(resolve(f.temp, "rejected/escrow.json")).exists()).toBe(false) + } finally { await rm(f.temp, { recursive: true, force: true }) } +}) + +test("refuses mismatched source/platform and incomplete images before writing completion manifest", async () => { + const f = await fixture() + try { + const wrongRevision = await f.archive("amd64", "f".repeat(40)) + for (const [name, archive] of [["platform", f.arm64Archive], ["source", wrongRevision]]) { + const destination = resolve(f.temp, name!) + await expect(captureEscrow({ ...f, amd64Archive: archive!, destination })).rejects.toThrow(name === "platform" ? "linux/amd64" : "source revision") + expect(await Bun.file(resolve(destination, "escrow.json")).exists()).toBe(false) + } + const invalid = resolve(f.temp, "invalid.tar") + await writeFile(invalid, "not a docker archive") + await expect(captureEscrow({ ...f, amd64Archive: invalid, destination: resolve(f.temp, "invalid") })).rejects.toThrow() + } finally { await rm(f.temp, { recursive: true, force: true }) } +}) + +test("rejects manifest path traversal and duplicate identities even with a supplied matching checksum", async () => { + const f = await fixture() + try { + await captureEscrow(f) + const path = resolve(f.destination, "escrow.json") + const original = await readFile(path, "utf8") + for (const kind of ["traversal", "duplicate"]) { + const manifest = JSON.parse(original) + if (kind === "traversal") manifest.files[0].path = "../outside" + else manifest.files.push(manifest.files[0]) + const content = JSON.stringify(manifest) + await writeFile(path, content) + await expect(verifyEscrow(f.destination, hash(content))).rejects.toThrow(kind === "traversal" ? "Invalid file identity" : "Duplicate file identity") + } + } finally { await rm(f.temp, { recursive: true, force: true }) } +}) + +test("rejects duplicate archive members, multiple images and a spoofed config filename checksum", async () => { + const f = await fixture() + try { + for (const kind of ["duplicate", "multiple", "config"]) { + const archive = await f.archive("amd64") + const folder = archive.slice(0, -4) + const manifestPath = resolve(folder, "manifest.json") + const images = JSON.parse(await readFile(manifestPath, "utf8")) + if (kind === "duplicate") { + await run(["tar", "-rf", archive, "manifest.json"], folder) + } else { + if (kind === "multiple") await writeFile(manifestPath, JSON.stringify([...images, ...images])) + else await writeFile(resolve(folder, images[0].Config), "{}") + await run(["tar", "-cf", archive, "manifest.json", images[0].Config, "layer.tar"], folder) + } + await expect(captureEscrow({ ...f, amd64Archive: archive, destination: resolve(f.temp, kind) })).rejects.toThrow( + kind === "duplicate" ? "duplicate archive member" : kind === "multiple" ? "exactly one image" : "config checksum mismatch", + ) + } + } finally { await rm(f.temp, { recursive: true, force: true }) } +}) diff --git a/scripts/docker/server-release-escrow.ts b/scripts/docker/server-release-escrow.ts new file mode 100644 index 000000000..5303dc5b7 --- /dev/null +++ b/scripts/docker/server-release-escrow.ts @@ -0,0 +1,279 @@ +import { createHash } from "node:crypto" +import { constants } from "node:fs" +import { lstat, mkdir, open, readdir } from "node:fs/promises" +import { basename, resolve } from "node:path" +import { isBuildInput, prepareServerContext } from "./server-context" + +type FileIdentity = { path: string; bytes: number; sha256: string } +type ImageIdentity = { + path: string + platform: "linux/amd64" | "linux/arm64" + target: "runtime" | "coolify" + configSha256: string +} +type EscrowManifest = { + version: 1 + sourceRevision: string + verification: "archive-bytes-and-image-config-only" + files: FileIdentity[] + images: ImageIdentity[] +} + +const archivePaths = ["images/runtime-amd64.tar", "images/coolify-arm64.tar"] as const +const toolPaths = ["tools/server-release-escrow.ts", "tools/server-context.ts"] as const +const hex = (value: unknown, length: number): value is string => + typeof value === "string" && new RegExp(`^[a-f0-9]{${length}}$`).test(value) +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex") + +function object(value: unknown): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("Expected an object") + return value as Record +} + +function safeRelative(path: string): boolean { + return path.length > 0 && !path.includes("\\") && path.split("/").every((part) => part !== "" && part !== "." && part !== "..") +} + +function allowedFile(path: string): boolean { + if (!safeRelative(path)) return false + if (archivePaths.some((entry) => entry === path) || toolPaths.some((entry) => entry === path)) return true + return path.startsWith("source/") && isBuildInput(path.slice("source/".length)) +} + +// Bounded memory, no symlink following, and exclusive destination creation. +// Leave a failed capture in place for inspection; only a final manifest marks completion. +async function fingerprint(path: string, destination?: string): Promise> { + const source = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW) + let output: Awaited> | undefined + try { + const before = await source.stat() + if (!before.isFile() || !Number.isSafeInteger(before.size)) throw new Error("Expected a regular file") + if (destination) output = await open(destination, "wx", 0o600) + const hash = createHash("sha256") + const buffer = Buffer.alloc(1024 * 1024) + let bytes = 0 + for (;;) { + const chunk = await source.read(buffer, 0, buffer.length, null) + if (chunk.bytesRead === 0) break + const data = buffer.subarray(0, chunk.bytesRead) + hash.update(data) + if (output) await output.writeFile(data) + bytes += chunk.bytesRead + if (bytes > before.size) throw new Error("File changed during capture") + } + const after = await source.stat() + if (bytes !== before.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) + throw new Error("File changed during capture") + if (output) await output.sync() + return { bytes, sha256: hash.digest("hex") } + } finally { + await output?.close() + await source.close() + } +} + +async function readSmall(path: string, maximum = 1024 * 1024): Promise { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW) + try { + const metadata = await file.stat() + if (!metadata.isFile() || metadata.size > maximum) throw new Error("Oversized or non-regular metadata") + const buffer = Buffer.alloc(maximum + 1) + let bytes = 0 + while (bytes <= maximum) { + const chunk = await file.read(buffer, bytes, buffer.length - bytes, null) + if (!chunk.bytesRead) return buffer.subarray(0, bytes) + bytes += chunk.bytesRead + } + throw new Error("Oversized metadata") + } finally { + await file.close() + } +} + +async function syncFile(path: string) { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW) + try { await file.sync() } finally { await file.close() } +} + +async function tar(archive: string, member?: string): Promise { + const child = Bun.spawn(member ? ["tar", "-xOf", archive, "--", member] : ["tar", "-tf", archive], { + env: { PATH: process.env.PATH, LANG: "C" }, + stdout: "pipe", + stderr: "ignore", + }) + const timer = setTimeout(() => child.kill("SIGKILL"), 120_000) + try { + const chunks: Uint8Array[] = [] + let bytes = 0 + for await (const chunk of child.stdout) { + bytes += chunk.length + if (bytes > 2 * 1024 * 1024) throw new Error("Oversized archive metadata") + chunks.push(chunk) + } + if ((await child.exited) !== 0) throw new Error("Cannot inspect Docker-save archive") + return Buffer.concat(chunks).toString("utf8") + } finally { + clearTimeout(timer) + child.kill("SIGKILL") + await child.exited + } +} + +async function inspectArchive(archive: string, path: string, revision: string, index: number): Promise { + const members = (await tar(archive)).trimEnd().split("\n") + if (new Set(members).size !== members.length || members.some((entry) => !safeRelative(entry.replace(/\/$/, "")))) + throw new Error("Unsafe or duplicate archive member") + if (!members.includes("manifest.json")) throw new Error("Expected an uncompressed Docker-save archive") + const manifest: unknown = JSON.parse(await tar(archive, "manifest.json")) + if (!Array.isArray(manifest) || manifest.length !== 1) throw new Error("Archive must contain exactly one image") + const image = object(manifest[0]) + const configPath = image.Config + if (typeof configPath !== "string" || !/^(?:blobs\/sha256\/[a-f0-9]{64}|[a-f0-9]{64}\.json)$/.test(configPath)) + throw new Error("Invalid image config path") + if (!members.includes(configPath) || !Array.isArray(image.Layers) || image.Layers.length === 0 || + image.Layers.some((layer) => typeof layer !== "string" || !safeRelative(layer) || !members.includes(layer))) + throw new Error("Missing image config or layers") + const configText = await tar(archive, configPath) + const configHash = sha256(Buffer.from(configText)) + if (configHash !== basename(configPath).replace(/\.json$/, "")) throw new Error("Image config checksum mismatch") + const config = object(JSON.parse(configText)) + const platform = index === 0 ? "linux/amd64" : "linux/arm64" + if (`${config.os}/${config.architecture}` !== platform) throw new Error(`Expected ${platform}`) + const labels = object(object(config.config).Labels) + if (labels["org.opencontainers.image.revision"] !== revision) throw new Error("Image source revision mismatch") + return { path, platform, target: index === 0 ? "runtime" : "coolify", configSha256: configHash } +} + +async function listFiles(root: string, prefix = ""): Promise { + const metadata = await lstat(resolve(root, prefix)) + if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("Expected a real directory") + const files: string[] = [] + for (const name of (await readdir(resolve(root, prefix))).sort()) { + const path = prefix ? `${prefix}/${name}` : name + const entry = await lstat(resolve(root, path)) + if (entry.isSymbolicLink()) throw new Error("Symlinks are not permitted") + if (entry.isDirectory()) { + if (!["source", "images", "tools"].some((base) => path === base || path.startsWith(`${base}/`)) || + (path.startsWith("source/") && !isBuildInput(`${path.slice(7)}/placeholder`))) + throw new Error("Unexpected escrow directory") + files.push(...await listFiles(root, path)) + } else { + if (!entry.isFile() || (path !== "escrow.json" && !allowedFile(path))) throw new Error("Unexpected escrow file") + files.push(path) + } + } + return files.sort() +} + +export async function captureEscrow(options: { + root: string + revision: string + amd64Archive: string + arm64Archive: string + destination: string +}): Promise<{ manifestSha256: string; sourceRevision: string }> { + for (const archive of [options.amd64Archive, options.arm64Archive]) { + if (!allowedFile(`source/${basename(archive)}`) || !archive.endsWith(".tar")) + throw new Error("Supply explicit non-hidden .tar image archives") + } + const destination = resolve(options.destination) + await mkdir(destination, { mode: 0o700 }) + const source = await prepareServerContext(options.root, options.revision, resolve(destination, "source")) + await mkdir(resolve(destination, "images"), { mode: 0o700 }) + await mkdir(resolve(destination, "tools"), { mode: 0o700 }) + const files: FileIdentity[] = [] + const images: ImageIdentity[] = [] + for (const [index, archive] of [options.amd64Archive, options.arm64Archive].entries()) { + const path = archivePaths[index]! + files.push({ path, ...await fingerprint(archive, resolve(destination, path)) }) + images.push(await inspectArchive(resolve(destination, path), path, source.commit, index)) + } + // Tool bytes are separately hashed; they are not claimed to belong to the image commit. + for (const path of toolPaths) { + files.push({ path, ...await fingerprint(resolve(import.meta.dir, basename(path)), resolve(destination, path)) }) + } + for (const path of await listFiles(destination)) { + if (!files.some((file) => file.path === path)) { + files.push({ path, ...await fingerprint(resolve(destination, path)) }) + await syncFile(resolve(destination, path)) + } + } + const manifest: EscrowManifest = { + version: 1, + sourceRevision: source.commit, + verification: "archive-bytes-and-image-config-only", + files: files.sort((a, b) => a.path.localeCompare(b.path)), + images, + } + const content = Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`) + const completion = await open(resolve(destination, "escrow.json"), "wx", 0o600) + try { + await completion.writeFile(content) + await completion.sync() + } finally { await completion.close() } + return { manifestSha256: sha256(content), sourceRevision: source.commit } +} + +function parseManifest(content: Buffer): EscrowManifest { + const value = object(JSON.parse(content.toString("utf8"))) + if (value.version !== 1 || !hex(value.sourceRevision, 40) || + value.verification !== "archive-bytes-and-image-config-only" || !Array.isArray(value.files) || !Array.isArray(value.images)) + throw new Error("Invalid escrow manifest") + const files = value.files.map((entry): FileIdentity => { + const file = object(entry) + if (typeof file.path !== "string" || !allowedFile(file.path) || !hex(file.sha256, 64) || + typeof file.bytes !== "number" || !Number.isSafeInteger(file.bytes) || file.bytes < 0) throw new Error("Invalid file identity") + return { path: file.path, bytes: file.bytes, sha256: file.sha256 } + }) + if (new Set(files.map((file) => file.path)).size !== files.length) throw new Error("Duplicate file identity") + const required = [...archivePaths, ...toolPaths, "source/source.json"] + if (required.some((path) => !files.some((file) => file.path === path))) throw new Error("Missing required artifact") + const images = value.images.map((entry, index): ImageIdentity => { + const image = object(entry) + const platform = index === 0 ? "linux/amd64" : "linux/arm64" + const target = index === 0 ? "runtime" : "coolify" + if (image.path !== archivePaths[index] || image.platform !== platform || image.target !== target || !hex(image.configSha256, 64)) + throw new Error("Invalid image identity") + return { path: archivePaths[index]!, platform, target, configSha256: image.configSha256 } + }) + if (images.length !== 2) throw new Error("Both image architectures are required") + return { version: 1, sourceRevision: value.sourceRevision, verification: "archive-bytes-and-image-config-only", files, images } +} + +export async function verifyEscrow(directory: string, expectedManifestSha256: string): Promise { + if (!hex(expectedManifestSha256, 64)) throw new Error("Supply an independently retained manifest SHA-256") + const root = resolve(directory) + const inventory = await listFiles(root) + const content = await readSmall(resolve(root, "escrow.json")) + if (sha256(content) !== expectedManifestSha256) throw new Error("Manifest checksum mismatch") + const manifest = parseManifest(content) + const expected = [...manifest.files.map((file) => file.path), "escrow.json"].sort() + if (JSON.stringify(inventory) !== JSON.stringify(expected)) throw new Error("Escrow inventory mismatch") + for (const file of manifest.files) { + const actual = await fingerprint(resolve(root, file.path)) + if (actual.bytes !== file.bytes || actual.sha256 !== file.sha256) throw new Error(`Artifact checksum mismatch: ${file.path}`) + } + for (const [index, image] of manifest.images.entries()) { + const actual = await inspectArchive(resolve(root, image.path), image.path, manifest.sourceRevision, index) + if (actual.configSha256 !== image.configSha256) throw new Error("Image identity mismatch") + } + const source = object(JSON.parse((await readSmall(resolve(root, "source/source.json"))).toString("utf8"))) + if (source.commit !== manifest.sourceRevision) throw new Error("Source identity mismatch") + return manifest +} + +if (import.meta.main) { + const [command, ...args] = process.argv.slice(2) + if (command === "capture" && args.length === 4) { + const result = await captureEscrow({ + root: resolve(import.meta.dir, "../.."), revision: args[0]!, amd64Archive: resolve(args[1]!), + arm64Archive: resolve(args[2]!), destination: resolve(args[3]!), + }) + console.info(JSON.stringify(result)) + } else if (command === "verify" && args.length === 2) { + const manifest = await verifyEscrow(args[0]!, args[1]!) + console.info(JSON.stringify({ sourceRevision: manifest.sourceRevision, files: manifest.files.length, verification: manifest.verification })) + } else { + throw new Error("Usage: bun --no-env-file server-release-escrow.ts capture | verify ") + } +} diff --git a/server/docs/server-release-escrow.md b/server/docs/server-release-escrow.md new file mode 100644 index 000000000..25f1887b9 --- /dev/null +++ b/server/docs/server-release-escrow.md @@ -0,0 +1,95 @@ +# Retained server release escrow + +`scripts/docker/server-release-escrow.ts` captures existing image archives and +the exact committed server build context into a new local directory. It needs +Bun, Git for capture, and the system `tar`; verification needs Bun and `tar` +only. It does not build, run Docker, fetch packages, access credentials, upload, +migrate, deploy, or change running services. No publication workflow calls it. + +This is the first preparation step toward deployment independence. It does not +establish that any two failed build systems can be bypassed. + +## Capture an existing release + +Supply separately produced, **uncompressed single-image Docker-save archives**: +the Linux AMD64 runtime and Linux ARM64 Coolify images of the same committed +source. Creating these archives is a separate operation; this tool never starts +Docker. Use archives from the already qualified release, rather than rebuilding +an image during recovery. + +```sh +bun --no-env-file scripts/docker/server-release-escrow.ts capture \ + COMMIT /retained/runtime-amd64.tar /retained/coolify-arm64.tar \ + /retained/new-release-directory +``` + +Capture refuses an existing destination and symlinked image files. It exports +only committed build inputs through the existing source exporter, excluding +environment files, credential candidates and dirty/untracked source. Image +archives must carry the expected source revision label and platform. Each image +config's bytes must match its Docker config filename checksum. The two target +names describe their intended use; the archive cannot prove which Dockerfile +target was built. + +The resulting `escrow.json` records source revision, every retained file's byte +count/SHA-256, image config identity and platform. `source/source.json` preserves +the commit/blob inventory. A copy of the two recovery scripts is included and +hashed separately; these tool bytes are not claimed to belong to the image's +source commit. No runtime configuration or credentials are captured. + +The final JSON output includes `manifestSha256`. **Retain that checksum outside +the escrow**, in a trusted release record. A checksum stored only alongside an +altered bundle cannot authenticate it. The completion manifest is written last; +failed captures remain available for inspection and cannot be verified as a +complete escrow. Do not reuse or merge a partial destination. + +## Verify after transfer or provider loss + +Copy the complete directory to storage independent of GitHub, Fly and Hetzner +(for example, the existing private recovery store). This tool does not perform +that upload. Retain the current and a schema-compatible rollback release, and +record independent destination/readback evidence before calling either stored. + +```sh +bun --no-env-file scripts/docker/server-release-escrow.ts verify \ + /recovered/release-directory INDEPENDENT_MANIFEST_SHA256 +``` + +Use a trusted copy of the verifier first. The bundled copy can verify a moved +directory without the original checkout, but executing an unverified bundled +script cannot authenticate that script. Verification rejects changed archives +or source, changed manifests, missing/extra files, unsafe paths and symlinks. +It does not extract archive members onto disk or run image content. + +The success label is deliberately `archive-bytes-and-image-config-only`. +Archive checksums establish retained byte identity; config identity establishes +the recorded source label and platform. They do **not** establish registry +manifest digest equivalence, ordered layer content validity, Docker import, +startup, tests, migration compatibility, safe rollback, key recovery, or serving +capacity. Keep the original qualification receipt and registry digest as +separate release provenance. Docker import and runtime checks remain required +on the intended host before deployment. + +## Remaining outage gates + +- Independently accessible deployment credentials, runtime keys, configuration, + SSH/controller access and routing controls. These do not belong in this bundle. +- An independent image delivery path for Hetzner when Fly registry is down. + Coolify Docker Image deployments pull the saved reference; importing a local + image alone is not proof that the installed controller can redeploy it offline. +- A portable release runner that preserves target-specific preflight, explicit + migration execution, graceful replacement and authenticated verification while + GitHub Actions is unavailable. +- A complete current-lockfile dependency/toolchain kit for **new-source builds**: + the patched GitHub `node-forge` dependency, Bun base, package payloads, native + dependencies, Coolify tools and isolated test dependencies. A source archive + alone cannot cold-build through a full GitHub outage. New dependencies require + refreshing that kit. +- Native build/qualification capacity on the surviving executor. Retained-image + redeployment needs no builder and is a distinct acceptance case. +- Whole Fly plus whole Hetzner loss removes both existing origin providers. + Independent artifact preparation remains possible; serving requires a third + compute target. + +No live recovery, independent upload/readback, production rollout, or pairwise +outage drill is implied by this tooling or its synthetic tests. From 0a16d9542f0c5be01ad35b976d5eb16cfec7d8dc Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Fri, 16 Oct 2026 19:47:03 +0330 Subject: [PATCH 12/14] docs: clarify direct origin bypass without cloudflare load balancing --- server/docs/hetzner-standby.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/docs/hetzner-standby.md b/server/docs/hetzner-standby.md index 2cbdaf72e..4bb341f9e 100644 --- a/server/docs/hetzner-standby.md +++ b/server/docs/hetzner-standby.md @@ -278,7 +278,19 @@ address families so an old AAAA record cannot route clients to an unavailable origin. Verify DNS answers and a recovered authenticated client operation. Restore the captured records only after the normal path is healthy. -If an optional load balancer exists, use the following LB-specific procedure. +If an optional load balancer exists and **both the Cloudflare proxy and load +balancing must be bypassed**, capture its configuration and the ordinary DNS +records, disable the load balancer, then use ordinary DNS-only A/AAAA records +(`proxied=false`) pointing to the verified direct origin. Prepare the fallback +addresses first and require a valid public-trust certificate for +`api.inline.chat` at that origin. Verify both address families, DNS propagation +and an authenticated operation before considering the bypass complete. This +requires the Cloudflare management API/dashboard and DNS updates to remain +available; healthy authoritative DNS alone does not provide that control. + +A DNS-only **load balancer** still uses Cloudflare's health and steering logic. +When only the proxy must be bypassed and the LB remains available, the following +single-pool DNS-only LB procedure is an alternative. The load balancer takes precedence over an A/AAAA record with the same name. Editing only the old A record or its orange-cloud state does not bypass the LB. From 0d2c37eaa4036a4a15fa5c5294098ccb575f3b01 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Fri, 16 Oct 2026 20:01:05 +0330 Subject: [PATCH 13/14] server: honor explicit health monitor opt-out for secondary workers --- .../modules/databaseHealthMonitor.test.ts | 11 ++++++++++- .../modules/monitoring/databaseHealthMonitor.ts | 14 ++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/server/src/__tests__/modules/databaseHealthMonitor.test.ts b/server/src/__tests__/modules/databaseHealthMonitor.test.ts index 8bd8b15e1..db0c25498 100644 --- a/server/src/__tests__/modules/databaseHealthMonitor.test.ts +++ b/server/src/__tests__/modules/databaseHealthMonitor.test.ts @@ -1,4 +1,4 @@ -import { DatabaseHealthMonitor } from "@in/server/modules/monitoring/databaseHealthMonitor" +import { DatabaseHealthMonitor, isDatabaseHealthMonitorEnabled } from "@in/server/modules/monitoring/databaseHealthMonitor" import type { InlineProtocolClockHealth } from "@in/server/modules/inlineProtocol/clockHealth" import { describe, expect, it, spyOn } from "bun:test" @@ -36,6 +36,15 @@ const downSample = (): HealthSample => ({ }) describe("DatabaseHealthMonitor", () => { + it("lets a secondary production worker suppress alerts while retaining the primary default", () => { + expect(isDatabaseHealthMonitorEnabled(true, undefined)).toBe(true) + expect(isDatabaseHealthMonitorEnabled(true, "1")).toBe(true) + expect(isDatabaseHealthMonitorEnabled(true, "0")).toBe(false) + expect(isDatabaseHealthMonitorEnabled(false, undefined)).toBe(false) + expect(isDatabaseHealthMonitorEnabled(false, "1")).toBe(true) + expect(isDatabaseHealthMonitorEnabled(false, "0")).toBe(false) + }) + it("alerts only after reaching failure threshold", async () => { const alerts: string[] = [] const samples = [downSample(), downSample()] diff --git a/server/src/modules/monitoring/databaseHealthMonitor.ts b/server/src/modules/monitoring/databaseHealthMonitor.ts index 14af49281..a32b9eb25 100644 --- a/server/src/modules/monitoring/databaseHealthMonitor.ts +++ b/server/src/modules/monitoring/databaseHealthMonitor.ts @@ -276,16 +276,14 @@ export class DatabaseHealthMonitor { let monitorInstance: DatabaseHealthMonitor | null = null -const shouldStartDatabaseMonitor = (): boolean => { - if (NODE_ENV === "production") { - return true - } - - return process.env["ENABLE_DATABASE_HEALTH_MONITOR"] === "1" -} +/** A secondary worker can opt out of process-local alerts without disabling queue workers. */ +export const isDatabaseHealthMonitorEnabled = ( + isProduction: boolean, + setting: string | undefined, +): boolean => setting !== "0" && (isProduction || setting === "1") export const startDatabaseHealthMonitor = (): DatabaseHealthMonitor | null => { - if (!shouldStartDatabaseMonitor()) { + if (!isDatabaseHealthMonitorEnabled(NODE_ENV === "production", process.env["ENABLE_DATABASE_HEALTH_MONITOR"])) { return null } From f497aacb772408f8fea52980727452f92c294929 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Fri, 16 Oct 2026 23:31:44 +0330 Subject: [PATCH 14/14] docs: align warm fallback release operations --- server/docs/fly-deployment.md | 44 ++++++++++++++++++++++++++++------ server/docs/hetzner-standby.md | 29 +++++++++++++++++----- server/fly.toml | 3 ++- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/server/docs/fly-deployment.md b/server/docs/fly-deployment.md index ceee75e60..c0d8f5b22 100644 --- a/server/docs/fly-deployment.md +++ b/server/docs/fly-deployment.md @@ -77,6 +77,30 @@ ingress mode, direct-IP source and query-pool cap match the qualified profile. Keep credential values out of receipts and logs; a list of secret names alone does not establish these effective settings. +### Release contract with a warm fallback + +Since October 16, 2026, `api.inline.chat` uses Cloudflare load balancing with +Fly preferred and a running Nuremberg fallback. The Fly workflow publishes both +architecture-specific images but deploys and verifies only Fly. It does not +refresh Nuremberg or prove that automatic fallback remains usable. + +Before releasing the next selected `main` SHA to Fly, qualify that same SHA's +ARM64 image on Nuremberg against the intended schema and runtime configuration. +Record the selected SHA, both immutable image digests, each origin's effective +image/configuration, the compatible rollback release, and the unchanged +Cloudflare pool identities/order. Apply DDL once through the existing migration +gate only after old/new/rollback schema compatibility is established. Replace +and qualify Nuremberg before replacing Fly, keeping another ready origin during +each replacement. Verify direct HTTP/V2/V3 and cross-node recovery, then verify +canonical routing and both effective release identities after the Fly rollout. + +The current workflow does not enforce this fleet gate. Until a reviewed guard +is installed, it is an explicit operator prerequisite, and a green Fly-only +workflow is not completion of the fleet release. Deliberate mixed-version +overlap needs a recorded compatibility decision and an owner/deadline for +finishing it; it must not leave an indefinitely stale standby. See +[the standby release procedure](hetzner-standby.md#rolling-upgrades-and-removing-a-node). + ### Prepare an image for rehearsal or initial bootstrap For an exact-image rehearsal, manually dispatch from the selected branch in @@ -229,10 +253,13 @@ in the protected recovery capture. An image-only rollback is insufficient: binaries that predate optional ingress reject `cloudflare-optional`, so restore their strict `cloudflare` ingress configuration and preserve its origin-attribution secret alongside the old image. Those binaries also lack the query-pool cap and can use -10 query plus one health connection each. Recalculate the whole fleet's budget -and keep Hetzner stopped during the first rollback overlap; do not count the new -cap as a limit on an old binary. Verify the resulting image, effective settings, -schema compatibility and authenticated serving path before returning traffic. +10 query plus one health connection each. Such a legacy rollback needs a +separate ingress and whole-fleet connection-budget plan; do not count the new +cap as a limit on an old binary. If that plan requires withdrawing an origin, +first verify a ready alternative and its routing. This is not an instruction +to stop the current warm Nuremberg fallback during an ordinary compatible +rollback. Verify the selected image, effective settings, schema compatibility +and authenticated serving path before returning traffic. Preserve public `0150_space-profiles` and forward `0151_insights-query-indexes`. The current migrator wraps pending DDL in one transaction: locks from 0150 can @@ -243,10 +270,13 @@ rewrite historical migrations or place concurrent indexes in this transaction. Coolify fallback means deploying a compatible API image against the **current PlanetScale database**, with the same encryption keys, object storage and ingress -contract. Keep the fallback stopped until the manual failover. A hostname/IP -change does not stop its workers. Restoring a database backup is a separate +contract. The current Nuremberg fallback stays running so Cloudflare can route +new connections to it automatically when the preferred Fly pool is unhealthy. +A stopped fallback supplies only manual recovery capacity. Existing WebSockets +need reconnect/catch-up; a hostname or pool change neither moves their sockets +nor stops their workers. Restoring a database backup is a separate disaster-recovery action, not application rollback. There is no automatic schema -rollback or automatic cross-host traffic switch. +rollback. ## Initial app setup and regional reliability diff --git a/server/docs/hetzner-standby.md b/server/docs/hetzner-standby.md index 4bb341f9e..9a39c6351 100644 --- a/server/docs/hetzner-standby.md +++ b/server/docs/hetzner-standby.md @@ -1,7 +1,10 @@ # Hetzner standby and Cloudflare proxy bypass -This is a prepared deployment option, not a statement that the origins or routing -are active or qualified. Keep Cloudflare authoritative DNS. The preferred order is +As of October 16, 2026, `api.inline.chat` has a running Nuremberg fallback behind +Cloudflare load balancing, with Fly preferred. Verify current provider inventory +and dated qualification receipts before a change. Finland remains a prepared +third-origin option; this document's three-origin examples do not establish that +it is active or qualified. Keep Cloudflare authoritative DNS. The preferred order is **Fly → Nuremberg → Finland**: Fly serves normally, Nuremberg is the first fallback, and the existing Finland host is the second fallback. A second Fly Machine can later join the same app without another Cloudflare pool. @@ -185,7 +188,8 @@ and [Full](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes/ ### Rolling upgrades and removing a node Qualify the fallback candidates before changing the preferred Fly origin. For -an existing three-origin fleet, update Finland, then Nuremberg, then Fly, one +the current two-origin fleet, update Nuremberg, then Fly. If Finland is later +qualified and enabled, update Finland, then Nuremberg, then Fly, one host at a time. On each host start the new immutable container on a separate private backend, qualify it, switch only that API's proxy upstream, then SIGTERM the old container and wait for its real exit/drain. Keep the other origins ready. @@ -193,9 +197,22 @@ Source changes, image publication and local examples alone change no routing. For Fly, retain the existing exact-fleet preflight, migration and blue-green workflow; `--ha=false` preserves the selected public Machine count. It does not -update either Hetzner origin. Record all three effective release identities -before calling a fleet release complete. Mixed releases are acceptable only -for deliberately qualified additive overlap, not indefinitely stale fallbacks. +update either Hetzner origin. Before the next selected `main` release reaches +Fly, require Nuremberg to pass direct authenticated HTTP/V2/V3, effective +image/configuration and schema checks for that SHA's qualified ARM64 image. +Record both architecture-specific digests, the compatible rollback release, +and all enabled origins' effective identities before calling the fleet release +complete. Keep the existing Cloudflare LB/pool identities and priority order; +an application release does not authorize deleting or replacing fallback +routing. Recheck routing and origin health after the rollout. Mixed releases +are acceptable only for deliberately qualified additive overlap with a named +owner and completion deadline, not indefinitely stale fallbacks. + +The checked-in Fly workflow currently has no Nuremberg deployment/readback or +freshness gate. Until a reviewed release guard is added, the operator must +complete and record the standby prerequisite before dispatching the Fly +release. Automatic Git deployment remains disabled on the standby; enabling +it would bypass exact-image qualification rather than solve release coordination. Before a planned node stop, verify a ready alternative, withdraw that node from new ingress (its pool/endpoint or scoped reverse-proxy routing), then send diff --git a/server/fly.toml b/server/fly.toml index fdb6ea9f9..9a4849ab5 100644 --- a/server/fly.toml +++ b/server/fly.toml @@ -19,7 +19,8 @@ kill_timeout = "45s" [env] NODE_ENV = "production" PORT = "8000" - # The only traffic-serving role. It owns the shared workers and schedulers. + # This Fly role participates in shared worker claims alongside qualified + # warm fallback origins; routing priority does not fence background workers. INLINE_PROCESS_ROLE = "all" # Deployment overlap and the warm standby share PostgreSQL. Redis supplies # optional wake-up hints; durable catch-up must remain enabled without it.