diff --git a/CLAUDE.md b/CLAUDE.md index 1115db95..5fd64ab8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -159,6 +159,12 @@ changing first. The parts, and the one place each rule lives: - **Read through `throughCache(cached, direct)`.** A production build replaces the message of anything thrown inside a cached function, so when the cached call throws the direct one runs and reports the real error. It also covers a cache that cannot be read. + - **An administrator clears it, and changes federation settings, at `/admin`** (the users in + `ADMIN_EMAILS`; account menu → Admin). `lib/settings.ts`: an `app_setting` row overrides the + environment variable of the same name for every process within seconds; federation code reads + those settings through `setting(name)`, never `process.env`. `cacheVersion()` (data version plus + the settings' fingerprint, which includes the last clear) is every cached entry's key argument — + cohort locks keep `dataVersion()` alone. Database addresses and credentials are never editable. - **`RESULT_CACHE=off` turns it off.** `throughCache` then runs the direct call on every request, so a server answers everything cold — for measuring what a first reader waits for. Without a cache handler configured (anywhere but Vercel), the cache is the Node process's own diff --git a/app/admin/layout.tsx b/app/admin/layout.tsx new file mode 100644 index 00000000..a3d8c9a0 --- /dev/null +++ b/app/admin/layout.tsx @@ -0,0 +1,5 @@ +export const metadata = { title: "Admin — cbio-lab" }; + +export default function AdminLayout({ children }: { children: React.ReactNode }) { + return children; +} diff --git a/app/admin/page.tsx b/app/admin/page.tsx new file mode 100644 index 00000000..3181096b --- /dev/null +++ b/app/admin/page.tsx @@ -0,0 +1,193 @@ +"use client"; +// /admin — for the users in ADMIN_EMAILS (lib/auth.ts), reached from the account menu: clear the +// result cache, and see and change the federation settings for the whole deployment +// (lib/settings.ts, app/api/admin/route.ts). A change applies to every server process within a few +// seconds and gives every cached result a new key, so nothing computed under the old settings is +// served under the new ones. +import { useCallback, useEffect, useState } from "react"; +import { Loader2, RotateCcw, Trash2 } from "lucide-react"; +import { SiteHeader } from "../_ui/site-header.tsx"; +import { Button } from "../_ui/button"; +import { Input } from "../_ui/input"; +import { ErrorAlert } from "../_ui/error-alert.tsx"; +import { TB } from "../_ui/toolbar.ts"; +import type { SettingGroup, SettingRow, SettingsReport } from "../../lib/settings.ts"; + +const GROUPS: { key: SettingGroup; title: string; about: string }[] = [ + { key: "databases", title: "Databases", about: "The warehouses this instance queries directly. Their addresses and credentials are read from the environment and cannot be changed here." }, + { key: "peers", title: "Peers over MCP", about: "Studies this instance reaches through another cbio-lab instance's /api/mcp, which answers with aggregates only." }, + { key: "patient-level", title: "Patient-level federation", about: "The opt-in exception to aggregate-only federation (docs/DATA-BOUNDARY.md): data crosses only where the peer discloses a kind and the coordinator accepts it." }, + { key: "access", title: "Access", about: "Which studies are routed at all, and which are treated as aggregate-only." }, +]; + +async function call(init?: RequestInit): Promise { + const r = await fetch("/api/admin", init && { ...init, headers: { "content-type": "application/json" } }); + const j = await r.json().catch(() => ({ error: "the server answered " + r.status })); + if (!r.ok || j.error) throw new Error(j.error || "the server answered " + r.status); + return j; +} +const post = (body: object) => call({ method: "POST", body: JSON.stringify(body) }); + +const when = (iso: string) => new Date(iso).toLocaleString(); + +export default function AdminPage() { + const [report, setReport] = useState(null); + const [err, setErr] = useState(null); + const [busy, setBusy] = useState(null); + + const load = useCallback(async () => { + try { setReport(await call()); setErr(null); } + catch (e) { setErr(String((e as Error).message || e)); } + }, []); + useEffect(() => { void load(); }, [load]); + + /** run one change; the answer is the report after it */ + const act = async (key: string, body: object): Promise => { + setBusy(key); setErr(null); + try { setReport(await post(body)); return true; } + catch (e) { setErr(String((e as Error).message || e)); return false; } + finally { setBusy(null); } + }; + + const clearCache = () => { + if (confirm("Clear the result cache for the whole deployment? Every analysis, Explore page and study list is computed again on its next request.")) + void act("cache", { action: "clearCache" }); + }; + + return <> + +
+
+

Admin

+

+ Settings here apply to the whole deployment, for everyone, within a few seconds of saving. A changed setting + gives every cached result a new key, so nothing computed under the old settings is served under the new ones. +

+
+ {err && {err}} + {!report && !err &&

loading…

} + {report && <> +
+

Result cache

+

+ Analyses, oncoprints, Explore, study catalogs and the study list are cached for a day and shared by every reader + (Vercel's Runtime Cache in production, each server's own memory elsewhere). A data import, a new OncoKB + release and a deploy already clear it; clear it by hand after anything else that changes answers. +

+

+ {report.cacheCleared ? <>Last cleared {when(report.cacheCleared.at)}{report.cacheCleared.by && <> by {report.cacheCleared.by}}. : "Not cleared from here yet."} +

+ +
+ +
+

Federation settings

+ {report.unavailable + ?

Read-only: {report.unavailable}. The values below are the environment's.

+ :

A value saved here overrides the environment variable of the same name until it is reverted. + Leave a field empty and save to override the variable with nothing.

} + {GROUPS.map((gr) => ( +
+

{gr.title}

+

{gr.about}

+ {gr.key === "databases" && ( + + + + {report.databases.map((d) => ( + + + + + + ))} + +
connectionaddressuser · database · password
{d.id} {d.label} · · {d.passwordSet ? "password set (hidden)" : "no password"}
+ )} + + + + {report.rows.filter((r) => r.group === gr.key).map((r) => ( + { + if (r.group === "patient-level" && !confirm("Change " + r.name + " for the whole deployment? It decides which patient-level data crosses between instances.")) + return Promise.resolve(false); + return act(r.name, { action: "set", name: r.name, value }); + }} + onReset={() => act(r.name, { action: "reset", name: r.name })} /> + ))} + +
settingenvironmentin force
+
+ ))} +
+ + {report.log.length > 0 && ( +
+

Recent changes

+ + + + {report.log.map((l, i) => ( + + + + + ))} + +
whenwhosettingfromto
{when(l.at)}{l.by ?? "—"}{l.name}{l.new == null ? reverted to environment : }
+
+ )} + } +
+ ; +} + +function Value({ v }: { v: string | null | undefined }) { + if (v == null) return not set; + if (v === "") return empty; + return {v}; +} + +/** One setting: what the environment says, what is in force, and (when editable) a field that owns its draft. */ +function SettingLine({ row, busy, onSave, onReset }: { + row: SettingRow; busy: boolean; onSave: (value: string) => Promise; onReset: () => void; +}) { + const current = row.override ? row.override.value : row.env ?? ""; + const [draft, setDraft] = useState(current); + const [editing, setEditing] = useState(false); + useEffect(() => { if (!editing) setDraft(current); }, [current, editing]); + const env = row.secret ? {row.envSet ? "set (hidden)" : "not set"} : ; + + return ( + + + {row.name} +
{row.help}
+ + {env} + + {!row.editable && (row.override ? : {row.secret ? "—" : "the environment's"})} + {row.editable && !editing &&
+ {row.override ? : the environment's} + + {row.override && } +
} + {row.editable && editing &&
{ e.preventDefault(); void onSave(draft).then((ok) => { if (ok) setEditing(false); }); }}> + setDraft(e.target.value)} aria-label={row.name} autoFocus + className="h-8 min-w-60 flex-1 font-mono text-sm" /> + + +
} + {row.override &&
overridden {when(row.override.at)}
} + + + ); +} diff --git a/app/api/[transport]/route.ts b/app/api/[transport]/route.ts index 49a0d28c..3dc39454 100644 --- a/app/api/[transport]/route.ts +++ b/app/api/[transport]/route.ts @@ -4,6 +4,7 @@ import { registerTools } from "../../../mcp/tools.ts"; import { cachedAnalysis, cachedContract } from "../../../lib/next-cache.ts"; import { asLoopbackPeer, isLoopbackRequest } from "../../../lib/loopback-peer.ts"; import { logInbound } from "../../../lib/federation-log.ts"; +import { refreshSettings } from "../../../lib/settings.ts"; const handler = createMcpHandler( (server) => registerTools(server, { runAnalysis: (doc) => cachedAnalysis(doc), buildContract: cachedContract }), @@ -13,6 +14,8 @@ const handler = createMcpHandler( // a call this app made to itself is answered as its simulated peer (lib/loopback-peer.ts) const serve = async (req: Request) => { + // which tools exist and who counts as the simulated peer are settings an administrator may change + await refreshSettings(); const asPeer = isLoopbackRequest(req); const call = req.method === "POST" ? await toolCall(req) : null; const t0 = Date.now(); diff --git a/app/api/admin/route.ts b/app/api/admin/route.ts new file mode 100644 index 00000000..f29a2f13 --- /dev/null +++ b/app/api/admin/route.ts @@ -0,0 +1,39 @@ +import { connection } from "next/server"; +import { revalidateTag } from "next/cache"; +import { requireAdmin, AccessError } from "../../../lib/auth.ts"; +import { markResultCacheCleared, setSetting, settingsReport } from "../../../lib/settings.ts"; +import { RESULT_TAG } from "../../../lib/next-cache.ts"; + +// The administrator's page (/admin), for the users in ADMIN_EMAILS (lib/auth.ts). +// GET — every federation setting: its environment value (secrets only as set or not), the +// override in force, and the recent changes (lib/settings.ts) +// POST { action: "set", name, value } — override a setting for the whole deployment +// POST { action: "reset", name } — return it to its environment variable +// POST { action: "clearCache" } — clear the result cache (lib/next-cache.ts) +const fail = (e: unknown) => Response.json({ error: String((e as Error)?.message || e) }, { status: e instanceof AccessError ? e.status : 400 }); + +export async function GET(req: Request) { + await connection(); + try { + requireAdmin(req); + return Response.json(await settingsReport()); + } catch (e) { return fail(e); } +} + +export async function POST(req: Request) { + try { + const u = requireAdmin(req); + const body = await req.json().catch(() => ({})); + if (body.action === "clearCache") { + // a new key for every entry, in every process; and the entries themselves expire where the + // cache can expire them (Vercel's Runtime Cache, this process's memory) + await markResultCacheCleared(u.id); + revalidateTag(RESULT_TAG, { expire: 0 }); + } else if (body.action === "set" || body.action === "reset") { + if (typeof body.name !== "string") throw new Error("name the setting"); + if (body.action === "set" && typeof body.value !== "string") throw new Error("value must be text"); + await setSetting(body.name, body.action === "set" ? body.value : null, u.id); + } else throw new Error("action must be set, reset or clearCache"); + return Response.json(await settingsReport()); + } catch (e) { return fail(e); } +} diff --git a/app/api/auth/route.ts b/app/api/auth/route.ts index c8ed160d..8fe6bc9a 100644 --- a/app/api/auth/route.ts +++ b/app/api/auth/route.ts @@ -1,8 +1,10 @@ import { connection } from "next/server"; -import { loginEnabled, loginOptions, userFor } from "../../../lib/auth.ts"; +import { isAdmin, loginEnabled, loginOptions, userFor } from "../../../lib/auth.ts"; -// Whether this deployment signs people in, how, and who this browser is signed in as (lib/auth.ts). +// Whether this deployment signs people in, how, who this browser is signed in as and whether they +// administer it (lib/auth.ts). export async function GET(req: Request) { await connection(); - return Response.json({ enabled: loginEnabled(), options: loginOptions(), user: userFor(req) }); + const user = userFor(req); + return Response.json({ enabled: loginEnabled(), options: loginOptions(), user: user && { ...user, admin: isAdmin(user) } }); } diff --git a/app/api/catalog/route.ts b/app/api/catalog/route.ts index 1cfec494..a20cf95b 100644 --- a/app/api/catalog/route.ts +++ b/app/api/catalog/route.ts @@ -2,8 +2,7 @@ import { STUDY, withoutFailedQueries } from "../../../lib/ch"; import { studyRouting, restrictedStudies } from "../../../lib/federation"; import { catalogFor } from "../../../lib/study-catalog.ts"; import { peerForStudy, remoteCatalog } from "../../../lib/federation-remote.ts"; -import { cacheLife } from "next/cache"; -import { pack, throughCache, dataVersion } from "../../../lib/next-cache.ts"; +import { pack, throughCache, cacheVersion, resultEntry } from "../../../lib/next-cache.ts"; export async function GET(req: Request) { try { @@ -12,14 +11,14 @@ export async function GET(req: Request) { // A study PEER_MCP routes to a peer is described by that peer, over MCP: its database is // not this instance's to read. const peer = await peerForStudy(study); - if (peer) return Response.json(await throughCache(async () => peerCatalogEntry(study, peer.url, await dataVersion()), () => remoteCatalog(study, peer.url))); + if (peer) return Response.json(await throughCache(async () => peerCatalogEntry(study, peer.url, await cacheVersion()), () => remoteCatalog(study, peer.url))); // A study lives on whichever instance holds it (AACR GENIE is not on the primary), so ask // the same routing the analysis engine uses rather than assuming the default database. // A study the routing withholds by policy is unreachable here too, not served from PRIMARY. // Checked before the cache: a refusal is an answer to give, not one to keep. if (!(await studyRouting()).has(study)) return Response.json({ error: restrictedStudies.has(study) ? "Study '" + study + "' is restricted and not available here" : "Study '" + study + "' not found" }, { status: 400 }); - return Response.json(await throughCache(async () => catalogEntry(study, await dataVersion()), () => catalogFor(study))); + return Response.json(await throughCache(async () => catalogEntry(study, await cacheVersion()), () => catalogFor(study))); } catch (e) { return Response.json({ error: String((e as Error)?.message || e) }, { status: 500 }); } @@ -27,12 +26,12 @@ export async function GET(req: Request) { async function catalogEntry(study: string, _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await withoutFailedQueries(() => catalogFor(study))); } async function peerCatalogEntry(study: string, url: string, _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await withoutFailedQueries(() => remoteCatalog(study, url))); } diff --git a/app/api/explore/route.ts b/app/api/explore/route.ts index 31133ff3..92fd0a15 100644 --- a/app/api/explore/route.ts +++ b/app/api/explore/route.ts @@ -5,8 +5,7 @@ import { studyRouting, carriesPatientIds, instanceForStudy } from "../../../lib/ import { peerForStudy, remoteFacets } from "../../../lib/federation-remote.ts"; import { mergeExplore } from "../../../lib/explore.ts"; import { PRIMARY, withChContext, ensureChContext, withoutFailedQueries } from "../../../lib/ch.ts"; -import { cacheLife } from "next/cache"; -import { pack, throughCache, dataVersion } from "../../../lib/next-cache.ts"; +import { pack, throughCache, cacheVersion, resultEntry } from "../../../lib/next-cache.ts"; import { studyProfilesMany, resolveStudy } from "../../../lib/analysis.ts"; import { chQuery } from "../../../lib/ch.ts"; import type { ProfileInfo } from "../../../lib/profile-catalog.ts"; @@ -38,7 +37,7 @@ async function handle(req: Request) { // facets over uploaded data or a stored id set are their reader's alone: answered directly, never cached if ((Array.isArray(customData) && customData.length) || (Array.isArray(customRefs) && customRefs.length) || idSetRefsIn(body.draft).length) return Response.json(await withRequestCustomData(customData, () => ensureChContext(() => exploreFor(body)), { refs: customRefs, req, uses: body.draft })); - return Response.json(await throughCache(async () => exploreEntry(body, await dataVersion()), () => ensureChContext(() => exploreFor(body)))); + return Response.json(await throughCache(async () => exploreEntry(body, await cacheVersion()), () => ensureChContext(() => exploreFor(body)))); } catch (e) { // The reader superseded this request: not a failure, and there is nobody left to tell. if ((e as { name?: string })?.name === "AbortError" || req.signal.aborted) @@ -50,7 +49,7 @@ async function handle(req: Request) { async function exploreEntry(body: any, _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await ensureChContext(() => withoutFailedQueries(() => exploreFor(body)))); } diff --git a/app/api/studies/route.ts b/app/api/studies/route.ts index 9fe78099..c28e7a6a 100644 --- a/app/api/studies/route.ts +++ b/app/api/studies/route.ts @@ -3,8 +3,7 @@ import { studiesFor, type StudyRow } from "../../../lib/study-list.ts"; import { peerMcpEntries, peerPatientKinds, remoteStudies } from "../../../lib/federation-remote.ts"; import { instanceForStudy } from "../../../lib/federation.ts"; import type { StudyAccess } from "../../../lib/view-coverage.ts"; -import { cacheLife } from "next/cache"; -import { pack, throughCache, dataVersion } from "../../../lib/next-cache.ts"; +import { pack, throughCache, cacheVersion, resultEntry } from "../../../lib/next-cache.ts"; import { connection } from "next/server"; async function allStudies(): Promise { @@ -30,7 +29,7 @@ async function allStudies(): Promise { } async function studiesEntry(_version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await withoutFailedQueries(() => allStudies())); } /** what the reachable instances returned, when one of them did not answer */ @@ -40,7 +39,7 @@ export async function GET() { await connection(); try { let studies: StudyRow[]; - try { studies = await throughCache(async () => studiesEntry(await dataVersion()), allStudies); } + try { studies = await throughCache(async () => studiesEntry(await cacheVersion()), allStudies); } catch (e) { if (!(e instanceof Partial)) throw e; studies = e.rows; } return Response.json({ studies: await withAccess(studies) }); } catch (e) { diff --git a/app/datasets/page.tsx b/app/datasets/page.tsx index 4a19e514..603910c3 100644 --- a/app/datasets/page.tsx +++ b/app/datasets/page.tsx @@ -1,10 +1,9 @@ // /datasets — every study the configured instances hold, with how many samples carry each kind of // data: cBioPortal's Data Sets page, built from the same warehouse tables every view queries. -import { cacheLife } from "next/cache"; import { connection } from "next/server"; import { allDatasets, type DatasetRow } from "../../lib/datasets.ts"; import { withoutFailedQueries } from "../../lib/ch.ts"; -import { pack, throughCache, dataVersion } from "../../lib/next-cache.ts"; +import { pack, throughCache, cacheVersion, resultEntry } from "../../lib/next-cache.ts"; import { DatasetsTable } from "./DatasetsTable.tsx"; import { PageTools } from "../_agent/PageTools.tsx"; @@ -12,7 +11,7 @@ export const metadata = { title: "Data sets — cbio-lab" }; async function datasetsEntry(_version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); const r = await withoutFailedQueries(() => allDatasets()); // a catalog missing an instance is served, not kept for a day if (r.unreachable.length) throw new Error("an instance did not answer"); @@ -22,7 +21,7 @@ async function datasetsEntry(_version: string): Promise { export default async function DatasetsPage() { await connection(); const { rows, unreachable, overMcp = [] } = await throughCache<{ rows: DatasetRow[]; unreachable: string[]; overMcp?: string[] }>( - async () => datasetsEntry(await dataVersion()), allDatasets); + async () => datasetsEntry(await cacheVersion()), allDatasets); const sum = (k: keyof DatasetRow) => rows.reduce((a, r) => a + (Number(r[k]) || 0), 0); return (
diff --git a/app/globals.css b/app/globals.css index 22f2944c..ef6d9e3a 100644 --- a/app/globals.css +++ b/app/globals.css @@ -1732,3 +1732,21 @@ td.camp-links { flex-direction: column; align-items: flex-start; gap: var(--s1); .dt-table tr.dt-pick.sel td { background: var(--accent-soft); } .cohortsum-skel { display: inline-block; width: 4rem; height: 0.9rem; } } /* @layer legacy */ + +/* --------------------------------------------------------------------------- + /admin — the result cache and the federation settings (app/admin/page.tsx) + --------------------------------------------------------------------------- */ +.admin-page { max-width: 1100px; } +.admin-section { margin-top: var(--s5); } +.admin-section h2 { font-family: "IBM Plex Serif", Georgia, serif; font-size: var(--t-xl); margin: 0 0 var(--s2); } +.admin-section > p { max-width: 75ch; line-height: 1.6; } +.admin-group { margin-top: var(--s4); } +.admin-group h3 { font-size: var(--t-lg); margin: 0 0 var(--s1); } +.admin-table { width: 100%; table-layout: fixed; border-collapse: collapse; margin-top: var(--s2); font-size: var(--t-md); } +.admin-table th { text-align: left; font-weight: 600; color: var(--muted); font-size: var(--t-sm); border-bottom: 1px solid var(--line); padding: var(--s1) var(--s2); } +.admin-table td { vertical-align: top; border-bottom: 1px solid var(--line); padding: var(--s2); } +.admin-table th:first-child { width: 40%; } +.admin-table th:nth-child(2):not(:last-child) { width: 25%; } +.admin-table td { overflow-wrap: anywhere; } +.admin-table tr.overridden td:last-child { background: var(--mark-soft); } +.admin-value { word-break: break-all; } diff --git a/app/lab/custom-data-client.ts b/app/lab/custom-data-client.ts index edb00c93..0bb85542 100644 --- a/app/lab/custom-data-client.ts +++ b/app/lab/custom-data-client.ts @@ -51,7 +51,7 @@ export function referencedDatasets(x: unknown): string[] { /** A document's pin: the name it uses -> a stored dataset at one version. */ export interface UploadBinding { as: string; dataset: string; version: number } -export interface SignedInUser { id: string; email: string; name: string | null } +export interface SignedInUser { id: string; email: string; name: string | null; /** administers this deployment (/admin) */ admin?: boolean } /** How this deployment signs people in (lib/auth.ts): its provider's name, and the development sign-in. */ export interface LoginOptions { oidc: string | null; dev: boolean } export interface ServerState { diff --git a/app/lab/login-button.tsx b/app/lab/login-button.tsx index 3ea671c7..4d8f0f3c 100644 --- a/app/lab/login-button.tsx +++ b/app/lab/login-button.tsx @@ -1,11 +1,11 @@ "use client"; -// Top-right "Sign in", or once signed in the account menu (who, and sign out), like -// the portal's "Logged in as" menu. Signing in lets a +// Top-right "Sign in", or once signed in the account menu (who, Admin for an administrator, and sign +// out), like the portal's "Logged in as" menu. Signing in lets a // reader save and open their own cohorts and analyses and use custom data (lib/auth.ts). A // deployment offers its provider (Google, MSK…), the development sign-in, or both; with only a // provider the button goes straight there. Shown only where the deployment signs people in. import { useCallback, useEffect, useState } from "react"; -import { ChevronDown, LogIn, LogOut, UserRound } from "lucide-react"; +import { ChevronDown, LogIn, LogOut, Settings, UserRound } from "lucide-react"; import { loadCustomData, devSignIn, signInWithProvider, logOut, onLoginRequest, serverState } from "./custom-data-client.ts"; import { useCustomDataState } from "./custom-data-panel.tsx"; import { Button } from "../_ui/button"; @@ -60,6 +60,7 @@ export default function LoginButton() {
{server.user?.email}
+ {server.user?.admin && Admin} { close(); void logOut(); }}>Sign out } diff --git a/db/migrations/0010_app_settings.sql b/db/migrations/0010_app_settings.sql new file mode 100644 index 00000000..27002940 --- /dev/null +++ b/db/migrations/0010_app_settings.sql @@ -0,0 +1,31 @@ +-- Deployment settings an administrator changes at run time from /admin (lib/settings.ts): a row +-- overrides the environment variable of the same name for every server process of the deployment, +-- and deleting it returns to that variable. Only the settings lib/settings.ts lists as editable are +-- ever written; database credentials never are. +-- +-- `result_cache_cleared_at` is not an environment variable: it is when an administrator last +-- cleared the result cache, and it is part of every cached entry's key (lib/next-cache.ts). +-- +-- app_setting_log records every change, who made it and the value before and after. +-- +-- Applied by `npm run db:migrate` (lib/migrate.ts), after 0009. Re-runnable. + +create table if not exists app_setting ( + name text primary key check (name ~ '^[A-Za-z][A-Za-z0-9_]{0,63}$'), + value text not null, + updated_at timestamptz not null default now(), + updated_by uuid references app_user (id) +); + +create table if not exists app_setting_log ( + id bigint generated always as identity primary key, + name text not null, + old_value text, + new_value text, + changed_at timestamptz not null default now(), + changed_by uuid references app_user (id) +); + +-- the log is append-only for the app: it may read it back for /admin, never change it +grant select, insert, update, delete on app_setting to cohort_lab_app; +grant select, insert on app_setting_log to cohort_lab_app; diff --git a/docs/DATA-BOUNDARY.md b/docs/DATA-BOUNDARY.md index 33967581..a1ecaf2b 100644 --- a/docs/DATA-BOUNDARY.md +++ b/docs/DATA-BOUNDARY.md @@ -135,6 +135,12 @@ ignored. | peer (data owner) | `MCP_DISCLOSE_PATIENTS=` | registers `federated_patient_rows` and `federated_patient_timeline`; without it the tools **do not exist**. Every call is checked against the rules — the study and the kind its table belongs to (`PEER_TABLES` in `lib/federation.ts`), before anything is read — and answers only for studies the peer holds itself at patient level (never a study it reaches through a peer of its own, never an aggregate-only one) | | coordinator | `PEER_MCP_PATIENT_LEVEL=` | the `PEER_MCP` peer for a study becomes a `RemotePatientResolver` carrying the kinds accepted for it — a `RemoteTimelineResolver` when they include `timeline` — instead of a `RemoteAggregateResolver` | +Both settings, like the rest of federation's (`PEER_MCP`, `LOOPBACK_PEER`, `STUDY_DENY`, +`AGGREGATE_ONLY_*`…), can be overridden at run time by an administrator at `/admin` +(`lib/settings.ts`): a value the setting's parser refuses is refused there, every change is logged +with who made it, and the result cache is re-keyed so no answer computed under the old rules is +served under the new ones. + Data crosses only where both sides' rules allow the same study and kind. A coordinator that accepts a kind the peer does not disclose gets a refusal naming the peer's setting; a peer that discloses a kind the coordinator does not accept is never asked, and the view is not drawn diff --git a/instrumentation.ts b/instrumentation.ts new file mode 100644 index 00000000..d3ca0c50 --- /dev/null +++ b/instrumentation.ts @@ -0,0 +1,7 @@ +// Runs once when the Next server starts: administrators' setting overrides (lib/settings.ts) apply +// to this server. The CLI, the stdio MCP server and the tests never load this file, so they read the +// environment alone. +export async function register() { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + (await import("./lib/settings.ts")).enableSettingOverrides(); +} diff --git a/lib/auth.ts b/lib/auth.ts index 709b7ab2..8ef890ba 100644 --- a/lib/auth.ts +++ b/lib/auth.ts @@ -10,6 +10,10 @@ // Either way the result is a session: an httpOnly cookie carrying the user's id, email and name, // signed with AUTH_SESSION_SECRET (HMAC-SHA256). Requests are decided from the cookie alone, so a // disabled user keeps their session until it expires (SESSION_DAYS); signing in again is refused. +// +// Administrators (/admin: the result cache and the federation settings, lib/settings.ts) are the +// signed-in users whose email is in ADMIN_EMAILS. With the development sign-in anyone can claim an +// address, so there it is a convenience, not a guard; it is refused on Vercel production. import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; export const SESSION_COOKIE = "cl_session"; @@ -101,3 +105,16 @@ export function requirePrincipal(req: Request, what = "use saved cohorts, analys if (!p) throw new AccessError("sign in to " + what, 403); return p; } + +const adminEmails = (): string[] => (process.env.ADMIN_EMAILS || "").split(/[,\s]+/).map((e) => e.trim().toLowerCase()).filter(Boolean); + +export const isAdmin = (u: SessionUser | null): boolean => !!u?.email && adminEmails().includes(u.email.toLowerCase()); + +/** The signed-in administrator behind a request; refused for anyone else. */ +export function requireAdmin(req: Request): SessionUser { + if (!loginEnabled()) throw new AccessError("this deployment has no sign-in, so it has no administrators", 404); + const u = userFor(req); + if (!u) throw new AccessError("sign in as an administrator to see this page", 403); + if (!isAdmin(u)) throw new AccessError(u.email + " is not an administrator of this deployment (ADMIN_EMAILS)", 403); + return u; +} diff --git a/lib/ch.ts b/lib/ch.ts index 86aa3582..f334f715 100644 --- a/lib/ch.ts +++ b/lib/ch.ts @@ -15,6 +15,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import type { ChResult, ChRow } from "./types.ts"; import { customExternalsFor } from "./custom-data.ts"; import { idSetExternalsFor } from "./id-sets.ts"; +import { setting } from "./settings.ts"; export interface ChConn { id: string; @@ -38,7 +39,7 @@ export const PRIMARY: ChConn = { user: process.env.CLICKHOUSE_USER || "", password: process.env.CLICKHOUSE_PASSWORD || "", db: DB, - label: process.env.CLICKHOUSE_LABEL || "primary" + get label() { return setting("CLICKHOUSE_LABEL") || "primary"; } }; /** Secondary instance, present only when CLICKHOUSE2_* is configured. */ @@ -50,7 +51,7 @@ export function secondary(): ChConn | null { user: process.env.CLICKHOUSE2_USER || "", password: process.env.CLICKHOUSE2_PASSWORD || "", db: clean(process.env.CLICKHOUSE2_DB), - label: process.env.CLICKHOUSE2_LABEL || "secondary" + label: setting("CLICKHOUSE2_LABEL") || "secondary" }; } @@ -60,6 +61,15 @@ export function connections(): ChConn[] { return b ? [PRIMARY, b] : [PRIMARY]; } +/** The configured connections as /admin shows them: never a password, nor credentials in a URL. */ +export function describeConnections(): { id: string; label: string; url: string; user: string; db: string; passwordSet: boolean }[] { + return connections().map((c) => { + let url: string; + try { const u = new URL(c.url); u.username = ""; u.password = ""; url = u.toString(); } catch { url = c.url ? "(not a URL)" : ""; } + return { id: c.id, label: c.label, url, user: c.user, db: c.db, passwordSet: !!c.password }; + }); +} + export function connById(id: string): ChConn | null { return connections().find((c) => c.id === id) || null; } diff --git a/lib/datasets.ts b/lib/datasets.ts index c4e20d90..05774f21 100644 --- a/lib/datasets.ts +++ b/lib/datasets.ts @@ -3,6 +3,7 @@ // queries. Counts only; restricted studies are left out exactly as the router leaves them out. import { chQuery, directConnections, loopbackOnlyConn, type ChConn } from "./ch.ts"; import { isRestrictedStudy } from "./federation.ts"; +import { setting } from "./settings.ts"; export interface DatasetRow { id: string; name: string; cancerType: string; @@ -87,7 +88,7 @@ export async function datasetsFor(conn: ChConn): Promise { export async function allDatasets(): Promise<{ rows: DatasetRow[]; unreachable: string[]; overMcp: string[] }> { const unreachable: string[] = []; // the simulated peer's database is read only over MCP, which has no per-sample catalog: named, not listed - const overMcp = [loopbackOnlyConn()].filter((c) => c != null).map((c) => process.env.LOOPBACK_PEER_LABEL || c.label); + const overMcp = [loopbackOnlyConn()].filter((c) => c != null).map((c) => setting("LOOPBACK_PEER_LABEL") || c.label); const per = await Promise.all(directConnections().map((c) => datasetsFor(c).catch(() => { unreachable.push(c.label); return [] as DatasetRow[]; }))); const seen = new Set(), rows: DatasetRow[] = []; for (const list of per) for (const r of list) if (!seen.has(r.id)) { seen.add(r.id); rows.push(r); } diff --git a/lib/disclosure.ts b/lib/disclosure.ts index 2ec8a565..1f0dc098 100644 --- a/lib/disclosure.ts +++ b/lib/disclosure.ts @@ -9,6 +9,8 @@ // `genie_public:*`. A bare study id allows the timeline alone. Rules are separated by commas or // whitespace. An unknown kind is refused by name, never ignored: a typo must not read as "nothing". +import { setting } from "./settings.ts"; + /** The kinds of patient-level data a rule can allow — each one view's needs, and nothing else. */ export const PATIENT_KINDS = ["timeline", "alterations", "plot"] as const; export type PatientKind = (typeof PATIENT_KINDS)[number]; @@ -51,9 +53,9 @@ export function allowedKinds(rules: DisclosureRules, study: string): Set parseRules(process.env.MCP_DISCLOSE_PATIENTS, "MCP_DISCLOSE_PATIENTS"); +export const disclosedByThisInstance = () => parseRules(setting("MCP_DISCLOSE_PATIENTS"), "MCP_DISCLOSE_PATIENTS"); /** The coordinator's rules: what this instance accepts from the peer serving each study. */ -export const acceptedFromPeers = () => parseRules(process.env.PEER_MCP_PATIENT_LEVEL, "PEER_MCP_PATIENT_LEVEL"); +export const acceptedFromPeers = () => parseRules(setting("PEER_MCP_PATIENT_LEVEL"), "PEER_MCP_PATIENT_LEVEL"); /** "patient timelines and patient alterations" */ export const kindsText = (kinds: Iterable) => { diff --git a/lib/federation-remote.ts b/lib/federation-remote.ts index c404128d..2fec5616 100644 --- a/lib/federation-remote.ts +++ b/lib/federation-remote.ts @@ -26,6 +26,7 @@ import type { PatientTimeline, PatientEvent, PatientSample, PeerSource } from ". import type { StudyRow } from "./study-list.ts"; import type { StudyCatalog } from "./study-catalog.ts"; import { cohortSubquery, type CohortSql } from "./sql-kinds.ts"; +import { setting } from "./settings.ts"; interface RemoteAggregates { study: string; instance: string; count: number; @@ -53,13 +54,13 @@ export async function peerForStudy(study: string): Promise<{ url: string; label: if (!id || servingAsLoopbackPeer()) return null; const conn = (await studyRouting()).get(study); if (conn?.id !== id) return null; - return { url: loopbackUrl(), label: process.env.LOOPBACK_PEER_LABEL || conn.label + " (simulated peer)" }; + return { url: loopbackUrl(), label: setting("LOOPBACK_PEER_LABEL") || conn.label + " (simulated peer)" }; } /** PEER_MCP as a map from study to the peer URL that serves it. */ export function peerMcpEntries(): Map { const out = new Map(); - for (const entry of (process.env.PEER_MCP || "").split(/[,\s]+/).filter(Boolean)) { + for (const entry of (setting("PEER_MCP") || "").split(/[,\s]+/).filter(Boolean)) { const i = entry.indexOf("="); if (i < 0) continue; const study = entry.slice(0, i).trim(), url = entry.slice(i + 1).trim(); @@ -68,7 +69,7 @@ export function peerMcpEntries(): Map { return out; } -const peerLabel = (): string => process.env.PEER_LABEL || "peer via MCP"; +const peerLabel = (): string => setting("PEER_LABEL") || "peer via MCP"; /** * The study list rows of every PEER_MCP peer, for the studies PEER_MCP maps to it: each peer is @@ -91,7 +92,7 @@ export async function remoteStudies(): Promise<{ rows: StudyRow[]; failed: strin const loop = loopbackOnlyConn(); if (loop) { try { - const label = process.env.LOOPBACK_PEER_LABEL || loop.label + " (simulated peer)"; + const label = setting("LOOPBACK_PEER_LABEL") || loop.label + " (simulated peer)"; for (const r of await loopbackStudies()) rows.push({ ...r, instance: "peer", instanceLabel: label }); } catch { failed.push(loopbackUrl()); } } diff --git a/lib/federation.ts b/lib/federation.ts index 5c00d3c4..001d3c85 100644 --- a/lib/federation.ts +++ b/lib/federation.ts @@ -59,6 +59,7 @@ import { PATIENT_KINDS, type PatientKind } from "./disclosure.ts"; import { finiteSql } from "./sql-number.ts"; import { aggregateSql, cohortSubquery, patientSql, type AggregateSql, type CohortSql, type PatientSql } from "./sql-kinds.ts"; import { DAYS_PER_MONTH } from "./time-units.ts"; +import { onSettingsChange, refreshSettings, setting } from "./settings.ts"; export type InstanceMode = "patient-level" | "aggregate-only"; @@ -169,9 +170,9 @@ export interface PairCounts { * entitled to the data; add ids to STUDY_DENY to exclude more. */ export function isRestrictedStudy(id: string, name = ""): boolean { - const extra = (process.env.STUDY_DENY || "").split(/[,\s]+/).map((x) => x.trim()).filter(Boolean); + const extra = (setting("STUDY_DENY") || "").split(/[,\s]+/).map((x) => x.trim()).filter(Boolean); if (extra.includes(id)) return true; - if (process.env.STUDY_ALLOW_RESTRICTED === "1") return false; + if (setting("STUDY_ALLOW_RESTRICTED") === "1") return false; // GENIE's convention is a version SUFFIX: "v3.1-consortium" vs "v2.0-public". // Match that, not the bare word — several public studies are named after a // consortium (GLASS, A5, Pediatric Preclinical Testing) and must not be hidden. @@ -188,7 +189,11 @@ let _routing: Map | null = null; export const routingCollisions: string[] = []; /** ids withheld by policy — used to explain refusals rather than say "unknown" */ export const restrictedStudies = new Set(); +// the routing reads the settings (which studies are withheld, which connection is a simulated peer) +onSettingsChange("study-routing", () => { _routing = null; _routingEpoch++; routingCollisions.length = 0; restrictedStudies.clear(); }); +let _routingEpoch = 0; export async function studyRouting(): Promise> { + await refreshSettings(); if (_routing) return _routing; // One build at a time, shared by every caller that arrives while it runs: a cold request asks // for hundreds of studies' routes at once, and each starting its own build sent hundreds of @@ -199,6 +204,7 @@ export async function studyRouting(): Promise> { let _routingBuild: Promise> | null = null; async function buildRouting(): Promise> { + const epoch = _routingEpoch; const map = new Map(); let complete = true; for (const conn of connections()) { @@ -225,17 +231,19 @@ async function buildRouting(): Promise> { } // Only a complete answer is worth remembering; a partial one would outlive the outage that // produced it, and a transient failure would look permanent. + // nor is one built under settings that have since changed + if (epoch !== _routingEpoch) complete = false; if (complete) _routing = map; return map; } /** Instances the deployment treats as aggregate-only (prototype switch). */ function aggregateOnlyIds(): string[] { - return (process.env.AGGREGATE_ONLY_INSTANCES || "").split(/[,\s]+/).map(s => s.trim()).filter(Boolean); + return (setting("AGGREGATE_ONLY_INSTANCES") || "").split(/[,\s]+/).map(s => s.trim()).filter(Boolean); } /** Studies forced aggregate-only regardless of instance (finer-grained switch). */ function aggregateOnlyStudies(): string[] { - return (process.env.AGGREGATE_ONLY_STUDIES || "").split(/[,\s]+/).map(s => s.trim()).filter(Boolean); + return (setting("AGGREGATE_ONLY_STUDIES") || "").split(/[,\s]+/).map(s => s.trim()).filter(Boolean); } /** diff --git a/lib/loopback-peer.ts b/lib/loopback-peer.ts index de49eb80..2cde8266 100644 --- a/lib/loopback-peer.ts +++ b/lib/loopback-peer.ts @@ -20,11 +20,12 @@ // another build. The study list reads the connection directly. import { AsyncLocalStorage } from "node:async_hooks"; import { randomUUID, timingSafeEqual } from "node:crypto"; +import { setting } from "./settings.ts"; export const LOOPBACK_HEADER = "x-cbio-loopback-peer"; /** the connection served as a simulated peer, or null when there is none */ -export const loopbackConnId = (): string | null => (process.env.LOOPBACK_PEER || "").trim() || null; +export const loopbackConnId = (): string | null => (setting("LOOPBACK_PEER") || "").trim() || null; // one token and one context per process: Next may load this module once per route bundle const g = globalThis as unknown as { __cbioLoopback?: { token: string; als: AsyncLocalStorage } }; diff --git a/lib/next-cache.ts b/lib/next-cache.ts index 58275018..6bdb39a4 100644 --- a/lib/next-cache.ts +++ b/lib/next-cache.ts @@ -9,6 +9,9 @@ // - a data import or a new OncoKB release: `dataVersion()` is an argument of every cached // function, so new data means new keys // - time: `cacheLife("days")` regenerates an entry a day after it was written +// - an administrator (/admin): a changed federation setting or a cleared cache changes +// `settingsFingerprint()` (lib/settings.ts), which `cacheVersion()` adds to every key, so every +// process stops reading the old entries; a clear also expires them through RESULT_TAG // // Only a value the function returns is stored. A thrown error, a refused request or a query the // reader abandoned is never cached, so every path that means "no answer" must throw. Each cached @@ -27,11 +30,11 @@ // // A cached function takes only serializable arguments and computes its answer itself, so each // route keeps its own small one (`catalogEntry`, `studiesEntry`, `exploreEntry`) beside the code -// it caches, built from `pack`, `throughCache` and `dataVersion` here. +// it caches, built from `pack`, `throughCache`, `cacheVersion` and `resultEntry` here. // // Next-only: `next/cache` exists only under the Next runtime, so the stdio MCP server, the CLI // and the tests use the uncached engine directly. -import { cacheLife } from "next/cache"; +import { cacheLife, cacheTag } from "next/cache"; import { gzipSync, gunzipSync } from "node:zlib"; import { chQuery, directConnections, loopbackOnlyConn, connById, withoutFailedQueries, type ChConn } from "./ch.ts"; import { oncokbPin } from "./annotation.ts"; @@ -42,6 +45,7 @@ import { hasIdSets } from "./id-sets.ts"; import { riskFitDirect, type RiskFit, type RiskFitArgs } from "./views/riskmodel/run.ts"; import type { AnalysisDoc, AnalysisResult, Contract } from "./types.ts"; import type { OncoprintSpec } from "./views/oncoprint/types.ts"; +import { onSettingsChange, refreshSettings, settingsFingerprint } from "./settings.ts"; type Json = unknown; @@ -90,6 +94,27 @@ export function dataVersion(): Promise { value.catch(() => { if (versionMemo?.value === value) versionMemo = null; }); return value; } +// a cleared cache or a changed setting asks the databases again +onSettingsChange("data-version", () => { versionMemo = null; }); + +/** Every cached result's tag, which clearing the cache from /admin expires. */ +export const RESULT_TAG = "results"; + +/** The lifetime and tag of every cached result: called first in each `'use cache: remote'` body. */ +export function resultEntry(): void { + cacheLife("days"); + cacheTag(RESULT_TAG); +} + +/** + * What every cached result is keyed by besides its own arguments: the data (`dataVersion`) and the + * administrator's settings and last cache clear (lib/settings.ts). + */ +export async function cacheVersion(): Promise { + await refreshSettings(); + const data = await dataVersion(), s = settingsFingerprint(); + return s ? data + "|settings@" + s : data; +} /** * One cohort's size on one connection, by its compiled SQL — which already says everything that @@ -99,7 +124,7 @@ export function dataVersion(): Promise { */ async function countEntry(connId: string, sql: string, _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); const conn = connById(connId); if (!conn) throw new Error("no connection '" + connId + "'"); return pack(await withoutFailedQueries(async () => @@ -111,7 +136,7 @@ async function countDirect(conn: ChConn, sql: string): Promise { export async function cachedCount(conn: ChConn, sql: string): Promise { // a statement over uploaded data belongs to its uploader, never to the shared cache if (hasCustomData() || hasIdSets()) return countDirect(conn, sql); - return throughCache(async () => countEntry(conn.id, sql, await dataVersion()), () => countDirect(conn, sql)); + return throughCache(async () => countEntry(conn.id, sql, await cacheVersion()), () => countDirect(conn, sql)); } /** @@ -131,14 +156,14 @@ const RISK_FIT_CACHE_MAX_PATIENTS = 60_000; */ async function riskFitEntry(args: RiskFitArgs, _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await withoutFailedQueries(() => riskFitDirect(args))); } export async function cachedRiskFit(a: RiskFitArgs & { patients: number }): Promise { const { patients, ...args } = a; // a fit over uploaded data belongs to its uploader, never to the shared cache if (hasCustomData() || hasIdSets() || patients > RISK_FIT_CACHE_MAX_PATIENTS) return riskFitDirect(args); - return throughCache(async () => riskFitEntry(args, await dataVersion()), () => riskFitDirect(args)); + return throughCache(async () => riskFitEntry(args, await cacheVersion()), () => riskFitDirect(args)); } /** What the routes run a document with: the cached count and fit. */ @@ -146,19 +171,19 @@ const ROUTE_OPTS = { count: cachedCount, riskFit: cachedRiskFit }; async function analysisEntry(doc: AnalysisDoc, oncoprint: "svg" | "spec", _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await withoutFailedQueries(() => runAnalysis(doc, { oncoprint, ...ROUTE_OPTS }))); } export async function cachedAnalysis(doc: AnalysisDoc, opts: { oncoprint?: "svg" | "spec" } = {}): Promise { const oncoprint = opts.oncoprint ?? "svg"; // an answer that depends on uploaded data belongs to its uploader, never to the shared cache if (hasCustomData() || hasIdSets()) return runAnalysis(doc, { oncoprint }); - return throughCache(async () => analysisEntry(doc, oncoprint, await dataVersion()), () => runAnalysis(doc, { oncoprint, ...ROUTE_OPTS })); + return throughCache(async () => analysisEntry(doc, oncoprint, await cacheVersion()), () => runAnalysis(doc, { oncoprint, ...ROUTE_OPTS })); } async function contractEntry(spec: OncoprintSpec, provenance: boolean, _version: string): Promise { "use cache: remote"; - cacheLife("days"); + resultEntry(); return pack(await withoutFailedQueries(() => buildContract(spec, provenance ? { log: [], provenance } : {}))); } /** @@ -169,5 +194,5 @@ export async function cachedContract(spec: OncoprintSpec, opts: { provenance?: b const provenance = !!opts.provenance; const direct = () => buildContract(spec, provenance ? { log: [], provenance } : {}); if (hasCustomData() || hasIdSets()) return direct(); - return throughCache(async () => contractEntry(spec, provenance, await dataVersion()), direct); + return throughCache(async () => contractEntry(spec, provenance, await cacheVersion()), direct); } diff --git a/lib/provenance.ts b/lib/provenance.ts index b36975e2..8b94bc18 100644 --- a/lib/provenance.ts +++ b/lib/provenance.ts @@ -6,6 +6,7 @@ import { chQuery, isAbortError, type ChConn } from "./ch.ts"; import { sq } from "./compiler.ts"; import { BUILD_INFO } from "./build-info.ts"; import { studyRouting, type Resolver } from "./federation.ts"; +import { setting } from "./settings.ts"; import type { AnalysisSource, PeerSource } from "./types.ts"; const IMPORTED_TTL = 5 * 60_000; @@ -27,7 +28,7 @@ export function studyImported(study: string, conn: ChConn): Promise { const conn = study ? (await studyRouting()).get(study) : undefined; - const base = { name: conn?.label || process.env.CLICKHOUSE_LABEL || "primary", build: BUILD_INFO.label }; + const base = { name: conn?.label || setting("CLICKHOUSE_LABEL") || "primary", build: BUILD_INFO.label }; return conn && study ? { ...base, study, imported: await studyImported(study, conn) } : base; } diff --git a/lib/settings.ts b/lib/settings.ts new file mode 100644 index 00000000..8c9c7000 --- /dev/null +++ b/lib/settings.ts @@ -0,0 +1,263 @@ +// Deployment settings an administrator can see and change at run time (/admin). A setting is an +// environment variable; a row in app_setting (db/migrations/0010_app_settings.sql) overrides it for +// every server process of the deployment, and removing the row returns to the variable. +// +// `setting(name)` is what code reads instead of `process.env[name]` for the settings listed here. +// It is synchronous, so the 40-odd places federation consults one keep their shape: it answers from +// this process's copy of the overrides, which `refreshSettings()` reloads at most every few seconds. +// The request paths that decide federation await that refresh first — the study routing, every +// cached result's key (lib/next-cache.ts) and the MCP route — so a process answers by a change +// within REFRESH_MS of it, and a cold one never answers by the environment alone. +// +// Overrides apply only under the Next server, which turns them on in instrumentation.ts. The CLI, +// the stdio MCP server and the tests read the environment as before. +// +// When the overrides cannot be read: +// - no store (Vercel production without DATABASE_URL), or app_setting not migrated yet: there can +// be no overrides, so the environment is the whole answer and /admin says why it cannot edit; +// - the store fails: the last overrides read keep applying; a process that never read them +// answers by the environment, says so on /admin, and tries again on the next refresh. A store +// outage must not take analyses down with it, though an override that tightened federation is +// then not applied by that process until the store answers. +// +// Database addresses and credentials are not settings here: /admin shows the connections as +// lib/ch.ts describes them (the one module that reads them), and nothing changes them. Nor are +// LOOPBACK_PEER_URL and LOOPBACK_PEER_TOKEN editable: the app sends that token to that address, so +// the pair is a credential. +import { createHash } from "node:crypto"; +import { dbQuery, databaseUrl } from "./db.ts"; +import { parseRules } from "./disclosure.ts"; +import { describeConnections } from "./ch.ts"; + +export type SettingGroup = "databases" | "peers" | "patient-level" | "access"; + +export interface SettingSpec { + name: string; + group: SettingGroup; + /** what it does, for /admin */ + help: string; + /** changeable from /admin */ + editable: boolean; + /** its value is never shown, only whether it is set */ + secret?: boolean; + /** throws, naming the problem, on a value the app could not use */ + check?: (v: string) => void; +} + +const list = (v: string) => v.split(/[,\s]+/).filter(Boolean); +const label = (v: string) => { if (v.length > 120) throw new Error("a label is at most 120 characters"); }; + +export const SETTINGS: readonly SettingSpec[] = [ + { name: "CLICKHOUSE_LABEL", group: "databases", editable: true, check: label, help: "how results and captions name the primary warehouse (connection a)" }, + { name: "CLICKHOUSE2_LABEL", group: "databases", editable: true, check: label, help: "how results and captions name the second warehouse (connection b)" }, + { + name: "PEER_MCP", group: "peers", editable: true, + help: "studies held by peer cbio-lab instances, as study=URL pairs (the peer's /api/mcp); this instance asks them for aggregates over MCP", + check: (v) => { + for (const e of list(v)) { + const i = e.indexOf("="); + if (i < 1) throw new Error("'" + e.slice(0, 80) + "' is not study=URL"); + let u: URL; + try { u = new URL(e.slice(i + 1)); } catch { throw new Error("'" + e.slice(i + 1, i + 81) + "' is not a URL"); } + if (u.protocol !== "https:" && u.protocol !== "http:") throw new Error("a peer is reached over http(s), not " + u.protocol); + } + }, + }, + { name: "PEER_LABEL", group: "peers", editable: true, check: label, help: "how results and captions name a PEER_MCP peer" }, + { + name: "LOOPBACK_PEER", group: "peers", editable: true, + help: "a connection id (e.g. b, the second warehouse) this app serves to itself over MCP as a simulated peer", + check: (v) => { if (v.trim() && !/^[ab]$/.test(v.trim())) throw new Error("LOOPBACK_PEER names a connection: a or b"); }, + }, + { name: "LOOPBACK_PEER_LABEL", group: "peers", editable: true, check: label, help: "how results and captions name the simulated peer" }, + { name: "LOOPBACK_PEER_URL", group: "peers", editable: false, help: "this app's own /api/mcp, where the simulated peer is reached (else 127.0.0.1 on PORT); not editable, as the loopback token is sent to it" }, + { name: "LOOPBACK_PEER_TOKEN", group: "peers", editable: false, secret: true, help: "what tells this app's calls to itself from anyone else's, where several processes serve it" }, + { + name: "PEER_MCP_PATIENT_LEVEL", group: "patient-level", editable: true, + help: "as the coordinator: the kinds of patient-level data accepted from the peer serving each study (e.g. genie_public:timeline+alterations, *:timeline)", + check: (v) => { parseRules(v, "PEER_MCP_PATIENT_LEVEL"); }, + }, + { + name: "MCP_DISCLOSE_PATIENTS", group: "patient-level", editable: true, + help: "as a peer: the kinds of patient-level data this instance sends to any caller of its /api/mcp, per study. There is no caller authentication yet (docs/DATA-BOUNDARY.md)", + check: (v) => { parseRules(v, "MCP_DISCLOSE_PATIENTS"); }, + }, + { + name: "FEDERATION_HIDE_SQL", group: "peers", editable: true, + help: "1 keeps this instance's compiled SQL out of what it answers peers over MCP", + check: (v) => { if (v !== "" && v !== "1") throw new Error("FEDERATION_HIDE_SQL is 1 or empty"); }, + }, + { + name: "AGGREGATE_ONLY_INSTANCES", group: "access", editable: true, + help: "connection ids (a, b) treated as aggregate-only: never asked for a patient id", + check: (v) => { for (const id of list(v)) if (!/^[ab]$/.test(id)) throw new Error("'" + id.slice(0, 40) + "' is not a connection id (a or b)"); }, + }, + { + name: "AGGREGATE_ONLY_STUDIES", group: "access", editable: true, + help: "study ids treated as aggregate-only whichever connection holds them", + check: (v) => { for (const id of list(v)) if (!/^[A-Za-z0-9._-]{1,120}$/.test(id)) throw new Error("'" + id.slice(0, 80) + "' is not a study id"); }, + }, + { + name: "STUDY_DENY", group: "access", editable: true, + help: "study ids never routed, so unreachable from every surface", + check: (v) => { for (const id of list(v)) if (!/^[A-Za-z0-9._-]{1,120}$/.test(id)) throw new Error("'" + id.slice(0, 80) + "' is not a study id"); }, + }, + { + name: "STUDY_ALLOW_RESTRICTED", group: "access", editable: true, + help: "1 routes consortium- and institution-restricted studies, which are otherwise unreachable", + check: (v) => { if (v !== "" && v !== "1") throw new Error("STUDY_ALLOW_RESTRICTED is 1 or empty"); }, + }, +]; + +const SPEC = new Map(SETTINGS.map((s) => [s.name, s])); +export const settingSpec = (name: string): SettingSpec | undefined => SPEC.get(name); + +/** When an administrator last cleared the result cache: a row of app_setting, never a variable. */ +export const CACHE_CLEARED = "result_cache_cleared_at"; + +const REFRESH_MS = 5_000; +const READ_TIMEOUT_MS = 3_000; + +type Overrides = Map; +interface State { + enabled: boolean; + overrides: Overrides | null; + /** why the overrides are not in use, when they are not */ + unavailable: string | null; + at: number; + loading: Promise | null; + fingerprint: string; + listeners: Map void>; +} +// one copy per process: Next may load this module once per route bundle +const g = globalThis as unknown as { __cbioSettings?: State }; +const state = g.__cbioSettings ??= { enabled: false, overrides: null, unavailable: null, at: 0, loading: null, fingerprint: "", listeners: new Map() }; + +/** Turn the overrides on for this process; the Next server does, at startup (instrumentation.ts). */ +export function enableSettingOverrides(): void { state.enabled = true; } + +/** The setting's value: the administrator's override, else the environment variable. */ +export function setting(name: string): string | undefined { + if (state.enabled && Date.now() - state.at > REFRESH_MS) void refreshSettings().catch(() => {}); + const o = SPEC.get(name)?.editable ? state.overrides?.get(name) : undefined; + return o ? o.value : process.env[name]; +} + +/** + * Run `fn` whenever the overrides change in this process (after the first read). Keyed, so a module + * loaded again replaces its listener rather than adding another. + */ +export function onSettingsChange(key: string, fn: () => void): void { state.listeners.set(key, fn); } + +/** What every cached result's key carries: the overrides in force and when the cache was cleared. */ +export function settingsFingerprint(): string { return state.fingerprint; } + +const storeMissing = (): string | null => + !databaseUrl() && process.env.VERCEL_ENV === "production" ? "this deployment has no DATABASE_URL, so nothing can override its environment" : null; +const notMigrated = (e: unknown) => /app_setting.*does not exist/i.test(String((e as Error)?.message || e)); + +/** Reload the overrides if they are older than REFRESH_MS; one load at a time per process. */ +export function refreshSettings(force = false): Promise { + if (!state.enabled) return Promise.resolve(); + if (!force && Date.now() - state.at <= REFRESH_MS) return Promise.resolve(); + return state.loading ??= load().finally(() => { state.loading = null; }); +} + +async function load(): Promise { + let next: Overrides | null = null, why = storeMissing(); + if (!why) { + try { + // requests wait on this read, so a store that hangs costs them READ_TIMEOUT_MS at most + let timer: ReturnType | undefined; + const rows = await Promise.race([ + dbQuery<{ name: string; value: string; updated_at: Date | string; updated_by: string | null }>( + "select name, value, updated_at, updated_by from app_setting order by name"), + new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("no answer in " + READ_TIMEOUT_MS + " ms")), READ_TIMEOUT_MS); }), + ]).finally(() => clearTimeout(timer)); + next = new Map(rows.map((r) => [r.name, { value: r.value, at: new Date(r.updated_at).toISOString(), by: r.updated_by }])); + } catch (e) { + const msg = String((e as Error)?.message || e); + if (!notMigrated(e)) { + console.warn("[settings] could not read the overrides:", msg); + // the last overrides read stand until the store answers again + if (state.overrides) { state.at = Date.now(); return; } + } + why = notMigrated(e) ? "the settings table has not been created; run `npm run db:migrate`" + : "the settings database could not be reached (" + msg.slice(0, 200) + "), so the environment's values are in force"; + } + } + const fp = next && next.size ? createHash("sha256").update(JSON.stringify([...next].map(([k, v]) => [k, v.value]))).digest("hex").slice(0, 16) : ""; + const changed = fp !== state.fingerprint; + Object.assign(state, { overrides: next, unavailable: why, at: Date.now(), fingerprint: fp }); + if (changed) for (const fn of state.listeners.values()) fn(); +} + +export interface SettingRow { + name: string; group: SettingGroup; help: string; editable: boolean; secret: boolean; + /** the environment's value (null for a secret: `envSet` says whether there is one) */ + env: string | null; envSet: boolean; + /** the administrator's override, when there is one */ + override: { value: string; at: string; by: string | null } | null; +} +export interface SettingsReport { + /** the warehouses this instance queries, read-only */ + databases: ReturnType; + rows: SettingRow[]; + /** why overrides cannot be made here, when they cannot */ + unavailable: string | null; + cacheCleared: { at: string; by: string | null } | null; + log: { name: string; old: string | null; new: string | null; at: string; by: string | null }[]; +} + +/** Every setting as /admin shows it, with its recent changes. Secrets are never included. */ +export async function settingsReport(): Promise { + await refreshSettings(true); + const o = state.overrides; + const rows = SETTINGS.map((s): SettingRow => { + const env = process.env[s.name]; + const ov = s.editable ? o?.get(s.name) ?? null : null; + return { name: s.name, group: s.group, help: s.help, editable: s.editable && !state.unavailable, secret: !!s.secret, + env: s.secret ? null : env ?? null, envSet: !!env, override: ov }; + }); + const cleared = o?.get(CACHE_CLEARED); + let log: SettingsReport["log"] = []; + if (!state.unavailable) { + log = (await dbQuery<{ name: string; old_value: string | null; new_value: string | null; changed_at: Date | string; email: string | null }>( + "select l.name, l.old_value, l.new_value, l.changed_at, u.email from app_setting_log l left join app_user u on u.id = l.changed_by" + + " order by l.id desc limit 50")).map((r) => ({ name: r.name, old: r.old_value, new: r.new_value, at: new Date(r.changed_at).toISOString(), by: r.email })); + } + const by = cleared?.by ? log.find((l) => l.name === CACHE_CLEARED)?.by ?? null : null; + return { databases: describeConnections(), rows, unavailable: state.unavailable, cacheCleared: cleared ? { at: cleared.value, by } : null, log }; +} + +async function write(name: string, value: string | null, by: string): Promise { + await refreshSettings(true); + if (state.unavailable) throw new Error("settings cannot be changed here: " + state.unavailable); + const [old] = await dbQuery<{ value: string }>("select value from app_setting where name = $1", [name]); + if (value == null) await dbQuery("delete from app_setting where name = $1", [name]); + else await dbQuery("insert into app_setting (name, value, updated_by) values ($1, $2, $3)" + + " on conflict (name) do update set value = excluded.value, updated_at = now(), updated_by = excluded.updated_by", [name, value, by]); + await dbQuery("insert into app_setting_log (name, old_value, new_value, changed_by) values ($1, $2, $3, $4)", [name, old?.value ?? null, value, by]); + await refreshSettings(true); +} + +/** + * Override a setting for the whole deployment (`value`), or return it to its environment variable + * (null). Refused for a setting that is not editable and for a value the app could not use. + */ +export async function setSetting(name: string, value: string | null, by: string): Promise { + const spec = SPEC.get(name); + if (!spec && /^CLICKHOUSE/.test(name)) throw new Error(name + " is part of a database connection's address or credentials, which cannot be changed here"); + if (!spec) throw new Error("no setting " + name.slice(0, 80) + "; the settings are " + SETTINGS.map((s) => s.name).join(", ")); + if (!spec.editable) throw new Error(name + " cannot be changed here"); + if (value != null) { + value = value.trim(); + spec.check?.(value); + } + await write(name, value, by); +} + +/** Mark the result cache cleared now: every cached entry's key changes, in every process. */ +export async function markResultCacheCleared(by: string): Promise { + await write(CACHE_CLEARED, new Date().toISOString(), by); +} diff --git a/mcp/tools.ts b/mcp/tools.ts index 264b1e87..f6a61330 100644 --- a/mcp/tools.ts +++ b/mcp/tools.ts @@ -31,6 +31,7 @@ import { encodeUrlJson } from "../lib/url-json.ts"; import type { OncoprintSpec } from "../lib/views/oncoprint/types.ts"; import { validateAxis, type PlotAxis, type PlotRequest, type AxisKind, type PointRank, type SparseCells, type Whiskers } from "../lib/plot.ts"; import { resolveAxes } from "../lib/plot-profile.ts"; +import { setting } from "../lib/settings.ts"; // ---------- shared zod fragments (the tools' input schemas) ---------- const zSelector = z.object({ @@ -220,7 +221,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { // above, so disclosing what we ran restores end-to-end provenance for the // caller. A deployment that treats its schema as sensitive sets // FEDERATION_HIDE_SQL=1 and the caller degrades gracefully. - const hideSql = process.env.FEDERATION_HIDE_SQL === "1"; + const hideSql = setting("FEDERATION_HIDE_SQL") === "1"; const cohortSql = hideSql ? null : res.cohortSql(cohort); const payload = { study: s, instance: res.instance.label, count, geneCounts, survivalCounts, pairCounts, attributeStats, pairMoments, variantCounts, profileMoments, survivalEndpoints: endpointList, ...(lollipopCounts ? { lollipopCounts } : {}), ...(spans ? { timelineSpan: spans } : {}), ...(courseCounts ? { eventCourse: courseCounts } : {}), ...(flows ? { treatmentSequence: flows } : {}), ...(keys ? { sequenceKeys: keys } : {}), cohortSql, queries: hideSql ? null : res.queries, oncokbPin: pin }; return { @@ -290,7 +291,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { case "highlightBox": need(hl, "highlight"); need(k, "kinds"); result = await res.plotHighlightBoxRows(args.cohort, hl, spec, k!.xk, k!.yk); break; default: throw new Error("unknown op"); } - const hideSql = process.env.FEDERATION_HIDE_SQL === "1"; + const hideSql = setting("FEDERATION_HIDE_SQL") === "1"; return { content: [{ type: "text" as const, text: "plot " + args.op + " for " + s + " (aggregates; no patient identifiers returned)" }], structuredContent: { study: s, op: args.op, result, cohortSql: hideSql || args.op === "axisKinds" ? null : res.cohortSql(args.cohort), queries: hideSql ? null : res.queries, source: await peerSource(s) } @@ -326,7 +327,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { // (PEER_TABLES) — for a study this instance holds at patient level itself. The coordinator must // accept the same kind (PEER_MCP_PATIENT_LEVEL); see docs/DATA-BOUNDARY.md. There is no caller // authentication yet: anyone who can reach this endpoint can read what it discloses. - if ((process.env.MCP_DISCLOSE_PATIENTS || "").trim()) { + if ((setting("MCP_DISCLOSE_PATIENTS") || "").trim()) { const held = async (study: string, kind: PatientKind): Promise<{ s: string; res: PatientLevelResolver }> => { // the rules are checked on the study as asked, before anything is read, and again as resolved const refuse = (x: string) => new Error(x.slice(0, 120) + " is not a study this instance discloses " + PATIENT_KIND_NAMES[kind] + " of"); @@ -413,7 +414,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { if (!patient || !PATIENT_ID.test(patient)) throw new Error("rank needs a patient id"); out = { rank: await res.rowsRank(tag, src, q, defaults, s, patient) }; } else out = { rows: (await res.rowsPage(tag, src, q, defaults, limit || 25, offset || 0)).rows }; - const hideSql = process.env.FEDERATION_HIDE_SQL === "1"; + const hideSql = setting("FEDERATION_HIDE_SQL") === "1"; return { content: [{ type: "text" as const, text: `${table} rows for ${s} (${mode}) — patient-level, disclosed under MCP_DISCLOSE_PATIENTS (${PATIENT_KIND_NAMES[PEER_TABLES[table]]})` }], structuredContent: { table, ...out, ...(annotated ? { oncokbPin: pin } : {}), queries: hideSql ? null : res.queries, source: await peerSource(s) } @@ -434,7 +435,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { const { s, res } = await held(study, "timeline"); const t = await res.patientTimeline(patient); if (!t.events.length && !t.samples.length) throw new Error(patient + " has no record in " + s); - const hideSql = process.env.FEDERATION_HIDE_SQL === "1"; + const hideSql = setting("FEDERATION_HIDE_SQL") === "1"; return { content: [{ type: "text" as const, text: `timeline of ${patient} (${s}): ${t.events.length} events, ${t.samples.length} samples — patient-level, disclosed under MCP_DISCLOSE_PATIENTS` }], structuredContent: { study: s, patient, events: t.events, samples: t.samples, queries: hideSql ? null : res.queries, source: await peerSource(s) } @@ -485,7 +486,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { if (!args.highlight || !args.kinds) throw new Error("highlightPoints needs highlight and kinds"); points = await res.plotHighlightPoints(args.cohort, args.highlight, spec, args.kinds.xk, args.kinds.yk, cap); } - const hideSql = process.env.FEDERATION_HIDE_SQL === "1"; + const hideSql = setting("FEDERATION_HIDE_SQL") === "1"; return { content: [{ type: "text" as const, text: "plot " + args.op + " for " + s + " — patient-level, disclosed under MCP_DISCLOSE_PATIENTS (" + PATIENT_KIND_NAMES.plot + ")" }], structuredContent: { study: s, op: args.op, points, queries: hideSql ? null : res.queries, source: await peerSource(s) } diff --git a/test/auth.test.ts b/test/auth.test.ts index a598fe79..ee60f968 100644 --- a/test/auth.test.ts +++ b/test/auth.test.ts @@ -43,6 +43,29 @@ describe("which sign-ins a deployment offers", () => { }); }); +describe("administrators", () => { + beforeEach(() => { reset(); process.env.AUTH_DEV_LOGIN = "1"; process.env.AUTH_SESSION_SECRET = "a".repeat(40); delete process.env.ADMIN_EMAILS; }); + after(() => { process.env = { ...env }; }); + const signedIn = (u = U) => req(auth.sessionCookie(req(), u).split(";")[0]); + + test("are the signed-in users ADMIN_EMAILS names, whatever the case", () => { + process.env.ADMIN_EMAILS = "someone@example.org, ADA@example.org"; + assert.equal(auth.isAdmin(U), true); + assert.deepEqual(auth.requireAdmin(signedIn()), U); + assert.equal(auth.isAdmin({ ...U, email: "bob@example.org" }), false); + assert.throws(() => auth.requireAdmin(signedIn({ ...U, email: "bob@example.org" })), (e) => e instanceof auth.AccessError && e.status === 403 && /bob@example.org is not an administrator/.test(e.message)); + }); + test("there are none without ADMIN_EMAILS, nobody signed in, or sign-in", () => { + assert.equal(auth.isAdmin(U), false); + assert.equal(auth.isAdmin(null), false); + assert.throws(() => auth.requireAdmin(signedIn()), /not an administrator/); + process.env.ADMIN_EMAILS = U.email; + assert.throws(() => auth.requireAdmin(req()), (e) => e instanceof auth.AccessError && e.status === 403); + delete process.env.AUTH_DEV_LOGIN; + assert.throws(() => auth.requireAdmin(req()), (e) => e instanceof auth.AccessError && e.status === 404); + }); +}); + describe("the session cookie", () => { beforeEach(() => { reset(); process.env.AUTH_DEV_LOGIN = "1"; process.env.AUTH_SESSION_SECRET = "a".repeat(40); }); after(() => { process.env = { ...env }; }); diff --git a/test/postgres.db-test.ts b/test/postgres.db-test.ts index 3db46563..0b9de03b 100644 --- a/test/postgres.db-test.ts +++ b/test/postgres.db-test.ts @@ -17,13 +17,13 @@ const db: PGlite = await startPostgres(); describe("migrations", () => { test("apply in order to an empty database, and again without harm", async () => { - assert.deepEqual(migrationFiles(), ["0001_saved_artifacts.sql", "0002_campaigns.sql", "0003_custom_datasets.sql", "0004_users.sql", "0005_sharing.sql", "0006_id_sets.sql", "0007_id_set_parts.sql", "0008_custom_dataset_note.sql", "0009_app_role.sql"]); + assert.deepEqual(migrationFiles(), ["0001_saved_artifacts.sql", "0002_campaigns.sql", "0003_custom_datasets.sql", "0004_users.sql", "0005_sharing.sql", "0006_id_sets.sql", "0007_id_set_parts.sql", "0008_custom_dataset_note.sql", "0009_app_role.sql", "0010_app_settings.sql"]); const tables = async () => (await db.query<{ t: string }>( "select table_name as t from information_schema.tables where table_schema = 'public' order by 1")).rows.map((r) => r.t); const before = await tables(); await reapplyMigrations(db); assert.deepEqual(await tables(), before); - for (const t of ["app_user", "user_identity", "saved_artifacts", "campaigns", "campaign_hypotheses", "custom_dataset", "custom_dataset_version", "custom_data_access_log", "user_group", "group_member", "id_set", "id_set_access_log"]) + for (const t of ["app_user", "user_identity", "saved_artifacts", "campaigns", "campaign_hypotheses", "custom_dataset", "custom_dataset_version", "custom_data_access_log", "user_group", "group_member", "id_set", "id_set_access_log", "app_setting", "app_setting_log"]) assert.ok(before.includes(t), t + " exists"); }); diff --git a/test/result-cache-tag.test.ts b/test/result-cache-tag.test.ts new file mode 100644 index 00000000..7d556fab --- /dev/null +++ b/test/result-cache-tag.test.ts @@ -0,0 +1,23 @@ +// Clearing the result cache from /admin expires one tag (RESULT_TAG, lib/next-cache.ts), so it +// clears only entries that carry it: every `'use cache: remote'` body must start with resultEntry(), +// which sets the tag with the lifetime. A cached function written without it would survive a clear. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readdirSync, readFileSync, statSync } from "node:fs"; + +test("every cached function carries the tag a cache clear expires", () => { + const files: string[] = []; + const walk = (d: string) => { for (const f of readdirSync(d)) { const p = d + "/" + f; if (statSync(p).isDirectory()) walk(p); else if (/\.tsx?$/.test(f)) files.push(p); } }; + for (const d of ["app", "lib", "mcp"]) walk(d); + const bodies: string[] = [], untagged: string[] = []; + for (const f of files) { + const lines = readFileSync(f, "utf8").split("\n"); + lines.forEach((l, i) => { + if (!/^\s*["']use cache(: \w+)?["'];/.test(l)) return; + bodies.push(f + ":" + (i + 1)); + if (!/^\s*resultEntry\(\);/.test(lines[i + 1] || "")) untagged.push(f + ":" + (i + 1)); + }); + } + assert.ok(bodies.length >= 9, "expected the cached functions of lib/next-cache.ts and the routes; found " + bodies.length); + assert.deepEqual(untagged, [], "start these cached bodies with resultEntry() (lib/next-cache.ts), or /admin's clear misses them"); +}); diff --git a/test/settings.db-test.ts b/test/settings.db-test.ts new file mode 100644 index 00000000..d6f67796 --- /dev/null +++ b/test/settings.db-test.ts @@ -0,0 +1,114 @@ +// The administrator's setting overrides (lib/settings.ts) on the in-process Postgres, as the app's +// role: an override wins over the environment and reverting returns to it, credentials and unusable +// values are refused, every change is logged, the fingerprint that keys the result cache moves with +// each change and with a cache clear, and the study routing is rebuilt under the new settings. +import { test, describe, before } from "node:test"; +import assert from "node:assert/strict"; + +process.env.PEER_LABEL = "from the environment"; +process.env.CLICKHOUSE_PASSWORD = "env-secret"; +process.env.CLICKHOUSE_URL = "https://user:env-secret@warehouse.invalid:8443"; +process.env.LOOPBACK_PEER_TOKEN = "t".repeat(40); +for (const k of ["PEER_MCP", "STUDY_DENY", "MCP_DISCLOSE_PATIENTS", "VERCEL_ENV"]) delete process.env[k]; + +const { startPostgres } = await import("./support/postgres.ts"); +const { userForEmail } = await import("../lib/users.ts"); +const settings = await import("../lib/settings.ts"); +const { isRestrictedStudy } = await import("../lib/federation.ts"); +const { acceptedFromPeers } = await import("../lib/disclosure.ts"); + +const db = await startPostgres(); +let admin: string; + +before(async () => { + admin = (await userForEmail("admin@example.org")).id; +}); + +describe("without the Next server, the environment is the whole answer", () => { + test("setting() reads the variable and the store is never asked", async () => { + await settings.refreshSettings(true); + assert.equal(settings.setting("PEER_LABEL"), "from the environment"); + assert.equal(settings.settingsFingerprint(), ""); + }); +}); + +describe("overrides", () => { + before(() => settings.enableSettingOverrides()); + + test("an override wins over the environment, and reverting returns to it", async () => { + await settings.setSetting("PEER_LABEL", " GENIE site ", admin); + assert.equal(settings.setting("PEER_LABEL"), "GENIE site"); + await settings.setSetting("PEER_LABEL", null, admin); + assert.equal(settings.setting("PEER_LABEL"), "from the environment"); + }); + + test("an empty override means the setting is empty, not the environment's", async () => { + await settings.setSetting("PEER_LABEL", "", admin); + assert.equal(settings.setting("PEER_LABEL"), ""); + await settings.setSetting("PEER_LABEL", null, admin); + }); + + test("credentials and unknown settings are refused", async () => { + await assert.rejects(settings.setSetting("CLICKHOUSE_PASSWORD", "x", admin), /credentials/); + await assert.rejects(settings.setSetting("CLICKHOUSE2_URL", "http://elsewhere.invalid", admin), /credentials/); + await assert.rejects(settings.setSetting("LOOPBACK_PEER_URL", "http://elsewhere.invalid/api/mcp", admin), /cannot be changed here/); + await assert.rejects(settings.setSetting("ANTHROPIC_API_KEY", "x", admin), /no setting/); + }); + + test("a value the app could not use is refused, naming what is wrong", async () => { + await assert.rejects(settings.setSetting("MCP_DISCLOSE_PATIENTS", "genie_public:everything", admin), /unknown kind 'everything'/); + await assert.rejects(settings.setSetting("PEER_MCP", "genie_public", admin), /not study=URL/); + await assert.rejects(settings.setSetting("PEER_MCP", "genie_public=file:///etc/passwd", admin), /http\(s\)/); + await assert.rejects(settings.setSetting("LOOPBACK_PEER", "c", admin), /a or b/); + await assert.rejects(settings.setSetting("STUDY_ALLOW_RESTRICTED", "yes", admin), /1 or empty/); + const n = (await db.query<{ n: number }>("select count(*)::int as n from app_setting where name in ('MCP_DISCLOSE_PATIENTS','PEER_MCP','LOOPBACK_PEER')")).rows[0].n; + assert.equal(n, 0); + }); + + test("the rules federation reads come from the override", async () => { + await settings.setSetting("PEER_MCP_PATIENT_LEVEL", "genie_public:timeline", admin); + assert.deepEqual([...acceptedFromPeers().get("genie_public")!], ["timeline"]); + await settings.setSetting("STUDY_DENY", "brca_tcga", admin); + assert.equal(isRestrictedStudy("brca_tcga", "Breast"), true); + await settings.setSetting("STUDY_DENY", null, admin); + assert.equal(isRestrictedStudy("brca_tcga", "Breast"), false); + await settings.setSetting("PEER_MCP_PATIENT_LEVEL", null, admin); + }); + + test("the fingerprint moves with every change and with a cache clear", async () => { + const seen = new Set([settings.settingsFingerprint()]); + await settings.setSetting("PEER_LABEL", "one", admin); + seen.add(settings.settingsFingerprint()); + await settings.markResultCacheCleared(admin); + seen.add(settings.settingsFingerprint()); + await settings.setSetting("PEER_LABEL", null, admin); + seen.add(settings.settingsFingerprint()); + assert.equal(seen.size, 4); + }); + + test("a change runs the listeners, so the study routing is rebuilt", async () => { + let ran = 0; + settings.onSettingsChange("test", () => { ran++; }); + await settings.setSetting("STUDY_DENY", "x_study", admin); + await settings.setSetting("STUDY_DENY", null, admin); + assert.equal(ran, 2); + }); + + test("every change is logged with who made it, and the report never holds a secret", async () => { + const r = await settings.settingsReport(); + assert.equal(r.databases[0].passwordSet, true); + assert.ok(!JSON.stringify(r).includes("env-secret")); + const token = r.rows.find((x) => x.name === "LOOPBACK_PEER_TOKEN")!; + assert.equal(token.env, null); + assert.equal(token.editable, false); + assert.ok(r.log.length >= 10); + assert.ok(r.log.every((l) => l.by === "admin@example.org")); + assert.ok(r.cacheCleared); + }); + + test("the app's role cannot rewrite the log", async () => { + const asApp = (sql: string) => db.transaction(async (tx) => { await tx.exec("set local role cohort_lab_app"); return tx.query(sql); }); + await assert.rejects(asApp("delete from app_setting_log"), /permission denied/); + await assert.rejects(asApp("update app_setting_log set new_value = 'forged'"), /permission denied/); + }); +});