From 3e77cc06ca3a6f3ecd09254e4b3603eef31371ae Mon Sep 17 00:00:00 2001 From: Aaron Lisman Date: Mon, 26 Oct 2026 23:20:24 +0000 Subject: [PATCH] Shorter links: ?doc= is deflated, and portal pages carry only their changed settings Every JSON value the app writes into a URL (lib/url-json.ts) is now `1.`: the JSON's UTF-8 bytes, raw deflate with no dictionary, so any zlib reads it (Python: zlib.decompress(data, -15)). A value with no "." is the plain base64url JSON that links were before, and still opens, so old links, scripts and hand-made links keep working. The lab, the Examples tab, the reproductions and campaign pages, the agent tools and MCP's run_analysis link all write it; the oncoprint lab's ?spec= is unchanged and shares the base64 helpers. The Results and Comparison pages no longer put their whole document in the URL. ?changes= (app/portal/page-changes.ts) holds only how each view differs from the page's defaults, which are rebuilt from the query, keyed by the kinds of the page's views in order: changes made on another layout are not applied, and the page says so. An untouched page carries nothing. Comparison keeps its cohorts in ?doc= without the views; a ?doc= link with views still opens. Measured on the 92 gallery documents: about half the plain length overall (largest 2,598 -> 568 characters). On the Results page, hiding Missense on the lollipop is a 225-character ?changes=, and a reload shows the same selection. Verified: unit tests (new test/url-json.test.ts, test/page-changes.test.ts; a pinned format-1 link is read by Node's own zlib), lint, and the e2e lane of scripts/gate.sh (185 pass), which now asserts the Results page writes ?changes= and no ?doc=, and decodes the lab's ?doc= with the same module. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TyVfAhLnjshk2kWewGnLum --- app/_agent/PageTools.tsx | 3 +- app/campaign/[id]/page.tsx | 4 +-- app/lab/page.tsx | 6 ++-- app/lab/use-doc-history.ts | 6 ++-- app/lab/use-library.ts | 6 ++-- app/portal/comparison/page.tsx | 7 +++-- app/portal/page-changes.ts | 50 +++++++++++++++++++++++++++++++ app/portal/page-doc.ts | 53 ++++++++++++++++++++------------- app/portal/results/page.tsx | 3 +- app/reproductions/page.tsx | 5 ++-- docs/ARCHITECTURE.md | 7 +++++ docs/dsl.md | 3 +- e2e-lab-races.cjs | 9 +++--- e2e.cjs | 9 ++++-- lib/dsl-reference.ts | 3 +- lib/lab-links.ts | 19 ++---------- lib/url-json.ts | 49 ++++++++++++++++++++++++++++++ mcp/tools.ts | 3 +- package-lock.json | 7 +++++ package.json | 1 + test/page-changes.test.ts | 54 ++++++++++++++++++++++++++++++++++ test/url-json.test.ts | 45 ++++++++++++++++++++++++++++ 22 files changed, 287 insertions(+), 65 deletions(-) create mode 100644 app/portal/page-changes.ts create mode 100644 lib/url-json.ts create mode 100644 test/page-changes.test.ts create mode 100644 test/url-json.test.ts diff --git a/app/_agent/PageTools.tsx b/app/_agent/PageTools.tsx index beec9b56..a5174b62 100644 --- a/app/_agent/PageTools.tsx +++ b/app/_agent/PageTools.tsx @@ -6,6 +6,7 @@ import { useEffect, useState } from "react"; import { exposeTools, type AgentTool } from "../../lib/webmcp.ts"; import { REPRODUCTIONS } from "../../lib/reproductions.ts"; import { encodeSpec } from "../../lib/lab-links.ts"; +import { encodeUrlJson } from "../../lib/url-json.ts"; import { AgentToolsMenu } from "./AgentToolsMenu.tsx"; type Kind = "reproductions" | "provenance" | "oncoprint-lab" | "rationale" | "datasets" | "verification"; @@ -33,7 +34,7 @@ function toolsFor(kind: Kind, spec: unknown): { tools: AgentTool[]; rows: [strin inputSchema: { type: "object", required: ["slug"], properties: { slug: { type: "string" } } }, run: async ({ slug }) => { const r = find(slug); - setTimeout(() => location.assign("/lab?doc=" + encodeSpec(r.doc)), 300); + setTimeout(() => location.assign("/lab?doc=" + encodeUrlJson(r.doc)), 300); return `Opening "${r.title}" in /lab. Once it loads, use analysis_get_results there.`; } } ]; diff --git a/app/campaign/[id]/page.tsx b/app/campaign/[id]/page.tsx index e43c8340..9424230c 100644 --- a/app/campaign/[id]/page.tsx +++ b/app/campaign/[id]/page.tsx @@ -5,7 +5,7 @@ // there and in the database, so this page cannot skip one even by accident. import { useCallback, useEffect, useState } from "react"; import { useParams } from "next/navigation"; -import { encodeSpec } from "../../../lib/lab-links.ts"; +import { encodeUrlJson } from "../../../lib/url-json.ts"; import { withSplit, primaryCohorts, type Hypothesis } from "../../../lib/hypothesis.ts"; import type { Campaign, Funnel, HypothesisRow } from "../../../lib/campaign.ts"; import { installAgentTools } from "../agent-tools.ts"; @@ -24,7 +24,7 @@ type View = { campaign: Campaign; funnel: Funnel; prior: Record const STEPS = ["Ask", "Propose", "Review", "Lock", "Test", "Report"] as const; const fmtP = (p?: number | null) => p == null ? "–" : p < 1e-3 ? p.toExponential(1) : p.toFixed(3); const labLink = (h: HypothesisRow, part?: "discovery" | "validation") => - "/lab?doc=" + encodeSpec(part ? withSplit(h.spec.doc, part) : h.spec.doc); + "/lab?doc=" + encodeUrlJson(part ? withSplit(h.spec.doc, part) : h.spec.doc); const asHyp = (h: HypothesisRow): Hypothesis => ({ statement: h.statement, ...h.spec }); function testLabel(h: HypothesisRow): string { diff --git a/app/lab/page.tsx b/app/lab/page.tsx index cc078bda..cee67336 100644 --- a/app/lab/page.tsx +++ b/app/lab/page.tsx @@ -5,6 +5,7 @@ import { ExploreCharts } from "./explore-charts"; import type { UnpackedContract } from "../../lib/oncoprint-binary.ts"; import type { AnalysisDoc, CohortDef } from "../../lib/types.ts"; import { storeCohortLists, storeDocLists } from "./id-sets-client.ts"; +import { decodeUrlJson } from "../../lib/url-json.ts"; import type { SignInState } from "./list-cohort.tsx"; import { useLabTools, LAB_TOOL_NAMES } from "./agent-tools.ts"; import { AgentToolsMenu } from "../_agent/AgentToolsMenu.tsx"; @@ -310,7 +311,7 @@ export default function AnalysisPage() { try { const b64 = openedWith.current; if (b64) { - const d = JSON.parse(new TextDecoder().decode(Uint8Array.from(atob(b64.replace(/-/g, "+").replace(/_/g, "/")), ch => ch.charCodeAt(0)))); + const d = decodeUrlJson(b64); if (d.study) { setStudy(d.study); setExStudies([d.study]); exSel.current = [d.study]; } docs.replace(d, { undoable: false }); // the link the page opened: the first step, not a new one if (tabRef.current === "explore") fetchExplore(); @@ -341,8 +342,7 @@ export default function AnalysisPage() { try { const q = new URLSearchParams(window.location.search); q.set("tab", nextTab || tabRef.current); - if (d) q.set("doc", btoa(String.fromCharCode(...new TextEncoder().encode(JSON.stringify(d)))) - .replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "")); + if (d) q.set("doc", docParam(d)); docHistory.record("?" + q.toString(), push); } catch { /* URL state is a convenience, never a failure */ } }; diff --git a/app/lab/use-doc-history.ts b/app/lab/use-doc-history.ts index 0250d1e3..c900b5d5 100644 --- a/app/lab/use-doc-history.ts +++ b/app/lab/use-doc-history.ts @@ -12,6 +12,7 @@ import { useEffect, useRef, useState } from "react"; import type { AnalysisDoc } from "../../lib/types.ts"; import type { DocStore } from "./doc-store.ts"; +import { readUrlJson } from "../../lib/url-json.ts"; const KEY = "labDepth"; @@ -31,10 +32,7 @@ const depthOf = (state: unknown): number | null => { /** `?doc=` → the document it holds; null when there is none or it does not decode */ export function decodeDocParam(search: string): AnalysisDoc | null { - const b64 = new URLSearchParams(search).get("doc"); - if (!b64) return null; - try { return JSON.parse(new TextDecoder().decode(Uint8Array.from(atob(b64.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0)))); } - catch { return null; } + return readUrlJson(new URLSearchParams(search).get("doc")); } export function useDocHistory(docs: DocStore, emptyDoc: AnalysisDoc, currentTab: () => string): DocHistory { diff --git a/app/lab/use-library.ts b/app/lab/use-library.ts index c59f7e62..8964a5b4 100644 --- a/app/lab/use-library.ts +++ b/app/lab/use-library.ts @@ -8,6 +8,7 @@ import type { AnalysisDoc, TemplateDoc, CohortDef } from "../../lib/types.ts"; import type { DslExample } from "../../lib/dsl-reference.ts"; import { DOC_VERSION } from "../../lib/doc-version.ts"; import { LINE_TYPE } from "../../lib/treatment-lines.ts"; +import { encodeUrlJson } from "../../lib/url-json.ts"; import type { DocStore } from "./doc-store.ts"; import type { SavedCohort } from "./use-saved.ts"; @@ -15,10 +16,9 @@ import type { SavedCohort } from "./use-saved.ts"; const IMMUNO_EVENT: any = { kind: "event", a: { eventType: "Treatment", attrs: [{ key: "SUBTYPE", value: "Immuno" }] }, rel: { type: "NONE" } }; interface Example { name: string; desc: string; def: CohortDef } -/** A document as a ?doc= value: base64url of its UTF-8 JSON, as the page reads it back. */ +/** A document as a ?doc= value (lib/url-json.ts), as the page reads it back. */ export function docParam(d: AnalysisDoc | TemplateDoc): string { - return btoa(String.fromCharCode(...new TextEncoder().encode(JSON.stringify(d)))) - .replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); + return encodeUrlJson(d); } export const EXAMPLES: Example[] = [ diff --git a/app/portal/comparison/page.tsx b/app/portal/comparison/page.tsx index 026e9dc4..19d6dfd9 100644 --- a/app/portal/comparison/page.tsx +++ b/app/portal/comparison/page.tsx @@ -1,8 +1,8 @@ "use client"; // cBioPortal's Comparison page for 2–6 cohorts, laid out as the portal lays it out. The cohorts // arrive in an analysis document (?doc=, the lab's own link encoding); the page's views replace -// any the link carries unless they are this page's own (a link made on this page), so every -// setting changed here travels in the same ?doc= (page-doc.ts). +// any the link carries unless they are this page's own (a link made on this page). A setting +// changed here travels as ?changes= beside the cohorts (page-doc.ts). import { useMemo, useState } from "react"; import { useSearchParams } from "next/navigation"; import type { AnalysisDoc, View } from "../../../lib/types"; @@ -41,7 +41,8 @@ export default function ComparisonPage() { const names = Object.keys(linked?.cohorts || {}); const { known, error } = useStudyKnown(study); const fresh = useMemo(() => linked && known && names.length >= 2 && names.length <= 6 ? comparisonLayout(linked, study, known) : null, [linked, known, study, names.length]); - const page = usePageDoc(fresh, linked); + const input = useMemo(() => linked && { dsl: linked.dsl, study: linked.study, cohorts: linked.cohorts }, [linked]); + const page = usePageDoc(fresh, linked, input); const [counts, setCounts] = useState | null>(null); const shell = (body: React.ReactNode) => <>
{body}
; if (!linked || !study) return shell(

This page compares the cohorts of an analysis document passed as ?doc= (the lab's own link encoding); none was given.

); diff --git a/app/portal/page-changes.ts b/app/portal/page-changes.ts new file mode 100644 index 00000000..f430b784 --- /dev/null +++ b/app/portal/page-changes.ts @@ -0,0 +1,50 @@ +// What a portal page's link carries of its settings: only how its views differ from the page's +// defaults. The defaults are rebuilt from the link's query (?study=, ?genes=, ?filters=) or its +// cohorts, so a link with nothing changed carries nothing, and a changed setting costs its own +// bytes rather than every view's. +// +// The changes are tied to the page's layout — the kinds of its views, in order — by a short key. +// When the layout has changed since the link was made (a tab added, a view moved), an index would +// name another view, so the changes are not applied at all and the page says so. A default whose +// value changed under an unchanged layout is taken up: the link holds what the reader changed, not +// what they left alone. +import type { View } from "../../lib/types"; + +/** each changed view: its index, its settings that differ, and the default settings it no longer has */ +export type ViewChange = [index: number, set: Record, unset?: string[]]; +export interface PageChanges { layout: string; views: ViewChange[] } + +/** a short key for the kinds of a page's views, in order (FNV-1a) */ +export function layoutKey(views: View[]): string { + let h = 0x811c9dc5; + for (const ch of views.map((v) => v.kind).join(",")) h = Math.imul(h ^ ch.charCodeAt(0), 0x01000193) >>> 0; + return h.toString(36); +} + +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); + +/** how `now` differs from the page's defaults `fresh`; null when it does not */ +export function pageChanges(fresh: View[], now: View[]): PageChanges | null { + const views: ViewChange[] = []; + now.forEach((v, i) => { + const d = (fresh[i] || {}) as unknown as Record, n = v as unknown as Record; + const set = Object.fromEntries(Object.entries(n).filter(([k, x]) => !same(x, d[k]))); + const unset = Object.keys(d).filter((k) => !(k in n)); + if (Object.keys(set).length || unset.length) views.push(unset.length ? [i, set, unset] : [i, set]); + }); + return views.length ? { layout: layoutKey(fresh), views } : null; +} + +/** the page's views with a link's changes made to them; null when they were made on another layout */ +export function applyChanges(fresh: View[], c: PageChanges): View[] | null { + if (!c || c.layout !== layoutKey(fresh) || !Array.isArray(c.views)) return null; + const out = fresh.slice(); + for (const [i, set, unset] of c.views) { + if (!Number.isInteger(i) || !out[i] || !set || typeof set !== "object") return null; + const v = { ...out[i], ...set } as Record; + for (const k of unset || []) delete v[k]; + if (v.kind !== fresh[i].kind) return null; + out[i] = v as unknown as View; + } + return out; +} diff --git a/app/portal/page-doc.ts b/app/portal/page-doc.ts index a0b1add6..ff901e40 100644 --- a/app/portal/page-doc.ts +++ b/app/portal/page-doc.ts @@ -1,12 +1,15 @@ "use client"; // A portal page's state is one ordinary analysis document — the page's cohorts and ALL its views — -// carried in the URL as the lab carries its own (?doc=), plus the open tab (?tab=). The tabs are -// the page's: each lists the indices of the views it shows, and opening one sends only that slice -// as an analysis request, so the engine never sees a tab or more views than one run computes. -// A setting changed on the page is an edit to that view in this document, so a shared link -// reproduces it. +// plus the open tab (?tab=). The tabs are the page's: each lists the indices of the views it shows, +// and opening one sends only that slice as an analysis request, so the engine never sees a tab or +// more views than one run computes. A setting changed on the page is an edit to that view in this +// document, and the URL carries how the views differ from the page's defaults (?changes=, +// page-changes.ts), so a shared link reproduces it. A link made before that carries the whole +// document in ?doc=, and still opens. import { useEffect, useState } from "react"; import type { AnalysisDoc, View } from "../../lib/types"; +import { encodeUrlJson, readUrlJson } from "../../lib/url-json.ts"; +import { applyChanges, pageChanges, type PageChanges } from "./page-changes.ts"; /** * A control over a whole tab — the portal's gene chips over its Mutations tab: one choice that @@ -36,18 +39,12 @@ export interface PageLayout { left: string[]; } -/** any JSON value as a URL parameter (base64url) */ -export function encodeParam(x: unknown): string { - return btoa(String.fromCharCode(...new TextEncoder().encode(JSON.stringify(x)))).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); -} +/** any JSON value as a URL parameter (lib/url-json.ts) */ +export const encodeParam = (x: unknown): string => encodeUrlJson(x); /** a URL parameter written by encodeParam; null when there is none or it does not decode */ -export function decodeParam(b64: string | null): T | null { - if (!b64) return null; - try { return JSON.parse(new TextDecoder().decode(Uint8Array.from(atob(b64.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0)))); } - catch { return null; } -} +export const decodeParam = (s: string | null): T | null => readUrlJson(s); export const encodeDoc = (d: AnalysisDoc): string => encodeParam(d); -export const decodeDoc = (b64: string | null): AnalysisDoc | null => decodeParam(b64); +export const decodeDoc = (s: string | null): AnalysisDoc | null => decodeParam(s); /** * The document a link carries, if it is this page's: the same cohorts and the same kinds of view @@ -87,22 +84,38 @@ export function arrange(base: Omit, spec: TabSpec[], left: } +const param = (k: string): string | null => { try { return new URLSearchParams(window.location.search).get(k); } catch { return null; } }; + /** - * The page's document and open tab, kept in the URL: the link's document when it is this page's, - * else the page's own defaults (and a note saying the link's could not be used). + * The page's document and open tab, kept in the URL: the page's defaults with the link's changes + * made to them when they were made on this layout, else the link's whole document when it is this + * page's, else the defaults (and a note saying the link's could not be used). `input` is what of + * the link's ?doc= the page is built from (the comparison page's cohorts), kept in the URL without + * the views; a page built from its query alone drops ?doc= once it writes its changes. */ -export function usePageDoc(fresh: PageLayout | null, linked: AnalysisDoc | null) { +export function usePageDoc(fresh: PageLayout | null, linked: AnalysisDoc | null, input: AnalysisDoc | null = null) { const [doc, setDoc] = useState(null); const [note, setNote] = useState(null); - const [open, setOpen] = useState(() => { try { return new URLSearchParams(window.location.search).get("tab") || ""; } catch { return ""; } }); + const [open, setOpen] = useState(() => param("tab") || ""); + const [changes] = useState(() => readUrlJson(param("changes"))); const freshKey = fresh ? JSON.stringify(fresh.doc) : ""; useEffect(() => { if (!fresh) return; + if (changes) { + const views = applyChanges(fresh.doc.views || [], changes); + setDoc(views ? { ...fresh.doc, views } : fresh.doc); + setNote(views ? null : "the settings in this link were made on another layout of this page, so its views are shown with their defaults"); + return; + } const kept = compatible(linked, fresh.doc); setDoc(kept || fresh.doc); setNote(linked && linked.views && linked.views.length && !kept ? "the views in this link are not this page's own, so the page's are shown with their defaults" : null); }, [freshKey]); // eslint-disable-line react-hooks/exhaustive-deps - const update = (d: AnalysisDoc) => { setDoc(d); syncUrl({ doc: encodeDoc(d) }); }; + const update = (d: AnalysisDoc) => { + setDoc(d); + const c = fresh ? pageChanges(fresh.doc.views || [], d.views || []) : null; + syncUrl({ changes: c ? encodeUrlJson(c) : null, doc: input ? encodeDoc(input) : null }); + }; const openTab = (k: string) => { setOpen(k); syncUrl({ tab: k }); }; return { doc, note, open, update, openTab }; } diff --git a/app/portal/results/page.tsx b/app/portal/results/page.tsx index ee71b673..46e09283 100644 --- a/app/portal/results/page.tsx +++ b/app/portal/results/page.tsx @@ -4,7 +4,8 @@ // the genes with the profiled rest. What is // queried is one study (?study=) or several (?studies=a,b), narrowed by Explore's filters // (?filters=) or not; ?genes= is the query, as OQL. The page's state — every setting changed on -// it — is one analysis document in ?doc= (page-doc.ts), built by results-layout.ts. +// it — is one analysis document built by results-layout.ts, whose changes travel in ?changes= +// (page-doc.ts). import { useMemo, useState } from "react"; import { useSearchParams } from "next/navigation"; import { PortalTabs } from "../PortalTabs"; diff --git a/app/reproductions/page.tsx b/app/reproductions/page.tsx index 7ab1950f..4880e596 100644 --- a/app/reproductions/page.tsx +++ b/app/reproductions/page.tsx @@ -3,13 +3,14 @@ // list of real AnalysisDocs — nothing here queries the database itself, that happens once each // reader clicks through to /lab, exactly like the Examples tab's "copy link"/"Run" pattern. // -// Each card's "Open in Lab" link carries the FULL document as a ?doc= base64url blob (the same +// Each card's "Open in Lab" link carries the FULL document as a ?doc= value (lib/url-json.ts, the same // encoding /lab's own docLink() produces client-side) — the document's own `notes` and each // view's own `note` are what actually explain the paper connection, so it travels with the // analysis rather than living only on this listing page. import { REPRODUCTIONS, type PaperReproduction } from "../../lib/reproductions.ts"; import { PageTools } from "../_agent/PageTools.tsx"; import { SiteHeader } from "../_ui/site-header.tsx"; +import { encodeUrlJson } from "../../lib/url-json.ts"; export const metadata = { title: "Paper reproductions — cbio-lab" }; @@ -18,7 +19,7 @@ const BADGE_LABEL: Record = { }; function docLink(doc: unknown): string { - return "/lab?doc=" + Buffer.from(JSON.stringify(doc), "utf-8").toString("base64url"); + return "/lab?doc=" + encodeUrlJson(doc); } export default function ReproductionsPage() { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4580b198..22a9473b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -295,6 +295,13 @@ The map of the code: what each part does, where it lives, and the one place each `use-explore.ts` (keyed by everything that decides the answer, so a superseded request is cancelled and can never land; `e2e-lab-races.cjs` holds requests to prove it). The standalone query-builder (`/`) and interactive oncoprint (`/oncoprint`) demo pages were retired once the Analysis app subsumed them. +- **Links (`lib/url-json.ts`)** — a document in a URL (`?doc=`, and every JSON parameter) is written + `1.`: the JSON raw-deflated, about half the plain form's length on the gallery and less on + large documents, readable by any zlib. The plain base64url JSON of + older links, scripts and hand-made links still opens. The portal pages (`app/portal/`) carry only how their + views differ from the page's defaults (`?changes=`, `page-changes.ts`), tied to the kinds of the + page's views in order, so changes made on another layout are refused and said so rather than + applied to the wrong view; the defaults themselves are rebuilt from the query. - **The homepage (`/`, `app/_home/`)** — cBioPortal's: choose studies, then **Explore selected studies** (the lab's Explore, `/lab?tab=explore&study=a,b`) or **Query by gene** (the gene query dialog, whose Submit opens the Results page). One study chooser serves it, Explore's picker and diff --git a/docs/dsl.md b/docs/dsl.md index acc64e54..df22474a 100644 --- a/docs/dsl.md +++ b/docs/dsl.md @@ -24,7 +24,8 @@ the first time an analysis uses a server dataset. A `custom:` name with no pin i browser that holds it. Sent to `POST /api/analysis`, edited by the researcher's assistant through the page's WebMCP tools, and encoded in shareable -`/analysis?doc=` links. +`/analysis?doc=` links: the base64url of the document's UTF-8 JSON opens as it is, and the app writes it +deflated (`1.`, `lib/url-json.ts`) to keep links short. ## Cohort definitions diff --git a/e2e-lab-races.cjs b/e2e-lab-races.cjs index bc276706..e1dc8bf0 100644 --- a/e2e-lab-races.cjs +++ b/e2e-lab-races.cjs @@ -174,11 +174,10 @@ async function labRaces(browser, ok, BASE) { await settled(page); await page.locator(".lp-colname").first().click(); await page.waitForTimeout(300); - const fromUrl = await page.evaluate(() => { - const b64 = new URLSearchParams(location.search).get("doc") || ""; - const json = new TextDecoder().decode(Uint8Array.from(atob(b64.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0))); - return JSON.parse(json).views[0].hide || null; - }).catch((e) => "unreadable: " + e.message); + const { decodeUrlJson } = await import("./lib/url-json.ts"); + let fromUrl; + try { fromUrl = decodeUrlJson(new URL(page.url()).searchParams.get("doc") || "").views[0].hide || null; } + catch (e) { fromUrl = "unreadable: " + e.message; } const onPage = (await doc(page)).views[0].hide; ok("lab: a browser-only edit is in the URL at once", Array.isArray(fromUrl) && JSON.stringify(fromUrl) === JSON.stringify(onPage), "url " + JSON.stringify(fromUrl) + " · page " + JSON.stringify(onPage)); diff --git a/e2e.cjs b/e2e.cjs index ae787b26..c087b290 100644 --- a/e2e.cjs +++ b/e2e.cjs @@ -37,6 +37,7 @@ async function misalignedControls(p, scope) { } async function main() { + const { decodeUrlJson } = await import("./lib/url-json.ts"); const browser = await launch(); const results = []; const ok = (name, cond, extra) => @@ -783,7 +784,7 @@ async function main() { await page.locator(".undo-error").first().click(); await page.waitForSelector(".onco-legend", { timeout: 120000 }); const errLeft = await page.locator(".error").filter({ hasText: "NOT_A_PROFILE" }).count(); - const docBack = await page.evaluate(() => JSON.parse(atob((new URLSearchParams(location.search).get("doc") || "").replace(/-/g, "+").replace(/_/g, "/")))); + const docBack = decodeUrlJson(new URL(page.url()).searchParams.get("doc") || ""); undoOk = grew === 1 && errLeft === 0 && !JSON.stringify(docBack).includes("NOT_A_PROFILE"); undoWhat = "history +" + grew + ", error after undo: " + errLeft + ", document back: " + !JSON.stringify(docBack).includes("NOT_A_PROFILE"); } catch (e) { undoWhat = String(e.message).split("\n")[0]; } @@ -1386,7 +1387,7 @@ async function main() { views: [{ kind: "riskmodel", cohorts: ["NSCLC"], runs: 2 }] }; const b64 = Buffer.from(JSON.stringify(doc)).toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); const fig = page.locator("svg[aria-label^='Risk model']"); - const urlDoc = () => { const d = new URL(page.url()).searchParams.get("doc") || ""; return JSON.parse(Buffer.from(d.replace(/-/g, "+").replace(/_/g, "/"), "base64").toString("utf8")); }; + const urlDoc = () => decodeUrlJson(new URL(page.url()).searchParams.get("doc") || ""); try { await page.goto(BASE + "/lab?tab=analysis&doc=" + b64, { waitUntil: "domcontentloaded" }); await fig.waitFor({ timeout: 240000 }); @@ -1596,6 +1597,10 @@ async function main() { await settle(); const t1 = await tableCount(), chip = await page.locator(".dt-chips").innerText().catch(() => ""); ok("lollipop: on the Results page the table follows, through its Alteration filter", !!t1 && t1.kept < t0.kept && t1.all === t0.all && /Alteration/.test(chip), JSON.stringify(t0) + " -> " + JSON.stringify(t1)); + // the link carries how the views differ from the page's defaults, not the page's whole document + const linkQ = new URL(page.url()).searchParams, linkChanges = linkQ.get("changes") || ""; + ok("Results page: a changed setting travels as ?changes= alone", !linkQ.has("doc") && /^1\./.test(linkChanges) && linkChanges.length < 400, + "doc: " + linkQ.has("doc") + ", changes: " + linkChanges.length + " chars"); await page.reload({ waitUntil: "domcontentloaded" }); await page.locator(".lp-grid").first().waitFor({ timeout: 240000 }); await settle(); diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts index 370d678c..13335654 100644 --- a/lib/dsl-reference.ts +++ b/lib/dsl-reference.ts @@ -1572,7 +1572,8 @@ the first time an analysis uses a server dataset. A \`custom:\` name with no pin browser that holds it. Sent to \`POST /api/analysis\`, edited by the researcher's assistant through the page's WebMCP tools, and encoded in shareable -\`/analysis?doc=\` links. +\`/analysis?doc=\` links: the base64url of the document's UTF-8 JSON opens as it is, and the app writes it +deflated (\`1.\`, \`lib/url-json.ts\`) to keep links short. ## Cohort definitions diff --git a/lib/lab-links.ts b/lib/lab-links.ts index f88a7886..317782af 100644 --- a/lib/lab-links.ts +++ b/lib/lab-links.ts @@ -8,30 +8,17 @@ // URL and the cut is stated, in the link and on the page, rather than being silently applied. // // Isomorphic: the figure encodes in the browser, the lab pages decode on the server. +import { fromBase64Url, toBase64Url } from "./url-json.ts"; /** encoded characters; Node's HTTP parser caps the whole request line at 16 KB */ export const SPEC_URL_MAX = 8000; -// Plain base64 both sides, url-safe by hand: the browser's bundled Buffer shim is not Node's and -// does not know the "base64url" encoding, so asking it for one throws at run time. -const toUrl = (s: string) => s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); -const fromUrl = (s: string) => s.replace(/-/g, "+").replace(/_/g, "/"); - export function encodeSpec(spec: unknown): string { - const bytes = new TextEncoder().encode(JSON.stringify(spec)); - if (typeof btoa === "function") { - let s = ""; - for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); - return toUrl(btoa(s)); - } - return toUrl(Buffer.from(bytes).toString("base64")); + return toBase64Url(new TextEncoder().encode(JSON.stringify(spec))); } export function decodeSpec(encoded: string): unknown { - const b64 = fromUrl(encoded); - const binary = typeof atob === "function" ? atob(b64) : Buffer.from(b64, "base64").toString("binary"); - const bytes = Uint8Array.from(binary, (c) => c.charCodeAt(0)); - return JSON.parse(new TextDecoder().decode(bytes)); + return JSON.parse(new TextDecoder().decode(fromBase64Url(encoded))); } /** what the cut removed, in the reader's terms; absent means the link carries the whole spec */ diff --git a/lib/url-json.ts b/lib/url-json.ts new file mode 100644 index 00000000..4e04d7d0 --- /dev/null +++ b/lib/url-json.ts @@ -0,0 +1,49 @@ +// JSON in a URL parameter — the analysis document in ?doc=, a page's changes, a filter list — kept +// as short as it can be while still standing alone: a link carries the whole value, so it opens the +// same thing for anyone, signed in or not, with nothing stored on a server. +// +// Written as `.`. Format 1 is the JSON's UTF-8 bytes, raw deflate (RFC 1951, +// no dictionary), so any language's zlib reads it: Python's `zlib.decompress(data, -15)`. A value +// with no "." (never in base64url) is the plain form, the UTF-8 JSON itself in base64url — what +// links were before, and what a script or a person may still write by hand — and reads as well. +// +// Isomorphic and synchronous: the lab writes its URL on every edit, the server writes links into +// the pages it renders. +import { deflateSync, inflateSync, strToU8, strFromU8 } from "fflate"; + +// Plain base64 both sides, url-safe by hand: the browser's bundled Buffer shim is not Node's and +// does not know the "base64url" encoding, so asking it for one throws at run time. +export function toBase64Url(bytes: Uint8Array): string { + let b64: string; + if (typeof btoa === "function") { + let s = ""; + for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + b64 = btoa(s); + } else b64 = Buffer.from(bytes).toString("base64"); + return b64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +export function fromBase64Url(s: string): Uint8Array { + const b64 = s.replace(/-/g, "+").replace(/_/g, "/"); + const binary = typeof atob === "function" ? atob(b64) : Buffer.from(b64, "base64").toString("binary"); + return Uint8Array.from(binary, (c) => c.charCodeAt(0)); +} + +/** any JSON value as a URL parameter */ +export function encodeUrlJson(x: unknown): string { + return "1." + toBase64Url(deflateSync(strToU8(JSON.stringify(x)), { level: 9 })); +} + +/** a value written by encodeUrlJson, or in the original plain base64url form; throws when it does not decode */ +export function decodeUrlJson(s: string): T { + const dot = s.indexOf("."); + if (dot < 0) return JSON.parse(new TextDecoder().decode(fromBase64Url(s))); + if (s.slice(0, dot) !== "1") throw new Error("this link was written in a newer format (" + s.slice(0, dot) + ") than this page reads"); + return JSON.parse(strFromU8(inflateSync(fromBase64Url(s.slice(dot + 1))))); +} + +/** decodeUrlJson, or null when there is no value or it does not decode */ +export function readUrlJson(s: string | null | undefined): T | null { + if (!s) return null; + try { return decodeUrlJson(s); } catch { return null; } +} diff --git a/mcp/tools.ts b/mcp/tools.ts index 4e91ee43..264b1e87 100644 --- a/mcp/tools.ts +++ b/mcp/tools.ts @@ -27,6 +27,7 @@ import { connections, directConnections } from "../lib/ch.ts"; import { studiesFor } from "../lib/study-list.ts"; import { catalogFor } from "../lib/study-catalog.ts"; import type { AnalysisDoc, AnalysisResult, Contract } from "../lib/types.ts"; +import { encodeUrlJson } from "../lib/url-json.ts"; import type { OncoprintSpec } from "../lib/views/oncoprint/types.ts"; import { validateAxis, type PlotAxis, type PlotRequest, type AxisKind, type PointRank, type SparseCells, type Whiskers } from "../lib/plot.ts"; import { resolveAxes } from "../lib/plot-profile.ts"; @@ -592,7 +593,7 @@ export function registerTools(server: any, engine: ToolEngine = DIRECT): void { try { r = await engine.runAnalysis(document); } catch (e) { return text("REJECTED: " + String((e as Error)?.message || e) + "\nCall describe_dsl for the grammar and runnable examples."); } const base = process.env.APP_BASE_URL || "https://cohort-lab.vercel.app"; - const docB64 = Buffer.from(JSON.stringify(document)).toString("base64url"); + const docB64 = encodeUrlJson(document); const content: any[] = []; const out = analysisText(document, r); // FIGURES. lib/view-svg.ts renders every view kind the app draws, and the diff --git a/package-lock.json b/package-lock.json index d6368db4..34b36eed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "clsx": "^2.1.1", "d3-scale": "4.0.2", "express": "^5.2.1", + "fflate": "^0.8.2", "lucide-react": "^1.56.0", "marked": "^18.0.11", "mcp-handler": "^1.1.0", @@ -8546,6 +8547,12 @@ } } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/figures": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", diff --git a/package.json b/package.json index c1ff3921..50545eb8 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "clsx": "^2.1.1", "d3-scale": "4.0.2", "express": "^5.2.1", + "fflate": "^0.8.2", "lucide-react": "^1.56.0", "marked": "^18.0.11", "mcp-handler": "^1.1.0", diff --git a/test/page-changes.test.ts b/test/page-changes.test.ts new file mode 100644 index 00000000..4d8717c2 --- /dev/null +++ b/test/page-changes.test.ts @@ -0,0 +1,54 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { applyChanges, layoutKey, pageChanges } from "../app/portal/page-changes.ts"; +import type { View } from "../lib/types.ts"; + +const v = (x: Record) => x as unknown as View; +const FRESH = [v({ kind: "overlap", cohorts: ["A", "B"] }), v({ kind: "survival", cohorts: ["A", "B"], endpoint: "OS" }), + v({ kind: "enrichment", cohorts: ["A", "B"], alteration: "mutation" })]; + +test("an unchanged page carries nothing", () => { + assert.equal(pageChanges(FRESH, FRESH.map((x) => ({ ...x }))), null); +}); + +test("a changed setting carries only itself, and applies back to the same views", () => { + const now = FRESH.slice(); + now[1] = v({ ...FRESH[1], endpoint: "PFS", yMax: 0.5 }); + const c = pageChanges(FRESH, now)!; + assert.deepEqual(c.views, [[1, { endpoint: "PFS", yMax: 0.5 }]]); + assert.deepEqual(applyChanges(FRESH, JSON.parse(JSON.stringify(c))), now); +}); + +test("a default setting taken away is carried, and taken away again", () => { + const now = FRESH.slice(); + const { alteration: _dropped, ...rest } = FRESH[2] as unknown as Record; + now[2] = v(rest); + const c = pageChanges(FRESH, now)!; + assert.deepEqual(c.views, [[2, {}, ["alteration"]]]); + assert.deepEqual(applyChanges(FRESH, c), now); +}); + +test("a default that changed under the same layout is taken up", () => { + const now = FRESH.slice(); + now[1] = v({ ...FRESH[1], yMax: 0.5 }); + const c = pageChanges(FRESH, now)!; + const newer = FRESH.slice(); + newer[1] = v({ ...FRESH[1], endpoint: "DSS" }); + assert.deepEqual(applyChanges(newer, c)![1], { kind: "survival", cohorts: ["A", "B"], endpoint: "DSS", yMax: 0.5 }); +}); + +test("changes made on another layout are not applied", () => { + const now = FRESH.slice(); + now[1] = v({ ...FRESH[1], endpoint: "PFS" }); + const c = pageChanges(FRESH, now)!; + const moved = [FRESH[1], FRESH[0], FRESH[2]]; + assert.notEqual(layoutKey(moved), layoutKey(FRESH)); + assert.equal(applyChanges(moved, c), null); + assert.equal(applyChanges(FRESH.concat(v({ kind: "table" })), c), null); +}); + +test("a change that names no view, or would change a view's kind, is refused", () => { + const layout = layoutKey(FRESH); + assert.equal(applyChanges(FRESH, { layout, views: [[7, { endpoint: "PFS" }]] }), null); + assert.equal(applyChanges(FRESH, { layout, views: [[1, { kind: "table" }]] }), null); +}); diff --git a/test/url-json.test.ts b/test/url-json.test.ts new file mode 100644 index 00000000..b854197b --- /dev/null +++ b/test/url-json.test.ts @@ -0,0 +1,45 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { decodeUrlJson, encodeUrlJson, readUrlJson } from "../lib/url-json.ts"; +import { EXAMPLES } from "../lib/dsl-reference.ts"; +import { inflateRawSync } from "node:zlib"; + +const DOC = { dsl: "2.9", study: "luad_tcga", cohorts: { "EGFR-mutant": { all: [{ kind: "oql", q: "EGFR: MUT" }] }, "KRAS-mutant": { all: [{ kind: "oql", q: "KRAS: MUT" }] } }, + views: [{ kind: "survival", cohorts: ["EGFR-mutant", "KRAS-mutant"], endpoint: "OS" }] }; +const plain = (x: unknown) => Buffer.from(JSON.stringify(x)).toString("base64url"); + +test("a round trip returns exactly what went in, for every gallery example", () => { + for (const ex of EXAMPLES) assert.deepEqual(decodeUrlJson(encodeUrlJson(ex.doc)), ex.doc, ex.title); +}); + +test("the value survives a query string untouched", () => { + const s = encodeUrlJson({ note: "+/= and ÿÿÿ?&# and ünïcode" }); + assert.match(s, /^1\.[A-Za-z0-9_-]+$/); + assert.equal(encodeURIComponent(s), s); + assert.deepEqual(decodeUrlJson(s), { note: "+/= and ÿÿÿ?&# and ünïcode" }); +}); + +test("a link written before compression still opens", () => { + assert.deepEqual(decodeUrlJson(plain(DOC)), DOC); + assert.deepEqual(readUrlJson(plain({ note: "ünïcode" })), { note: "ünïcode" }); +}); + +test("format 1 is plain raw deflate: any zlib reads it, and a link written with it reads forever", () => { + const link = "1.q1ZKKc5RslIy0rNU0lEqLilNqQTyckoTU-JLktMTgWLJ-Rn5RSXFSlbVSq7ubkG6uaUliXklIG5iDlBndLVSdmZeClBTfmEOUHkhkAVSZ6XgGxqiVBtbq6PkHeQYTIw2kDq4NqC-sszUcqC9CKXFpUVlmWWJIPVwV0WjuArVslgdpdS8lIL8TJDFSv7BIHMB"; + assert.deepEqual(decodeUrlJson(link), DOC); + assert.deepEqual(JSON.parse(inflateRawSync(Buffer.from(link.slice(2), "base64url")).toString("utf8")), DOC); + assert.deepEqual(JSON.parse(inflateRawSync(Buffer.from(encodeUrlJson(DOC).slice(2), "base64url")).toString("utf8")), DOC); +}); + +test("a format this page does not know is refused by name, not misread", () => { + assert.throws(() => decodeUrlJson("9.abc"), /newer format \(9\)/); + assert.equal(readUrlJson("9.abc"), null); + assert.equal(readUrlJson("1.not-deflate"), null); + assert.equal(readUrlJson(null), null); +}); + +test("the gallery's documents are about half their plain size", () => { + let before = 0, after = 0; + for (const ex of EXAMPLES) { before += plain(ex.doc).length; after += encodeUrlJson(ex.doc).length; } + assert.ok(after < before * 0.55, after + " of " + before); +});