diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index cd1a0ed9..1fdb02fb 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -812,6 +812,66 @@ What is left, measured at 128-551 studies: definition, which every pooled query then evaluates: 0.9s → 3.0s at 32 studies with an OQL draft when forced. No study on the primary connection is that large; the answers are the same. +## 3e. OncoPrint and plots at scale (October 2026) + +Measured by calling the engine directly (no result cache) against the public warehouse, two or +three alternating runs each against the code before; the warehouse is shared, so single runs move +by up to 30% and only alternating pairs are compared. Every case listed returns a contract +identical to the code before's (sha256 of the contract without its timings, 13 specs: heatmaps, +VAF, a generic assay, Ward clustering, overlays, derived rows, merged and sample-level tracks, +cohort groups, five refusals), and both oncoprint views of the analysis route render identically. + +In the browser the figure is not the cost: at 409k patients the binary contract decodes in under +1 ms and draws in 45-107 ms (`/oncoprint-lab`, headless Chromium). + +| oncoprint | before | after | +|---|---|---| +| msk_impact_50k_2026, 48k patients, 10 genes | 1.4-1.9s | 1.3-1.5s (assembly 0.5-0.8s → 0.3-0.4s) | +| 5 largest studies, 166k patients | 3.8-5.0s | 2.5-2.7s (assembly 2.1-2.6s → 0.6-0.7s) | +| GENIE public + 5 largest, 409k patients | 10.8-13.9s | 7.5-8.2s (assembly 5.4-7.0s → 1.7-2.2s) | +| 50 studies, 5 genes | 5.6-6.0s | 3.4-3.8s | +| 312 studies, 242k patients, 5 genes | 14.7-17.4s (1,560 queries) | 4.5-5.6s | + +- **Assembly addressed every row by its column id**, a string-keyed hash lookup per row per + phase, on ids rebuilt as `study + "_" + pid` for every row of a multi-study figure (1.0-1.4 µs + each measured, against 0.33 for a plain id). `buildContract` now meets each id once, in a + per-study map, and hands `assembleIndexed` rows by column position; `assembleContract` keeps the + id-keyed interface (CLI, local tables, the Python conformance tests) as a thin adapter onto it. + The three hottest loops are functions of their own: inside one 400-line function V8 optimised + each loop on its own and threw the code away on reaching the next ("insufficient type feedback"). +- **Every study's fetches start together**: clinical tracks, heatmaps, overlays, derived rows and + the driver annotation no longer wait for the alterations, and the annotation reuses the cached + OncoKB release instead of asking for it again (126-232 ms) and reads only driver rows. +- **The whole-study fetches are one statement per batch of studies** (`batchStudies`): small + studies share a statement up to 25,000 samples, a larger study keeps its own. One statement per + connection was measured slower for six of the largest studies (query phase 6.2s against 5.1-5.7s) + — §3c's finding again: separate statements spread across the cluster. Each study's sample count, + for this and nothing else, is cached for the process's life. +- **Panel coverage** computes each panel's gene mask before joining samples, and joins only these + studies' samples: 195 → 86 ms at 48k patients, identical rows over 42 study × gene list × unit cases. + +Plots changed less. One study was already 0.35-0.85s at 52k samples. Across studies: + +- The pooled box plot fetched every study's points and then dropped them all when the total + passed the 10,000-point cap (11 studies, 150k samples: 11 queries for nothing). It now skips + them when the box's own count is past the cap, which is exact: a study's points are at least + its box's cases. That fetch was also, by accident, the only place every study's metadata was + checked against the axes; the check is now explicit and first, so a plot one study cannot answer + is refused in 0.7-1.2s instead of 3.2-6.2s at 312 studies. +- Coverage counts run beside the second round of queries instead of after it; a single-study + scatter asks for its coefficients beside its extent. +- The analysis loaded every study's profiles three queries at a time per study, and listed each + study's import date one query after another once the views were done: 312 queries, 3.1s, after + a 312-study plot had finished. Both are now one statement per connection (`studyProfilesMany`, + `studiesImported`). +- **What is left is the plot's own query count**: four per study, 1,248 at 312 studies, 11-25s as the shared database's load moves. + Sent as one `UNION ALL` request per kind (built, measured, not shipped) they were as slow as + sending them one after another — 11 statements 600-700 ms against 160-250 ms eight at a time. + Two levers remain, neither taken here: `CLICKHOUSE_MAX_INFLIGHT=16` took that view from 19.4s + to 12.2s and 32 gained nothing more (a deployment setting: it is everyone's share of the + database); or a plot statement that reads many studies in one scan, which means compiling an + axis — clinical, OQL, profile, generic assay — for several studies at once, against invariant 3. + ## 4. Time to display vs cbioportal.org Oncoprint, 5 genes, measured from navigation to the figure's pixels going stable diff --git a/lib/analysis.ts b/lib/analysis.ts index 5d477442..56546ca9 100644 --- a/lib/analysis.ts +++ b/lib/analysis.ts @@ -386,6 +386,8 @@ export async function runAnalysis(template: AnalysisDoc | TemplateDoc, opts: Run const evalCohortDef = async (def: CohortDef): Promise => { const studies = []; for (const s of cohortStudies(def, docStudy)) studies.push(await resolveStudy(s, docStudy)); + // every study's profiles at once, rather than three queries for each + await studyProfilesMany(studies); const perStudy: Record = {}; // Every instance is asked for a COUNT, including the patient-level ones. // Cohort evaluation has no use for identities: the size is a count, and the diff --git a/lib/annotation.ts b/lib/annotation.ts index 9758daf1..4745e531 100644 --- a/lib/annotation.ts +++ b/lib/annotation.ts @@ -86,18 +86,20 @@ async function syncedStudies(): Promise { return _syncedStudies; } -export async function buildAnnotationIndex(eventRows: EvRow[], genes: string[], study?: string | string[]): Promise { +/** + * `eventRows` may still be on its way: the store is read without them, and only the mock that + * stands in for a missing store waits for them. + */ +export async function buildAnnotationIndex(eventRows: EvRow[] | Promise, genes: string[], study?: string | string[]): Promise { try { - const pin = process.env.ONCOKB_PIN || - (await chQuery( - `SELECT oncokb_version FROM ${ANNOT_DB}.variant_annotation - GROUP BY oncokb_version ORDER BY max(resolved_at) DESC LIMIT 1`)).data[0]?.oncokb_version; + const pin = await oncokbPin(); if (!pin) throw new Error("annotation store is empty"); const rows = await chQuery( `SELECT alteration_type AS t, hugo_gene_symbol AS gene, alteration, max(is_driver) AS is_driver FROM ${ANNOT_DB}.variant_annotation WHERE oncokb_version = ${sq(pin)} AND hugo_gene_symbol IN (${genes.map(sq).join(",")}) - GROUP BY t, gene, alteration`); + GROUP BY t, gene, alteration + HAVING is_driver > 0`); // Nested by gene rather than one set of "type|gene|alteration" keys: isDriver runs once per // event — 800k times on a 400k-patient figure — and composing that key allocated a string // every time. Two lookups on short strings cost far less than one on a long fresh one. @@ -137,6 +139,6 @@ export async function buildAnnotationIndex(eventRows: EvRow[], genes: string[], // or availability problem, and nobody is waiting for the answer anyway. if (isAbortError(e)) throw e; // not licensed / store empty / db absent -> degrade loudly to the mock - return mockIndex(eventRows); + return mockIndex(await eventRows); } } diff --git a/lib/federation.ts b/lib/federation.ts index 5c00d3c4..e2a8a13f 100644 --- a/lib/federation.ts +++ b/lib/federation.ts @@ -1750,11 +1750,14 @@ abstract class BaseResolver implements AggregateCapable { const done = async (d: PlotData): Promise => { const c = await coverage; return c ? { ...d, coverage: c } : d; }; if (shape === "scatter") { + // the coefficients need no extent, so they are asked for beside it + const stP = this.runLogged("plot:stats", plotStatsSql(xSel, ySel, ctx.cohortSql), conn); + stP.catch(() => {}); const c = await this.runLogged("plot:extent", plotCountSql(xSel, ySel, ctx.cohortSql, true), conn); const n = Number(c.data[0]?.n || 0); if (!n) return done({ shape, n: 0, ...label, note: "no " + unit + " has a value on both axes" }); const [st, bins] = await Promise.all([ - this.runLogged("plot:stats", plotStatsSql(xSel, ySel, ctx.cohortSql), conn), + stP, this.runLogged("plot:bins", plotBinsSql(xSel, ySel, ctx.cohortSql, spec.bins), conn) ]); const r = st.data[0] || {}; diff --git a/lib/oncoprint-assemble.ts b/lib/oncoprint-assemble.ts index 9deb81b6..cf3575bd 100644 --- a/lib/oncoprint-assemble.ts +++ b/lib/oncoprint-assemble.ts @@ -153,36 +153,77 @@ export interface AssembleInput { clusterRank?: Map; } +/** What a driver call reads of an event: everything but the column it is in. */ +export type EventFacts = Omit; + +/** + * AssembleInput with every row addressed by its column's position in `columns` instead of by its + * column id. The database path builds this directly — it meets each id once, as the rows arrive, + * where a column id per row would cost a hash lookup per row per phase — and `assembleContract` + * turns any AssembleInput into it. Each per-column array has one entry per column, `undefined` + * where the column has no value. + */ +export interface IndexedInput extends Omit { + /** every column id, the groups' in order; each appears once */ + columns: string[]; + /** consecutive runs of `columns` */ + groups: { name: string; count: number }[]; + /** each column's index in `studies`, when there is more than one */ + columnStudy?: Int32Array; + events: (EventFacts & { col: number })[]; + coverage: ({ col: number; genes: string[] } | { col: number; mask: number })[]; + profiled?: { types: ProfiledType[]; masks: { col: number; mask: number }[] }; + samples?: { col: number; n: number }[]; + isDriver: (r: EventFacts) => boolean; + clinical: { attr: string; values: (string | undefined)[] }[]; + /** per gene (an entity, for an assay), its value in each column */ + heatmap: { profileType: string; genes: string[]; isVaf: boolean; assay?: boolean; values: Map }[]; + overlays?: { kind: "mrna" | "protein"; profileType: string; threshold: number; values: Map }[]; + /** per column, 1 when it is in the row's set */ + derived: { label: string; cols: Uint8Array }[]; + /** per column, its place in the engine's clustering */ + clusterRank?: number[]; +} + +/** An AssembleInput keyed by column id, addressed by position; a row whose column is not drawn is dropped. */ export function assembleContract(input: AssembleInput): Contract { - const t0 = Date.now(); - const PROF = !!process.env.ONCOPRINT_PROFILE; - let tp = Date.now(); - const phase = (name: string) => { if (PROF) { process.stderr.write(" " + name.padEnd(16) + (Date.now() - tp) + "ms\n"); tp = Date.now(); } }; - const { groups, studies, events, coverage, isDriver, clinical, heatmap, derived, sort, lineGap } = input; - // the rows, and the flat symbol list whose order the coverage mask bits follow - const { tracks: geneTracks, symbols: geneSymbolList } = normalizeGeneTracks(input.genes); - const unit = input.unit === "sample" ? "sample" as const : "patient" as const; - const multi = studies.length > 1; - const studyOf = input.studyOf || new Map(); - const allPids = groups.flatMap((g) => g.pids); - if (!allPids.length) throw new Error("cohort is empty"); - const pidIndex = new Map(allPids.map((p, i) => [p, i])); + const { groups, studyOf, events, coverage, profiled, samples, isDriver, clinical, heatmap, overlays, derived, clusterRank, ...rest } = input; + const columns = groups.flatMap((g) => g.pids); + const at = new Map(columns.map((p, i) => [p, i])); + const positioned = (rows: T[]): (Omit & { col: number })[] => { + const out: (Omit & { col: number })[] = []; + for (const { pid, ...r } of rows) { const col = at.get(pid); if (col !== undefined) out.push({ ...r, col }); } + return out; + }; + const perGene = (values: Map, genes: string[]) => + new Map(genes.map((g) => [g, columns.map((p) => values.get(p + "|" + g))])); + const studyIdx = new Map(input.studies.map((s, i) => [s, i])); + return assembleIndexed({ + ...rest, + columns, + groups: groups.map((g) => ({ name: g.name, count: g.pids.length })), + ...(input.studies.length > 1 ? { columnStudy: Int32Array.from(columns, (p) => { + const st = studyOf?.get(p); + return st == null ? -1 : studyIdx.get(st)!; + }) } : {}), + events: positioned(events), + coverage: positioned(coverage) as IndexedInput["coverage"], + profiled: profiled && { types: profiled.types, masks: positioned(profiled.masks) }, + samples: samples && positioned(samples), + isDriver: isDriver as (r: EventFacts) => boolean, + clinical: clinical.map(({ attr, values }) => ({ attr, values: columns.map((p) => values.get(p)) })), + heatmap: heatmap.map((h) => ({ ...h, values: perGene(h.values, h.genes) })), + overlays: overlays?.map((ov) => ({ ...ov, values: perGene(ov.values, normalizeGeneTracks(input.genes).symbols) })), + derived: derived.map((de) => ({ label: de.label, cols: Uint8Array.from(columns, (p) => de.pids.has(p) ? 1 : 0) })), + ...(clusterRank ? { clusterRank: columns.map((p) => clusterRank.get(p) ?? Number.MAX_SAFE_INTEGER) } : {}) + }); +} - // ---- gene-track cells -------------------------------------------------- - // A merged row is the bitwise OR of its members, which makes it altered where any member is - // altered and profiled where any member was profiled — the union denominator, with no - // special case: the sentinel survives only where every member is unprofiled. A gene may - // appear in several rows (merged, and again on its own), so one symbol maps to a LIST. - const geneCells = new Map(geneTracks.map((t) => [t.label, new Int32Array(allPids.length).fill(NOT_PROFILED)])); - const rowsOfGene = new Map(); - for (const t of geneTracks) for (const m of t.members) { - const list = rowsOfGene.get(m); - if (list) list.push(geneCells.get(t.label)!); else rowsOfGene.set(m, [geneCells.get(t.label)!]); - } - const rowsByMaskBit = geneSymbolList.map((g) => rowsOfGene.get(g) ?? []); +/** a column's covered genes become profiled-unaltered (0) in every row they draw in */ +function markProfiled(coverage: IndexedInput["coverage"], rowsByMaskBit: Int32Array[][], rowsOfGene: Map): void { for (const r of coverage) { - const i = pidIndex.get(r.pid); - if (i === undefined) continue; + const i = r.col; if ("mask" in r) { const m = r.mask; for (let gi = 0; gi < rowsByMaskBit.length; gi++) @@ -191,28 +232,15 @@ export function assembleContract(input: AssembleInput): Contract { for (const g of r.genes) for (const cells of rowsOfGene.get(g) || []) if (cells[i] === NOT_PROFILED) cells[i] = 0; } } - // Hiding unknown significance drops the ALTERATION, not the cell's bits: a cell holding a - // driver amplification and a passenger missense keeps only the amplification, and a merged - // row's driver in one gene does not smuggle another gene's passenger through. What was - // dropped is counted so a caption can say so. - // - // Driver status is per glyph, as in the portal: SV_DRIVER for the structural variant, - // MUT_DRIVER for the mutation band, and DRIVER for "some alteration here is a driver" (the - // driver %, the sort). The portal ranks every driver mutation above every VUS one whatever - // the class, so a VUS mutation's class is held aside and joins the cell only where no driver - // mutation does — a VUS truncation beside a driver missense shows as the driver missense. - // - // A call on a cell no panel covers — a gene its column's panels do not include (off-panel) — - // is not drawn: the cell stays NOT_PROFILED, and the gene's % does not count it, as cBioPortal's - // results view does (altered AND sequenced over sequenced). A merged row is profiled where any - // member is, so it can still draw a call its gene's own row leaves out. Every call left out of - // a row is counted, so the caption says so. - const hideVus = !!input.display?.hideVUS; +} + +/** each event's bits into the rows its gene draws in, where a panel covers the cell; what was left out, counted */ +function placeEvents(events: IndexedInput["events"], rowsOfGene: Map, isDriver: IndexedInput["isDriver"], + hideVus: boolean): { hiddenVus: number; offPanel: number } { let hiddenVus = 0, offPanel = 0; const vusMut = new Map(); // mutation class bits all sit below 1024 for (const r of events) { - const i = pidIndex.get(r.pid); - if (i === undefined) continue; + const i = r.col; const rows = rowsOfGene.get(r.gene); if (!rows) continue; let bit = 0; @@ -239,6 +267,102 @@ export function assembleContract(input: AssembleInput): Contract { } for (const [cells, held] of vusMut) for (let i = 0; i < cells.length; i++) if (held[i] && !(cells[i] & BITS.MUT_DRIVER)) cells[i] |= held[i]; + return { hiddenVus, offPanel }; +} + +/** one clinical attribute's track over the columns: numeric bars, or up to eight categories and Other */ +function clinicalTrack(attr: string, val: (string | undefined)[], n: number): Track { + // numeric attribute (≥90% of present values parse, >10 distinct) -> continuous + // heat track: cells are the value scaled to 0..255 over the cohort's min..max + // one pass over the columns: how many have a value, how many parse, their range and + // distinct count. Never materialises a per-column array (a spread of 400k values throws). + let nPresent = 0, nNumeric = 0, min = Infinity, max = -Infinity; + const distinct = new Set(); + for (let i = 0; i < val.length; i++) { + const v = val[i]; + if (v == null) continue; + nPresent++; + const x = Number(v); + if (!Number.isFinite(x)) continue; + nNumeric++; + if (x < min) min = x; + if (x > max) max = x; + if (distinct.size <= 11) distinct.add(x); + } + if (!nPresent) { // unknown attribute or none of these patients have it — say so + return { kind: "clinical", label: attr + " (no data)", categories: [], cells: new Int32Array(n).fill(-1) }; + } + if (nNumeric >= 0.9 * nPresent && distinct.size > 10) { + // drawn as bars over the data's range, as the portal does; its fixed range for a fraction + // of the genome, and its log scale for mutation counts + const fraction = attr === "FRACTION_GENOME_ALTERED"; + if (fraction) { min = Math.min(min, 0); max = Math.max(max, 1); } + const logScale = attr === "MUTATION_COUNT" && min >= 0; + const cells = new Int32Array(n).fill(-1); + for (let i = 0; i < val.length; i++) { + const v = val[i]; + if (v == null) continue; + const x = Number(v); + if (Number.isFinite(x)) cells[i] = encodeNumeric(x, min, max, logScale); + } + return { kind: "clinical", numeric: true, bar: true, ...(logScale ? { logScale } : {}), label: attr, min, max, cells }; + } + const counts = new Map(); + for (let i = 0; i < val.length; i++) { const v = val[i]; if (v != null) counts.set(v, (counts.get(v) || 0) + 1); } + const categories = [...counts.entries()].sort((x, y) => y[1] - x[1]).slice(0, 8).map((e) => e[0]); + const catIdx = new Map(categories.map((c, i) => [c, i])); + const cells = new Int32Array(n).fill(-1); + for (let i = 0; i < val.length; i++) { + const v = val[i]; + if (v != null) { const ci = catIdx.get(v); cells[i] = ci === undefined ? categories.length : ci; } // last = Other + } + const hasOther = [...counts.keys()].some((v) => !catIdx.has(v)); + return { kind: "clinical", label: attr, categories: hasOther ? [...categories, "Other"] : categories, cells }; +} + +export function assembleIndexed(input: IndexedInput): Contract { + const t0 = Date.now(); + const PROF = !!process.env.ONCOPRINT_PROFILE; + let tp = Date.now(); + const phase = (name: string) => { if (PROF) { process.stderr.write(" " + name.padEnd(16) + (Date.now() - tp) + "ms\n"); tp = Date.now(); } }; + const { groups, studies, events, coverage, isDriver, clinical, heatmap, derived, sort, lineGap } = input; + // the rows, and the flat symbol list whose order the coverage mask bits follow + const { tracks: geneTracks, symbols: geneSymbolList } = normalizeGeneTracks(input.genes); + const unit = input.unit === "sample" ? "sample" as const : "patient" as const; + const multi = studies.length > 1; + const allPids = input.columns; + if (!allPids.length) throw new Error("cohort is empty"); + + // ---- gene-track cells -------------------------------------------------- + // A merged row is the bitwise OR of its members, which makes it altered where any member is + // altered and profiled where any member was profiled — the union denominator, with no + // special case: the sentinel survives only where every member is unprofiled. A gene may + // appear in several rows (merged, and again on its own), so one symbol maps to a LIST. + const geneCells = new Map(geneTracks.map((t) => [t.label, new Int32Array(allPids.length).fill(NOT_PROFILED)])); + const rowsOfGene = new Map(); + for (const t of geneTracks) for (const m of t.members) { + const list = rowsOfGene.get(m); + if (list) list.push(geneCells.get(t.label)!); else rowsOfGene.set(m, [geneCells.get(t.label)!]); + } + const rowsByMaskBit = geneSymbolList.map((g) => rowsOfGene.get(g) ?? []); + markProfiled(coverage, rowsByMaskBit, rowsOfGene); + // Hiding unknown significance drops the ALTERATION, not the cell's bits: a cell holding a + // driver amplification and a passenger missense keeps only the amplification, and a merged + // row's driver in one gene does not smuggle another gene's passenger through. What was + // dropped is counted so a caption can say so. + // + // Driver status is per glyph, as in the portal: SV_DRIVER for the structural variant, + // MUT_DRIVER for the mutation band, and DRIVER for "some alteration here is a driver" (the + // driver %, the sort). The portal ranks every driver mutation above every VUS one whatever + // the class, so a VUS mutation's class is held aside and joins the cell only where no driver + // mutation does — a VUS truncation beside a driver missense shows as the driver missense. + // + // A call on a cell no panel covers — a gene its column's panels do not include (off-panel) — + // is not drawn: the cell stays NOT_PROFILED, and the gene's % does not count it, as cBioPortal's + // results view does (altered AND sequenced over sequenced). A merged row is profiled where any + // member is, so it can still draw a call its gene's own row leaves out. Every call left out of + // a row is counted, so the caption says so. + const { hiddenVus, offPanel } = placeEvents(events, rowsOfGene, isDriver, !!input.display?.hideVUS); phase("gene cells"); @@ -249,7 +373,7 @@ export function assembleContract(input: AssembleInput): Contract { for (const t of geneTracks) { const cells = geneCells.get(t.label)!; for (const gene of t.members) for (let i = 0; i < allPids.length; i++) { - const v = ov.values.get(allPids[i] + "|" + gene); + const v = ov.values.get(gene)?.[i]; if (v === undefined) continue; if (v >= ov.threshold) cells[i] = (cells[i] === NOT_PROFILED ? 0 : cells[i]) | hi; else if (v <= -ov.threshold) cells[i] = (cells[i] === NOT_PROFILED ? 0 : cells[i]) | lo; @@ -262,14 +386,12 @@ export function assembleContract(input: AssembleInput): Contract { if (multi) { // synthetic per-column provenance track; also the honest caveat that clinical // vocabularies below it are merged by attribute name, not harmonized - const catIdx = new Map(studies.map((s, i) => [s, i])); - const cells = new Int32Array(allPids.length).fill(-1); - allPids.forEach((p, i) => { const s = studyOf.get(p); if (s != null) cells[i] = catIdx.get(s)!; }); + const cells = input.columnStudy ? Int32Array.from(input.columnStudy) : new Int32Array(allPids.length).fill(-1); clinTracks.push({ kind: "clinical", label: STUDY_TRACK, categories: studies.slice(), cells }); } if (input.samples && unit === "patient") { const n = new Int32Array(allPids.length); - for (const r of input.samples) { const i = pidIndex.get(r.pid); if (i !== undefined) n[i] = r.n; } + for (const r of input.samples) n[r.col] = r.n; let max = 0; for (let i = 0; i < n.length; i++) if (n[i] > max) max = n[i]; if (max > 1) { @@ -280,7 +402,7 @@ export function assembleContract(input: AssembleInput): Contract { } if (input.profiled) { const mask = new Int32Array(allPids.length); - for (const r of input.profiled.masks) { const i = pidIndex.get(r.pid); if (i !== undefined) mask[i] |= r.mask; } + for (const r of input.profiled.masks) mask[r.col] |= r.mask; const rows: Track[] = []; for (const type of input.profiled.types) { const bit = 1 << PROFILED_TYPES.findIndex(([t]) => t === type); @@ -292,61 +414,7 @@ export function assembleContract(input: AssembleInput): Contract { } clinTracks.push(...rows.sort((a, b) => a.label.localeCompare(b.label, "en", { numeric: true }))); } - for (const { attr, values: valByPid } of clinical) { - // One string-keyed lookup per column, then three passes over a positional array. The map - // is keyed by column id, and at 400k columns three passes over it was the single most - // expensive phase of assembly. - const val: (string | undefined)[] = new Array(allPids.length); - for (let i = 0; i < allPids.length; i++) val[i] = valByPid.get(allPids[i]); - // numeric attribute (≥90% of present values parse, >10 distinct) -> continuous - // heat track: cells are the value scaled to 0..255 over the cohort's min..max - // one pass over the columns: how many have a value, how many parse, their range and - // distinct count. Never materialises a per-column array (a spread of 400k values throws). - let nPresent = 0, nNumeric = 0, min = Infinity, max = -Infinity; - const distinct = new Set(); - for (let i = 0; i < val.length; i++) { - const v = val[i]; - if (v == null) continue; - nPresent++; - const x = Number(v); - if (!Number.isFinite(x)) continue; - nNumeric++; - if (x < min) min = x; - if (x > max) max = x; - if (distinct.size <= 11) distinct.add(x); - } - if (!nPresent) { // unknown attribute or none of these patients have it — say so - clinTracks.push({ kind: "clinical", label: attr + " (no data)", categories: [], cells: new Int32Array(allPids.length).fill(-1) }); - continue; - } - if (nNumeric >= 0.9 * nPresent && distinct.size > 10) { - // drawn as bars over the data's range, as the portal does; its fixed range for a fraction - // of the genome, and its log scale for mutation counts - const fraction = attr === "FRACTION_GENOME_ALTERED"; - if (fraction) { min = Math.min(min, 0); max = Math.max(max, 1); } - const logScale = attr === "MUTATION_COUNT" && min >= 0; - const cells = new Int32Array(allPids.length).fill(-1); - for (let i = 0; i < val.length; i++) { - const v = val[i]; - if (v == null) continue; - const x = Number(v); - if (Number.isFinite(x)) cells[i] = encodeNumeric(x, min, max, logScale); - } - clinTracks.push({ kind: "clinical", numeric: true, bar: true, ...(logScale ? { logScale } : {}), label: attr, min, max, cells }); - continue; - } - const counts = new Map(); - for (let i = 0; i < val.length; i++) { const v = val[i]; if (v != null) counts.set(v, (counts.get(v) || 0) + 1); } - const categories = [...counts.entries()].sort((x, y) => y[1] - x[1]).slice(0, 8).map((e) => e[0]); - const catIdx = new Map(categories.map((c, i) => [c, i])); - const cells = new Int32Array(allPids.length).fill(-1); - for (let i = 0; i < val.length; i++) { - const v = val[i]; - if (v != null) { const ci = catIdx.get(v); cells[i] = ci === undefined ? categories.length : ci; } // last = Other - } - const hasOther = [...counts.keys()].some((v) => !catIdx.has(v)); - clinTracks.push({ kind: "clinical", label: attr, categories: hasOther ? [...categories, "Other"] : categories, cells }); - } + for (const { attr, values } of clinical) clinTracks.push(clinicalTrack(attr, values, allPids.length)); phase("clinical"); // ---- heatmap tracks (continuous molecular-profile values per gene) ---- @@ -364,8 +432,9 @@ export function assembleContract(input: AssembleInput): Contract { const ref = (input.zReference || {})[h.profileType + "|" + gene]; const usingRef = !!(ref && ref.sd > 0); const tx = (v: number) => usingRef ? (v - ref!.mean) / ref!.sd : v; + const byCol = h.values.get(gene) || []; const vals: number[] = []; - for (const p of allPids) { const v = h.values.get(p + "|" + gene); if (v !== undefined) vals.push(tx(v)); } + for (let i = 0; i < allPids.length; i++) { const v = byCol[i]; if (v !== undefined) vals.push(tx(v)); } if (!vals.length) { heatTracks.push({ kind: "clinical", label: label + " (no data)", categories: [], cells: new Int32Array(allPids.length).fill(-1) }); continue; @@ -380,10 +449,10 @@ export function assembleContract(input: AssembleInput): Contract { } const span = max - min || 1; const cells = new Int32Array(allPids.length).fill(-1); - allPids.forEach((p, i) => { - const v = h.values.get(p + "|" + gene); + for (let i = 0; i < allPids.length; i++) { + const v = byCol[i]; if (v !== undefined) cells[i] = Math.round(255 * (Math.max(min, Math.min(max, tx(v))) - min) / span); - }); + } heatTracks.push({ kind: "heatmap", numeric: true, label, gene, profileType: h.profileType, note: usingRef ? "z-scored against " + ref!.n.toLocaleString() + " patients pooled from " + ref!.studies.length + @@ -399,8 +468,7 @@ export function assembleContract(input: AssembleInput): Contract { // ---- derived-event tracks (cohort-DSL expressions as phenotypes) ------ const devTracks: Track[] = []; for (const de of derived) { - const cells = new Int32Array(allPids.length); - allPids.forEach((p, i) => { cells[i] = de.pids.has(p) ? 1 : 0; }); + const cells = Int32Array.from(de.cols); devTracks.push({ kind: "derivedEvent", label: String(de.label || "derived event").slice(0, 60), cells }); } @@ -463,8 +531,7 @@ export function assembleContract(input: AssembleInput): Contract { if (s.track === "cluster") { const rank = input.clusterRank; if (!rank) throw new Error("sort: 'cluster' needs a clustered profile — use \"cluster:\""); - const at = (i: number): number => rank.get(allPids[i]) ?? Number.MAX_SAFE_INTEGER; - return (x: number, y: number) => sign * (at(x) - at(y)); + return (x: number, y: number) => sign * (rank[x] - rank[y]); } const t = sortableTracks.get(s.track); if (!t) throw new Error("sort: '" + s.track + "' is not a track in this oncoprint — sortable: genomic, id, cluster:" + @@ -477,7 +544,7 @@ export function assembleContract(input: AssembleInput): Contract { let cursor = 0; const groupRanges: GroupRange[] = []; for (const g of groups) { - const idxs = g.pids.map((p) => pidIndex.get(p)!); + const idxs = Array.from({ length: g.count }, (_, k) => cursor + k); idxs.sort((x, y) => { for (const c of cmps) { const r = c(x, y); if (r) return r; } return idRank[x] - idRank[y]; @@ -500,9 +567,11 @@ export function assembleContract(input: AssembleInput): Contract { // Shared by the pooled numbers below and the per-group breakdown: same formula, just // over a slice of columns instead of all of them. const geneCounts = (slice: number[]) => { - const profiled = slice.filter((c) => c !== NOT_PROFILED).length; - const altered = slice.filter((c) => c > 0 && (c & ALT)).length; - const driverAltered = slice.filter((c) => c > 0 && (c & ALT) && (c & BITS.DRIVER)).length; + let profiled = 0, altered = 0, driverAltered = 0; + for (const c of slice) { + if (c !== NOT_PROFILED) profiled++; + if (c > 0 && (c & ALT)) { altered++; if (c & BITS.DRIVER) driverAltered++; } + } return { profiled, altered, driverAltered, freqPct: profiled ? Math.round(1000 * altered / profiled) / 10 : null, driverPct: profiled ? Math.round(1000 * driverAltered / profiled) / 10 : null }; @@ -535,7 +604,7 @@ export function assembleContract(input: AssembleInput): Contract { study: multi ? studies.join(" + ") : studies[0], studies, unit, groups: groupRanges, columns, - tracks: tracks.map((t) => ({ ...t, cells: Array.from(t.cells as number[]) })), + tracks, conventions: { cna: "high-level only (±2)", notProfiled: "distinct cell state", frequencyDenominator: unit === "sample" ? "profiled samples" : "profiled patients", column: unit === "sample" diff --git a/lib/oncoprint.ts b/lib/oncoprint.ts index f558b8ea..eb1dd8a5 100644 --- a/lib/oncoprint.ts +++ b/lib/oncoprint.ts @@ -4,7 +4,7 @@ // conventions: {lineGap} } // Contract: sorted columns, typed cells (bitmask for gene tracks, NOT_PROFILED = -1), // panel-aware track frequencies, group ranges. The renderer does zero data logic. -import { chQuery, chQueryRows, DB, STUDY, PRIMARY, type ChConn } from "./ch.ts"; +import { chQuery, chQueryRows, DB, STUDY, PRIMARY, type ChConn, type ChRows } from "./ch.ts"; import { compileCohort, sq } from "./compiler.ts"; import type { CohortDef, Contract, ChRow } from "./types.ts"; @@ -24,7 +24,7 @@ import { compileCohortDef } from "./cohort2.ts"; import { wardOrder, CLUSTER_LIMIT } from "./oncoprint-cluster.ts"; import { PROFILED_TYPES, type ProfiledType } from "./profile-names.ts"; -import { assembleContract, normalizeDisplay, normalizeSort, type AssembleInput, type EventRow } from "./oncoprint-assemble.ts"; +import { assembleIndexed, normalizeDisplay, normalizeSort, type EventFacts, type IndexedInput } from "./oncoprint-assemble.ts"; import { normalizeGeneTracks, toGeneList } from "./gene-tracks.ts"; import { clinicalTable, assertCustomDataPresent } from "./custom-data.ts"; import { finiteSql } from "./sql-number.ts"; @@ -64,10 +64,7 @@ export function cohortPatientsSql(dsl: PidsCarrier | null | undefined, study: st // DSL v2 definition ({all: [...]}) — the analysis document's cohort language, // including an explicit sample/patient list ({kind:"sampleList"|"patientList"}). if (dsl && Array.isArray((dsl as { all?: unknown }).all)) return compileCohortDef(dsl as CohortDef, { study, db, oncokbPin: pin }); - if (!dsl) { - return "SELECT DISTINCT replaceOne(patient_unique_id, " + sq(study + "_") + ", '') AS pid\n" + - "FROM " + db + ".sample_derived WHERE cancer_study_identifier = " + sq(study) + " ORDER BY pid"; - } + if (!dsl) return "SELECT pid FROM (\n" + allColumnsSql([study], db) + "\n) ORDER BY pid"; const { patientsSql } = compileCohort({ db, study, a: dsl.a!, b: dsl.b, rel: dsl.rel, priorGuard: !!dsl.priorGuard, lineGap: dsl.lineGap, patientsLimit: 1000000 @@ -91,17 +88,30 @@ async function cohortPatients(dsl: PidsCarrier | null | undefined, study: string * one that has to collapse — which is why sample mode removes joins rather than adding them. */ export type Unit = "patient" | "sample"; -/** the id column for a unit, stripped of the study prefix that qualifies it in the warehouse */ -const idCol = (unit: Unit, prefix: string, alias = ""): string => - "replaceOne(" + alias + (unit === "sample" ? "sample_unique_id" : "patient_unique_id") + ", " + sq(prefix) + ", '')"; +/** + * The id column for a unit, stripped of the prefix that qualifies it in the warehouse: each row's + * own study's, so one statement can read several studies. + */ +const idCol = (unit: Unit, alias = ""): string => + "replaceOne(" + alias + (unit === "sample" ? "sample_unique_id" : "patient_unique_id") + ", concat(" + alias + "cancer_study_identifier, '_'), '')"; +/** a study filter over a list: the builders below read every study of one database in one statement */ +const inStudies = (col: string, studies: string[]): string => col + " IN (" + studies.map(sq).join(",") + ")"; + +/** + * Every column of each study, `(s, pid)`, ordered by study then id: the columns when there is no + * cohort. A patient column is a patient with a sample; a sample column, each of their samples. + */ +export function allColumnsSql(studies: string[], db: string = DB, unit: Unit = "patient"): string { + return "SELECT DISTINCT cancer_study_identifier AS s, " + idCol(unit) + " AS pid\n" + + "FROM " + db + ".sample_derived WHERE " + inStudies("cancer_study_identifier", studies) + " ORDER BY s, pid"; +} -export function genomicEventsSql(genes: string[], study: string = STUDY, db: string = DB, unit: Unit = "patient"): string { - const puidPrefix = study + "_"; - return "SELECT " + idCol(unit, puidPrefix) + " AS pid,\n" + +export function genomicEventsSql(genes: string[], studies: string[] = [STUDY], db: string = DB, unit: Unit = "patient"): string { + return "SELECT cancer_study_identifier AS s, " + idCol(unit) + " AS pid,\n" + " hugo_gene_symbol AS gene, variant_type AS vt, mutation_type AS mt,\n" + " mutation_variant AS mv, cna_alteration AS cna\n" + "FROM " + db + ".genomic_event_derived\n" + - "WHERE cancer_study_identifier = " + sq(study) + " AND hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + + "WHERE " + inStudies("cancer_study_identifier", studies) + " AND hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")\n" + " AND mutation_status != 'UNCALLED'"; // read support, not a call (lib/oql.ts specCond) } @@ -112,17 +122,22 @@ export function genomicEventsSql(genes: string[], study: string = STUDY, db: str * the one the other views' denominators use). An integer per patient instead of an array of gene names — * at 400k patients that is the difference between a few megabytes and a few hundred kilobytes. */ -export function coverageSql(genes: string[], study: string = STUDY, db: string = DB, unit: Unit = "patient"): string { - const puidPrefix = study + "_"; +export function coverageSql(genes: string[], studies: string[] = [STUDY], db: string = DB, unit: Unit = "patient"): string { const list = "[" + genes.map(sq).join(",") + "]"; - // panels are recorded per sample; patient mode joins out to the patient, sample mode does not - return "SELECT " + (unit === "sample" ? idCol(unit, puidPrefix, "sp.") : idCol(unit, puidPrefix, "sd.")) + " AS pid,\n" + - " toUInt32(groupBitOr(bitShiftLeft(toUInt32(1), toUInt8(indexOf(" + list + ", pg.gene) - 1)))) AS mask\n" + + // Each panel's mask is computed once, from the few panels there are, before it meets the + // samples. Panels are recorded per sample; patient mode joins out to the patient (these + // studies' samples only), sample mode does not. + return "WITH pm AS (SELECT gene_panel_id, toUInt32(groupBitOr(bitShiftLeft(toUInt32(1), toUInt8(indexOf(" + list + ", gene) - 1)))) AS m\n" + + " FROM " + db + ".gene_panel_to_gene_derived WHERE gene IN (" + genes.map(sq).join(",") + ") GROUP BY gene_panel_id)\n" + + "SELECT sp.cancer_study_identifier AS s, " + idCol(unit, unit === "sample" ? "sp." : "sd.") + " AS pid,\n" + + " toUInt32(groupBitOr(pm.m)) AS mask\n" + "FROM " + db + ".sample_to_gene_panel_derived sp\n" + - "JOIN " + db + ".gene_panel_to_gene_derived pg ON pg.gene_panel_id = sp.gene_panel_id AND pg.gene IN (" + genes.map(sq).join(",") + ")\n" + - (unit === "sample" ? "" : "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = sp.sample_unique_id\n") + - "WHERE sp.cancer_study_identifier = " + sq(study) + " AND sp.alteration_type IN ('MUTATION_EXTENDED','COPY_NUMBER_ALTERATION','STRUCTURAL_VARIANT')\n" + - "GROUP BY pid"; + "JOIN pm ON pm.gene_panel_id = sp.gene_panel_id\n" + + (unit === "sample" ? "" : "JOIN (SELECT sample_unique_id, patient_unique_id, cancer_study_identifier FROM " + db + ".sample_derived WHERE " + + inStudies("cancer_study_identifier", studies) + ") sd\n" + + " ON sd.sample_unique_id = sp.sample_unique_id\n") + + "WHERE " + inStudies("sp.cancer_study_identifier", studies) + " AND sp.alteration_type IN ('MUTATION_EXTENDED','COPY_NUMBER_ALTERATION','STRUCTURAL_VARIANT')\n" + + "GROUP BY s, pid"; } /** @@ -130,40 +145,77 @@ export function coverageSql(genes: string[], study: string = STUDY, db: string = * some sample of the column is in a profile of that type whatever its genes — the portal's * "# Samples per Patient" and "Profiled for " rows, from one scan. */ -export function columnFactsSql(study: string = STUDY, db: string = DB, unit: Unit = "patient"): string { - const puidPrefix = study + "_"; +export function columnFactsSql(studies: string[] = [STUDY], db: string = DB, unit: Unit = "patient"): string { const types = PROFILED_TYPES.map(([t]) => sq(t)).join(","); - return "SELECT " + idCol(unit, puidPrefix, "sd.") + " AS pid,\n" + + return "SELECT sd.cancer_study_identifier AS s, " + idCol(unit, "sd.") + " AS pid,\n" + " uniqExact(sd.sample_unique_id) AS samples,\n" + " toUInt8(groupBitOr(if(sp.alteration_type = '', 0, bitShiftLeft(toUInt8(1), toUInt8(indexOf([" + types + "], sp.alteration_type) - 1))))) AS mask\n" + "FROM " + db + ".sample_derived sd\n" + "LEFT JOIN (SELECT sample_unique_id, alteration_type FROM " + db + ".sample_to_gene_panel_derived\n" + - " WHERE cancer_study_identifier = " + sq(study) + " AND alteration_type IN (" + types + ")) sp ON sp.sample_unique_id = sd.sample_unique_id\n" + - "WHERE sd.cancer_study_identifier = " + sq(study) + "\n" + - "GROUP BY pid"; + " WHERE " + inStudies("cancer_study_identifier", studies) + " AND alteration_type IN (" + types + ")) sp ON sp.sample_unique_id = sd.sample_unique_id\n" + + "WHERE " + inStudies("sd.cancer_study_identifier", studies) + "\n" + + "GROUP BY s, pid"; } -export function clinicalSql(attrs: string[], study: string = STUDY, db: string = DB, unit: Unit = "patient"): string { - const puidPrefix = study + "_"; +export function clinicalSql(attrs: string[], studies: string[] = [STUDY], db: string = DB, unit: Unit = "patient"): string { const inAttrs = "attribute_name IN (" + attrs.map(sq).join(",") + ") AND attribute_value != ''"; if (unit === "sample") // sample-level rows are exact; patient-level rows broadcast to each of that patient's // samples (a patient's stage is the stage of every sample of theirs) - return "SELECT replaceOne(sd.sample_unique_id, " + sq(puidPrefix) + ", '') AS pid,\n" + + return "SELECT sd.cancer_study_identifier AS s, " + idCol(unit, "sd.") + " AS pid,\n" + " cd.attribute_name AS k, argMax(cd.attribute_value, cd.type = 'sample') AS v\n" + "FROM " + clinicalTable(db, attrs) + " cd\n" + "JOIN " + db + ".sample_derived sd ON sd.patient_unique_id = cd.patient_unique_id\n" + " AND (cd.type = 'patient' OR cd.sample_unique_id = sd.sample_unique_id)\n" + - "WHERE cd.cancer_study_identifier = " + sq(study) + " AND cd." + inAttrs + "\n" + - "GROUP BY pid, k"; + "WHERE " + inStudies("cd.cancer_study_identifier", studies) + " AND cd." + inAttrs + "\n" + + "GROUP BY s, pid, k"; // patient-level rows win; sample-level attributes (e.g. CANCER_TYPE in MSK studies) // fall back to one of the patient's sample values via argMax - return "SELECT replaceOne(patient_unique_id, " + sq(puidPrefix) + ", '') AS pid,\n" + + return "SELECT cancer_study_identifier AS s, " + idCol(unit) + " AS pid,\n" + " attribute_name AS k, argMax(attribute_value, type = 'patient') AS v\n" + "FROM " + clinicalTable(db, attrs) + "\n" + - "WHERE cancer_study_identifier = " + sq(study) + "\n" + + "WHERE " + inStudies("cancer_study_identifier", studies) + "\n" + " AND " + inAttrs + "\n" + - "GROUP BY pid, k"; + "GROUP BY s, pid, k"; +} + +/** + * Each study's sample count, per connection: how the whole-study fetches are batched into + * statements, and nothing else. It never enters an answer, so a stale count costs only speed. + */ +const _sizeCache = new Map>>(); +function studySizes(conn: ChConn): Promise> { + const key = conn.id + "." + conn.db; + let p = _sizeCache.get(key); + if (!p) { + p = chQueryRows("SELECT cancer_study_identifier, count() FROM " + conn.db + ".sample_derived GROUP BY cancer_study_identifier", conn) + .then((r) => new Map(r.rows.map((x) => [x[0] as string, Number(x[1])]))); + _sizeCache.set(key, p); + p.catch(() => _sizeCache.delete(key)); + } + return p; +} +/** + * Samples one statement reads at most, unless a single study is larger. Many small studies share + * a statement — at 312 studies, one statement each is 1,560 queued queries — while a large study + * keeps its own, which the database runs beside the others: one statement over six of the largest + * was measured slower than six. + */ +const BATCH_SAMPLES = 25_000; +/** studies in batches of up to BATCH_SAMPLES samples, each batch on one connection */ +export function batchStudies(studies: string[], connId: (s: string) => string, size: (s: string) => number, limit = BATCH_SAMPLES): string[][] { + const out: string[][] = []; + const open = new Map(); + for (const s of studies) { + const k = connId(s), n = size(s); + if (n >= limit) { out.push([s]); continue; } + let b = open.get(k); + if (b && b.n + n > limit) { out.push(b.studies); b = undefined; } + if (!b) { b = { studies: [], n: 0 }; open.set(k, b); } + b.studies.push(s); b.n += n; + } + for (const b of open.values()) out.push(b.studies); + return out; } // per-study molecular-profile inventory (validation for heatmap tracks) @@ -298,12 +350,12 @@ export function specToQueries(spec: OncoprintSpec, pin?: string | null) { queries.push({ label: "column group \u201c" + (g.name || "group") + "\u201d \u2014 cohort " + unit + "s", sql: cohortColumnsSql({ ...g.dsl, lineGap }, study, DB, unit, pin) }); } else { queries.push({ label: spec.cohort ? "cohort " + unit + "s" : "all study " + unit + "s", - sql: cohortColumnsSql(spec.cohort ? { ...spec.cohort.dsl, lineGap } : null, study, DB, unit, pin) }); + sql: spec.cohort ? cohortColumnsSql({ ...spec.cohort.dsl, lineGap }, study, DB, unit, pin) : allColumnsSql([study], DB, unit) }); } - queries.push({ label: "genomic events (mutations, CNA, SV) for the gene tracks", sql: genomicEventsSql(genes, study, DB, unit) }); - queries.push({ label: "panel coverage (mutation, copy number and structural variant panels) \u2014 the NOT_PROFILED cell source and frequency denominators", sql: coverageSql(genes, study, DB, unit) }); - queries.push({ label: "samples per column and the alteration types it was profiled for \u2014 the \u201c# Samples per Patient\u201d and \u201cProfiled for\u201d rows", sql: columnFactsSql(study, DB, unit) }); - if (clinical.length) queries.push({ label: "clinical attribute tracks", sql: clinicalSql(clinical, study, DB, unit) }); + queries.push({ label: "genomic events (mutations, CNA, SV) for the gene tracks", sql: genomicEventsSql(genes, [study], DB, unit) }); + queries.push({ label: "panel coverage (mutation, copy number and structural variant panels) \u2014 the NOT_PROFILED cell source and frequency denominators", sql: coverageSql(genes, [study], DB, unit) }); + queries.push({ label: "samples per column and the alteration types it was profiled for \u2014 the \u201c# Samples per Patient\u201d and \u201cProfiled for\u201d rows", sql: columnFactsSql([study], DB, unit) }); + if (clinical.length) queries.push({ label: "clinical attribute tracks", sql: clinicalSql(clinical, [study], DB, unit) }); for (const de of derivedEvents) queries.push({ label: "derived-event track \u201c" + (de.label || "derived event") + "\u201d \u2014 cohort-DSL patients", sql: cohortColumnsSql({ ...de.expression, lineGap }, study, DB, unit, pin) }); return { genes, clinical, lineGap, queries }; @@ -324,10 +376,11 @@ export async function buildContract(spec: OncoprintSpec, opts: ContractOpts = {} // the four hot fetches read columns by index: no object per row, ~45% fewer bytes // A study's queries go to the instance that holds it, not merely to its database name: the // two differ the moment a peer lives on another host. - const lqRows = async (label: string, sql: string, study?: string) => { - const entry = { label, sql, ...(study ? { study } : {}) } as { label: string; sql: string; ms?: number; study?: string }; + const lqRows = async (label: string, sql: string, study?: string | string[]) => { + const where = Array.isArray(study) ? study.join(", ") : study; + const entry = { label, sql, ...(where ? { study: where } : {}) } as { label: string; sql: string; ms?: number; study?: string }; if (opts.log) opts.log.push(entry); - const r = await chQueryRows(sql, study ? connOf(study) : PRIMARY); + const r = await chQueryRows(sql, study ? connOf(Array.isArray(study) ? study[0] : study) : PRIMARY); entry.ms = r._ms; return r; }; @@ -361,95 +414,39 @@ export async function buildContract(spec: OncoprintSpec, opts: ContractOpts = {} if (unit === "sample" && derivedEvents.length) throw new Error("cohort rows are patient-level: a cohort selects patients, so drawing one per sample would repeat a patient's answer across their samples. Drop the rows or switch columns back to patients."); - // ---- columns: groups of patients -------------------------------------- - let groups: { name: string; pids: string[] }[]; // [{name, pids: [columnId]}] — a patient lands in its first matching group - const studyOf = new Map(); // columnId -> study (avoids prefix-ambiguity parsing) const gb = spec.layout && spec.layout.groupBy; - if (gb && gb.kind === "cohorts" && Array.isArray(gb.groups) && gb.groups.length) { - groups = []; - const seen = new Set(); - for (const g of gb.groups.slice(0, 4)) { - const v = (g.dsl as PidsCarrier | undefined)?.[PIDS_SQL]; - const raw: string[] = []; - if (v && typeof v === "object") { - // federated group: {study: sql} map from analysis.js — fetch per study in - // parallel, merge in study order (deterministic column order) - const fetched = await Promise.all(studies.filter((s) => typeof (v as Record)[s] === "string").map(async (s) => - ({ s, r: await lqRows("oncoprint:columns", "SELECT pid FROM (\n" + (v as Record)[s] + "\n)", s) }))); - for (const { s, r } of fetched) - for (const x of r.rows) { const id = qual(s, x[0] as string); raw.push(id); studyOf.set(id, s); } - } else { - // no pre-compiled SQL: compile the definition against each study in turn - const per = await Promise.all(studies.map(async (s) => - ({ s, pids: await cohortPatients({ ...g.dsl, lineGap }, s, opts.log, "oncoprint:columns", dbOf(s), unit, pin, connOf(s)) }))); - for (const { s, pids } of per) for (const p of pids) { const id = qual(s, p); raw.push(id); studyOf.set(id, s); } - } - const pids = raw.filter(p => !seen.has(p)); - pids.forEach(p => seen.add(p)); - groups.push({ name: g.name || "group " + (groups.length + 1), pids }); - } - } else if (multi) { - // no groups: every patient of every study, one column group, study order - if (spec.cohort) throw new Error("multi-study oncoprint with a cohort requires cohort groups (layout.groupBy)"); - const fetched = await Promise.all(studies.map(async (s) => - ({ s, r: await lqRows("oncoprint:columns", cohortColumnsSql(null, s, dbOf(s), unit, pin), s) }))); - const pids: string[] = []; - for (const { s, r } of fetched) for (const x of r.rows) { const id = qual(s, x[0] as string); pids.push(id); studyOf.set(id, s); } - groups = [{ name: "all patients", pids }]; - } else { - const pids = await cohortPatients(spec.cohort ? { ...spec.cohort.dsl, lineGap } : null, study, opts.log, "oncoprint:columns", dbOf(study), unit, pin, connOf(study)); - for (const p of pids) studyOf.set(p, study); - groups = [{ name: "cohort", pids }]; - } - const allPids = groups.flatMap(g => g.pids); - if (!allPids.length) throw new Error("cohort is empty"); + const grouped = !!(gb && gb.kind === "cohorts" && Array.isArray(gb.groups) && gb.groups.length); + if (multi && !grouped && spec.cohort) throw new Error("multi-study oncoprint with a cohort requires cohort groups (layout.groupBy)"); - // ---- genomic events + panel coverage, per study -------------------- - const events: EventRow[] = []; - const coverage: AssembleInput["coverage"] = []; - const profiledMasks: { pid: string; mask: number }[] = []; - const samplesPer: { pid: string; n: number }[] = []; - const evFetched = await Promise.all(studies.map(s => - Promise.all([lqRows("oncoprint:alterations", genomicEventsSql(genes, s, dbOf(s), unit), s), - lqRows("oncoprint:coverage", coverageSql(genes, s, dbOf(s), unit), s), - lqRows("oncoprint:profiledTypes", columnFactsSql(s, dbOf(s), unit), s)]).then(([e, c, p]) => ({ s, e, c, p })))); - for (const { s, e, c, p } of evFetched) { - const pref = multi ? s + "_" : ""; - // column order is the SELECT's: pid, gene, vt, mt, mv, cna - for (const r of e.rows) events.push({ pid: pref + (r[0] as string), gene: r[1] as string, vt: r[2] as string, mt: r[3] as string, mv: r[4] as string, cna: r[5] as string }); - for (const r of c.rows) coverage.push({ pid: pref + (r[0] as string), mask: Number(r[1]) }); - // column order is the SELECT's: pid, samples, mask - for (const r of p.rows) { - profiledMasks.push({ pid: pref + (r[0] as string), mask: Number(r[2]) }); - samplesPer.push({ pid: pref + (r[0] as string), n: Number(r[1]) }); - } - } - // the alteration types this figure draws: the gene cells' three, and an overlay's - const profiledTypes: ProfiledType[] = ["MUTATION_EXTENDED", "COPY_NUMBER_ALTERATION", "STRUCTURAL_VARIANT", - ...overlays.map((o) => (o.kind === "mrna" ? "MRNA_EXPRESSION" : "PROTEIN_LEVEL") as ProfiledType)]; + // Every fetch below reads one study's rows for the whole study and none waits on another's + // answer, so they all start here, beside the columns. Each is awaited before the contract is + // built; a figure refused on the way (an empty cohort) leaves the rest to settle unread. + const started = (p: Promise): Promise => { p.catch(() => {}); return p; }; + // A fetch that reads whole studies is one statement per batch of them (batchStudies), its rows + // led by their study (s, pid, ...). + const sizes = new Map(await Promise.all([...new Set(studies.map(connOf))].map(async (c) => [c.id, await studySizes(c)] as const))); + const dbStudies = batchStudies(studies, (s) => connOf(s).id, (s) => sizes.get(connOf(s).id)?.get(s) ?? 0); + const evFetchedP = started(Promise.all(dbStudies.map((ss) => + Promise.all([lqRows("oncoprint:alterations", genomicEventsSql(genes, ss, dbOf(ss[0]), unit), ss), + lqRows("oncoprint:coverage", coverageSql(genes, ss, dbOf(ss[0]), unit), ss), + lqRows("oncoprint:profiledTypes", columnFactsSql(ss, dbOf(ss[0]), unit), ss)]).then(([e, c, p]) => ({ e, c, p }))))); + // every event of each study, cohort or not: the rule-based stand-in for a missing annotation + // store counts recurrence over all of them. Each is given its column once the columns are known. + type Placed = EventFacts & { col: number }; + const eventsP = started(evFetchedP.then((fetched) => fetched.map(({ e }) => ({ rows: e.rows, + // column order is the SELECT's: s, pid, gene, vt, mt, mv, cna + events: e.rows.map((r): Placed => ({ col: -1, gene: r[2] as string, vt: r[3] as string, mt: r[4] as string, mv: r[5] as string, cna: r[6] as string })) })))); // "annotate: none" is a deliberate choice to draw without driver calls; say so rather than // letting the figure look as though nothing was annotated - const annotation = display.annotate === "none" - ? { isDriver: () => false, source: "none — driver annotation turned off for this figure" } - : await buildAnnotationIndex(events as Parameters[0], genes, multi ? studies : study); - - // ---- clinical values per attribute ------------------------------------- - const clinicalIn: AssembleInput["clinical"] = []; - if (clinical.length) { - const byAttr = new Map>(clinical.map((a) => [a, new Map()])); - const cdFetched = await Promise.all(studies.map(async (s) => - ({ s, r: await lqRows("oncoprint:clinicalTracks", clinicalSql(clinical, s, dbOf(s), unit), s) }))); - // rows are (pid, k, v); bucket straight into per-attribute maps, no intermediate array - for (const { s, r } of cdFetched) { - const pref = multi ? s + "_" : ""; - for (const x of r.rows) { const m = byAttr.get(x[1] as string); if (m) m.set(pref + (x[0] as string), x[2] as string); } - } - for (const attr of clinical) clinicalIn.push({ attr, values: byAttr.get(attr)! }); - } + const annotationP = started(display.annotate === "none" + ? Promise.resolve({ isDriver: () => false, source: "none — driver annotation turned off for this figure" }) + : buildAnnotationIndex(eventsP.then((per) => per.flatMap((x) => x.events)), genes, multi ? studies : study)); + const cdFetchedP = started(Promise.all((clinical.length ? dbStudies : []).map((ss) => + lqRows("oncoprint:clinicalTracks", clinicalSql(clinical, ss, dbOf(ss[0]), unit), ss)))); // ---- heatmap values per profile (validated against each study's inventory) ---- - const heatmapIn: AssembleInput["heatmap"] = []; - for (const h of heatmap) { + type Fetched = { s: string; r: ChRows }[]; + const heatmapRows = async (h: (typeof heatmap)[number]): Promise<{ profileType: string; genes: string[]; isVaf: boolean; assay?: boolean; fetched: Fetched }> => { const isVaf = /^vaf$/i.test(h.profileType); const assays = await Promise.all(studies.map(async (s) => (await studyAssayProfiles(s, dbOf(s), connOf(s))).get(h.profileType))); if (!isVaf && assays.some((a) => a)) { @@ -458,13 +455,9 @@ export async function buildContract(spec: OncoprintSpec, opts: ContractOpts = {} throw new Error("heatmap: '" + h.profileType + "' is categorical; a heatmap track draws numbers"); if (h.reference === "federation") throw new Error("heatmap: a federation reference z-scores a gene's values; '" + h.profileType + "' is a generic assay"); const okStudies = studies.filter((_, i) => assays[i]); - const values = new Map(); const fetched = await Promise.all(okStudies.map(async (s) => ({ s, r: await lqRows("oncoprint:assay heatmap", assayHeatmapSql(h.profileType, h.genes, s, dbOf(s), unit, !!assays[studies.indexOf(s)]!.patientLevel), s) }))); - for (const { s, r } of fetched) - for (const row of r.rows) values.set(qual(s, row[0] as string) + "|" + (row[1] as string), Number(row[2])); - heatmapIn.push({ profileType: h.profileType, genes: h.genes, isVaf: false, assay: true, values }); - continue; + return { profileType: h.profileType, genes: h.genes, isVaf: false, assay: true, fetched }; } if (h.assay) throw new Error("heatmap: entities name a generic assay's tracks; '" + h.profileType + "' is not a generic-assay profile of " + studies.join("/")); const avail = await Promise.all(studies.map(async (s) => @@ -477,69 +470,183 @@ export async function buildContract(spec: OncoprintSpec, opts: ContractOpts = {} " (available: " + ([...(await studyProfileTypes(studies[0], dbOf(studies[0]), connOf(studies[0])))].join(", ") || "none") + ", VAF" + [...(await studyAssayProfiles(studies[0], dbOf(studies[0]), connOf(studies[0]))).entries()].filter(([, a]) => !isCategoricalAssay(a.datatype)) .map(([pt]) => ", " + pt + " (generic assay: name its entities)").join("") + ")"); - const values = new Map(); // columnId|gene -> value - const hFetched = await Promise.all(okStudies.map(async (s) => + const fetched = await Promise.all(okStudies.map(async (s) => ({ s, r: await lqRows(isVaf ? "oncoprint:vaf" : "oncoprint:heatmap", isVaf ? vafSql(h.genes, s, dbOf(s), unit) : heatmapSql(h.profileType, h.genes, s, dbOf(s), unit), s) }))); - for (const { s, r } of hFetched) - for (const row of r.rows) values.set(qual(s, row[0] as string) + "|" + (row[1] as string), Number(row[2])); - heatmapIn.push({ profileType: h.profileType, genes: h.genes, isVaf, values }); + return { profileType: h.profileType, genes: h.genes, isVaf, fetched }; + }; + const heatmapP = started(Promise.allSettled(heatmap.map(heatmapRows))); + + // ---- molecular overlays: values thresholded into the gene cells --------- + const overlayRows = async (ov: (typeof overlays)[number]) => { + const avail = await Promise.all(studies.map(async (s) => (await studyProfileTypes(s, dbOf(s), connOf(s))).has(ov.profileType) ? s : null)); + const okStudies = avail.filter((s): s is string => s !== null); + if (!okStudies.length) + throw new Error("overlay: profile '" + ov.profileType + "' not in " + studies.join("/") + + " (available: " + ([...(await studyProfileTypes(studies[0], dbOf(studies[0]), connOf(studies[0])))].join(", ") || "none") + ")"); + const fetched: Fetched = await Promise.all(okStudies.map(async (s) => + ({ s, r: await lqRows("oncoprint:overlay " + ov.kind, heatmapSql(ov.profileType, genes, s, dbOf(s), unit), s) }))); + return { kind: ov.kind, profileType: ov.profileType, threshold: ov.threshold, fetched }; + }; + const overlaysP = started(Promise.allSettled(overlays.map(overlayRows))); + + // ---- derived-event rows (cohort-DSL expressions as phenotypes) --------- + const derivedP = started(Promise.allSettled(derivedEvents.map(async (de) => { + const derivedConn = conns.get(study) || await patientLevelConn(study, "derived-event rows list patients"); + return { label: de.label, pids: await cohortPatients({ ...de.expression, lineGap }, study, opts.log, "oncoprint:derivedEvent", derivedConn.db, unit, pin, derivedConn) }; + }))); + + // ---- columns: groups of patients -------------------------------------- + // Each group's patients as (study, id) pairs, in fetch order; a patient lands in the first group + // that has them. + let fetchedGroups: { name: string; members: { s: string; ids: string[] }[] }[]; + if (grouped) { + fetchedGroups = await Promise.all(gb!.groups!.slice(0, 4).map(async (g, gi) => { + const v = (g.dsl as PidsCarrier | undefined)?.[PIDS_SQL]; + const name = g.name || "group " + (gi + 1); + if (v && typeof v === "object") { + // federated group: {study: sql} map from analysis.js — fetch per study in + // parallel, merge in study order (deterministic column order) + const fetched = await Promise.all(studies.filter((s) => typeof (v as Record)[s] === "string").map(async (s) => + ({ s, r: await lqRows("oncoprint:columns", "SELECT pid FROM (\n" + (v as Record)[s] + "\n)", s) }))); + return { name, members: fetched.map(({ s, r }) => ({ s, ids: r.rows.map((x) => x[0] as string) })) }; + } + // no pre-compiled SQL: compile the definition against each study in turn + return { name, members: await Promise.all(studies.map(async (s) => + ({ s, ids: await cohortPatients({ ...g.dsl, lineGap }, s, opts.log, "oncoprint:columns", dbOf(s), unit, pin, connOf(s)) }))) }; + })); + } else if (!spec.cohort) { + // no cohort: every column of every study, one group, in study order + const idsOf = new Map(studies.map((s) => [s, [] as string[]])); + for (const r of await Promise.all(dbStudies.map((ss) => lqRows("oncoprint:columns", allColumnsSql(ss, dbOf(ss[0]), unit), ss)))) + for (const x of r.rows) idsOf.get(x[0] as string)?.push(x[1] as string); + fetchedGroups = [{ name: multi ? "all patients" : "cohort", members: studies.map((s) => ({ s, ids: idsOf.get(s)! })) }]; + } else { + fetchedGroups = [{ name: "cohort", members: [{ s: study, ids: await cohortPatients({ ...spec.cohort.dsl, lineGap }, study, opts.log, + "oncoprint:columns", dbOf(study), unit, pin, connOf(study)) }] }]; } + // each study's own ids -> column position: the one lookup a row from that study costs + const colOf = new Map(studies.map((s) => [s, new Map()])); + const studyIndex = new Map(studies.map((s, i) => [s, i])); + const columns: string[] = []; + const columnStudy: number[] = []; + const groups: IndexedInput["groups"] = []; + for (const { name, members } of fetchedGroups) { + const start = columns.length; + for (const { s, ids } of members) { + const at = colOf.get(s)!, si = studyIndex.get(s)!; + for (const id of ids) { + if (at.has(id)) continue; + at.set(id, columns.length); + columns.push(qual(s, id)); + columnStudy.push(si); + } + } + groups.push({ name, count: columns.length - start }); + } + if (!columns.length) throw new Error("cohort is empty"); + const n = columns.length; + + // ---- genomic events + panel coverage, per study -------------------- + const [evFetched, eventsPer, annotation, cdFetched] = await Promise.all([evFetchedP, eventsP, annotationP, cdFetchedP]); + // A row's column, from its study (column 0) and id (column 1); undefined when it is not drawn. + // A study's rows tend to arrive together, so its map is looked up once per run of them. + const locate = () => { + let lastS: unknown, at: Map | undefined; + return (r: unknown[]): number | undefined => { + if (r[0] !== lastS) { lastS = r[0]; at = colOf.get(r[0] as string); } + return at?.get(r[1] as string); + }; + }; + const events: Placed[] = []; + for (const { rows, events: evs } of eventsPer) { + const where = locate(); + for (let i = 0; i < evs.length; i++) { const col = where(rows[i]); if (col !== undefined) { evs[i].col = col; events.push(evs[i]); } } + } + const coverage: IndexedInput["coverage"] = []; + const profiledMasks: { col: number; mask: number }[] = []; + const samplesPer: { col: number; n: number }[] = []; + for (const { c, p } of evFetched) { + let where = locate(); + for (const r of c.rows) { const col = where(r); if (col !== undefined) coverage.push({ col, mask: Number(r[2]) }); } + // column order is the SELECT's: s, pid, samples, mask + where = locate(); + for (const r of p.rows) { + const col = where(r); + if (col === undefined) continue; + profiledMasks.push({ col, mask: Number(r[3]) }); + samplesPer.push({ col, n: Number(r[2]) }); + } + } + // the alteration types this figure draws: the gene cells' three, and an overlay's + const profiledTypes: ProfiledType[] = ["MUTATION_EXTENDED", "COPY_NUMBER_ALTERATION", "STRUCTURAL_VARIANT", + ...overlays.map((o) => (o.kind === "mrna" ? "MRNA_EXPRESSION" : "PROTEIN_LEVEL") as ProfiledType)]; + + // ---- clinical values per attribute ------------------------------------- + // rows are (s, pid, k, v), straight into each attribute's per-column array + const byAttr = new Map(clinical.map((a) => [a, new Array(n)])); + for (const r of cdFetched) { + const where = locate(); + for (const x of r.rows) { + const vals = byAttr.get(x[2] as string); + if (!vals) continue; + const col = where(x); + if (col !== undefined) vals[col] = x[3] as string; + } + } + const clinicalIn: IndexedInput["clinical"] = clinical.map((attr) => ({ attr, values: byAttr.get(attr)! })); + + // rows are (pid, gene, value): each gene's value per column + const perGene = (fetched: Fetched, genesOf: string[]): Map => { + const values = new Map(genesOf.map((g) => [g, new Array(n)])); + for (const { s, r } of fetched) { + const at = colOf.get(s)!; + for (const row of r.rows) { + const vals = values.get(row[1] as string); + if (!vals) continue; + const col = at.get(row[0] as string); + if (col !== undefined) vals[col] = Number(row[2]); + } + } + return values; + }; + // a track that could not be drawn fails the figure, the first in the spec's order + const inOrder = (rs: PromiseSettledResult[]): T[] => rs.map((r) => { if (r.status === "rejected") throw r.reason; return r.value; }); + const heatmapIn: IndexedInput["heatmap"] = inOrder(await heatmapP).map(({ fetched, ...h }) => ({ ...h, values: perGene(fetched, h.genes) })); // ---- clustering: an order computed here, once, from the molecular values ---- // "cluster:" orders columns by a Ward dendrogram over that profile's genes. // It runs where the values already are, so the SVG, the canvas, the CLI and Python all draw // the same order, and the order travels in the contract beside the numbers it describes. - let clusterRank: Map | undefined; + let clusterRank: number[] | undefined; const clusterKey = sort.find((k) => k.track.startsWith("cluster:")); if (clusterKey) { const profileType = clusterKey.track.slice("cluster:".length); const h = heatmapIn.find((x) => x.profileType === profileType); if (!h) throw new Error("sort: cluster needs '" + profileType + "' as a heatmap profile on this oncoprint" + (heatmapIn.length ? " (it has " + heatmapIn.map((x) => x.profileType).join(", ") + ")" : " (it has none)")); - if (allPids.length > CLUSTER_LIMIT) - throw new Error("clustering " + allPids.length.toLocaleString() + " columns is a quadratic computation; the limit is " + + if (n > CLUSTER_LIMIT) + throw new Error("clustering " + n.toLocaleString() + " columns is a quadratic computation; the limit is " + CLUSTER_LIMIT.toLocaleString() + ". Narrow the cohort, or sort by one of the profile's gene tracks instead."); - const columns = h.genes.map((g) => { - const col = new Float64Array(allPids.length); - for (let i = 0; i < allPids.length; i++) { - const v = h.values.get(allPids[i] + "|" + g); - col[i] = v === undefined ? NaN : v; - } - return col; - }); - const order = wardOrder({ columns, n: allPids.length }); - clusterRank = new Map(); - order.forEach((idx, rank) => clusterRank!.set(allPids[idx], rank)); + const byGene = h.genes.map((g) => Float64Array.from(h.values.get(g)!, (v) => v === undefined ? NaN : v)); + const order = wardOrder({ columns: byGene, n }); + clusterRank = new Array(n).fill(Number.MAX_SAFE_INTEGER); + order.forEach((idx, rank) => { clusterRank![idx] = rank; }); clusterKey.track = "cluster"; } - // ---- molecular overlays: values thresholded into the gene cells --------- - const overlaysIn: AssembleInput["overlays"] = []; - for (const ov of overlays) { - const avail = await Promise.all(studies.map(async (s) => (await studyProfileTypes(s, dbOf(s), connOf(s))).has(ov.profileType) ? s : null)); - const okStudies = avail.filter((s): s is string => s !== null); - if (!okStudies.length) - throw new Error("overlay: profile '" + ov.profileType + "' not in " + studies.join("/") + - " (available: " + ([...(await studyProfileTypes(studies[0], dbOf(studies[0]), connOf(studies[0])))].join(", ") || "none") + ")"); - const values = new Map(); - const fetched = await Promise.all(okStudies.map(async (s) => - ({ s, r: await lqRows("oncoprint:overlay " + ov.kind, heatmapSql(ov.profileType, genes, s, dbOf(s), unit), s) }))); - for (const { s, r } of fetched) - for (const row of r.rows) values.set(qual(s, row[0] as string) + "|" + (row[1] as string), Number(row[2])); - overlaysIn.push({ kind: ov.kind, profileType: ov.profileType, threshold: ov.threshold, values }); - } - - // ---- derived-event rows (cohort-DSL expressions as phenotypes) --------- - const derivedIn: AssembleInput["derived"] = []; - const derivedConn = derivedEvents.length ? conns.get(study) || await patientLevelConn(study, "derived-event rows list patients") : null; - for (const de of derivedEvents) - derivedIn.push({ label: de.label, pids: new Set(await cohortPatients({ ...de.expression, lineGap }, study, opts.log, "oncoprint:derivedEvent", derivedConn!.db, unit, pin, derivedConn!)) }); + const overlaysIn: IndexedInput["overlays"] = inOrder(await overlaysP).map(({ fetched, ...ov }) => ({ ...ov, values: perGene(fetched, genes) })); + const derivedIn: IndexedInput["derived"] = inOrder(await derivedP).map(({ label, pids }) => { + const cols = new Uint8Array(n), at = colOf.get(study)!; + for (const p of pids) { const col = at.get(p); if (col !== undefined) cols[col] = 1; } + return { label, cols }; + }); const queryMs = Date.now() - t0; const tAsm = Date.now(); - const contract = assembleContract({ - genes: geneList, groups, studies, studyOf, events, coverage, profiled: { types: profiledTypes, masks: profiledMasks }, samples: samplesPer, - isDriver: (r) => annotation.isDriver(r as Parameters[0]), annotationSource: annotation.source, + const contract = assembleIndexed({ + genes: geneList, columns, groups, studies, ...(multi ? { columnStudy: Int32Array.from(columnStudy) } : {}), + events, coverage, profiled: { types: profiledTypes, masks: profiledMasks }, samples: samplesPer, + isDriver: (r) => annotation.isDriver(r), annotationSource: annotation.source, clinical: clinicalIn, heatmap: heatmapIn, overlays: overlaysIn, zReference: spec.zReference, derived: derivedIn, sort, lineGap, display, unit, clusterRank }); contract.stats.ms = Date.now() - t0; diff --git a/lib/provenance.ts b/lib/provenance.ts index b36975e2..e5ad07d9 100644 --- a/lib/provenance.ts +++ b/lib/provenance.ts @@ -13,15 +13,25 @@ const imported = new Map }>( /** When `study`'s data was last imported into the database `conn` reaches, or null if it can't tell. */ export function studyImported(study: string, conn: ChConn): Promise { - const key = conn.id + "|" + conn.db + "|" + study; - const hit = imported.get(key); - if (hit && Date.now() - hit.at < IMPORTED_TTL) return hit.value; - const value = chQuery<{ d: string }>("SELECT toString(max(import_date)) AS d FROM " + conn.db + ".cancer_study WHERE cancer_study_identifier = " + sq(study), conn) - .then((r) => (r.data[0]?.d || "").replace(/\.0+$/, "") || null) - .catch((e) => { if (isAbortError(e)) throw e; return null; }); - imported.set(key, { at: Date.now(), value }); - value.catch(() => { if (imported.get(key)?.value === value) imported.delete(key); }); - return value; + return studiesImported([study], conn).get(study)!; +} + +/** The same for several studies of one database, asked in one statement for those not already known. */ +export function studiesImported(studies: string[], conn: ChConn): Map> { + const keyOf = (study: string) => conn.id + "|" + conn.db + "|" + study; + const todo = [...new Set(studies)].filter((s) => { const hit = imported.get(keyOf(s)); return !(hit && Date.now() - hit.at < IMPORTED_TTL); }); + if (todo.length) { + const rows = chQuery<{ s: string; d: string }>("SELECT cancer_study_identifier AS s, toString(max(import_date)) AS d FROM " + conn.db + ".cancer_study\n" + + "WHERE cancer_study_identifier IN (" + todo.map(sq).join(",") + ") GROUP BY s", conn) + .then((r) => new Map(r.data.map((x) => [x.s, (x.d || "").replace(/\.0+$/, "") || null]))); + for (const study of todo) { + const key = keyOf(study); + const value = rows.then((m) => m.get(study) ?? null).catch((e) => { if (isAbortError(e)) throw e; return null; }); + imported.set(key, { at: Date.now(), value }); + value.catch(() => { if (imported.get(key)?.value === value) imported.delete(key); }); + } + } + return new Map(studies.map((s) => [s, imported.get(keyOf(s))!.value])); } /** What this instance reports about itself when it answers as a peer, for `study` if one is named. */ @@ -59,6 +69,16 @@ export function sourceIndex(studies: string[], resolvers: Record): Promise { const out: AnalysisSource[] = [], byKey = new Map(); + // every local study's import date, one statement per database + const local = new Map(); + for (const study of studies) { + const r = resolvers[study]; + if (!r || peerProvenance(r)) continue; + const k = r.instance.conn.id + "|" + r.instance.conn.db; + (local.get(k) ?? local.set(k, { conn: r.instance.conn, studies: [] }).get(k)!).studies.push(study); + } + const importedAt = new Map>(); + for (const { conn, studies: ss } of local.values()) for (const [s, v] of studiesImported(ss, conn)) importedAt.set(s, v); for (const study of studies) { const r = resolvers[study]; if (!r) continue; @@ -75,7 +95,7 @@ export async function describeSources(studies: string[], resolvers: Record; const { xk, yk, shape, xLabel, yLabel } = await fr[sites[0]].plotAxisKinds(spec); if (shape === "waterfall") throw new Error("an ordered-cases axis (a waterfall) is not implemented yet; give both axes a source"); + // a box's cases are read by every site against its own metadata, so an axis a site does not + // declare refuses the view before anything is counted + if (shape === "box") await Promise.all(sites.slice(1).map((s) => fr[s].plotAxisKinds(spec))); const label = { x: { label: xLabel, kind: xk }, y: { label: yLabel, kind: yk } }; const scaleNote = sites.length + " studies pooled (" + sites.join(", ") + ")"; + // Each site counts its own cases and their axes, whatever the shape. It needs nothing the + // figure's queries return, so it is asked for once their first round has answered — beside the + // second, rather than after it, and not at all for a view refused in the first. A failed count + // fails the view: dropping the coverage note would hide how many cases the figure leaves out. + let covsP: Promise[]> | undefined; + const startCoverage = () => { + if (!covsP) { covsP = Promise.all(sites.map((s) => fr[s].plotCoverage(doc.cohorts[cn], spec, xk, yk))); covsP.catch(() => {}); } + }; let data: PlotData; if (shape === "scatter") { const extents = await Promise.all(sites.map((s) => fr[s].plotExtent(doc.cohorts[cn], spec, xk, yk))); + startCoverage(); const have = extents.filter((e) => e.n > 0); const n = have.reduce((a, e) => a + e.n, 0); if (!n) { data = { shape, n: 0, ...label, note: "no " + unit + " has a value on both axes" }; } @@ -488,14 +500,16 @@ export async function runPlot(view: DocView, c: ViewRun): Promise fr[s].plotBoxRows(doc.cohorts[cn], spec, xk, yk)))); if (groups.length > CATEGORY_CAP) throw new Error("more than " + CATEGORY_CAP + " groups on the categorical axis; narrow the cohort or pick another attribute"); + startCoverage(); const n = groups.reduce((a, g) => a + g.n, 0); data = { shape, n, ...label, groups, overlay: "none", note: scaleNote + ". Counts and extremes per group sum exactly; the quartiles are a size-weighted" + " approximation across sites (it can shift where an outlier bar falls, never a count)." }; // Points, unlike a density, just concatenate: each entitled site's own case-level // rows are already independent of any shared scale. A site that may not disclose - // cases contributes none, the same rule the oncoprint's columns follow. - if (pv.allPoints !== false) { + // cases contributes none, the same rule the oncoprint's columns follow. A site's points + // are at least its box's cases, so past the cap in all there is nothing to ask for. + if (pv.allPoints !== false && n <= POINTS_CAP) { const perSite = await peerSafe(sites.map((s) => resolvers[s]), () => Promise.all(sites.map((s) => { const rc = plotCasesOf(resolvers[s]); return rc ? rc.plotPoints(doc.cohorts[cn], spec) : Promise.resolve([]); @@ -524,9 +538,9 @@ export async function runPlot(view: DocView, c: ViewRun): Promise fr[s].plotCoverage(doc.cohorts[cn], spec, xk, yk))); + // the sites' coverage counts sum exactly + startCoverage(); + const covs = await covsP!; data.coverage = covs.reduce>( (a, c) => ({ total: a.total + c.total, nx: a.nx + c.nx, ny: a.ny + c.ny, ...(c.multi != null || a.multi != null ? { multi: (a.multi || 0) + (c.multi || 0) } : {}) }), { total: 0, nx: 0, ny: 0 }); diff --git a/test/oncoprint-sql.test.ts b/test/oncoprint-sql.test.ts index bca883a5..a9dff671 100644 --- a/test/oncoprint-sql.test.ts +++ b/test/oncoprint-sql.test.ts @@ -3,17 +3,36 @@ // these pin the rules, so a later edit cannot quietly narrow them. import test from "node:test"; import assert from "node:assert/strict"; -import { columnFactsSql, coverageSql, assayHeatmapSql, normalizeSpec } from "../lib/oncoprint.ts"; +import { allColumnsSql, batchStudies, clinicalSql, columnFactsSql, coverageSql, genomicEventsSql, assayHeatmapSql, normalizeSpec } from "../lib/oncoprint.ts"; import { assayTrackLabel, entityShortName } from "../lib/profile-names.ts"; test("a gene cell is profiled by a mutation, copy number or structural variant panel, as in the portal", () => { - const sql = coverageSql(["TP53", "CDKN2A"], "s"); + const sql = coverageSql(["TP53", "CDKN2A"], ["s"]); assert.match(sql, /alteration_type IN \('MUTATION_EXTENDED','COPY_NUMBER_ALTERATION','STRUCTURAL_VARIANT'\)/); - assert.match(sql, /pg\.gene IN \('TP53','CDKN2A'\)/, "and only for the genes that panel covers"); + assert.match(sql, /gene_panel_to_gene_derived WHERE gene IN \('TP53','CDKN2A'\) GROUP BY gene_panel_id/, "and only for the genes that panel covers"); + assert.match(sql, /JOIN pm ON pm\.gene_panel_id = sp\.gene_panel_id/, "each sample takes its own panels' masks"); +}); + +test("whole-study fetches read several studies in one statement, each row led by its own study", () => { + for (const sql of [genomicEventsSql(["TP53"], ["s1", "s2"], "db"), coverageSql(["TP53"], ["s1", "s2"], "db"), columnFactsSql(["s1", "s2"], "db"), + clinicalSql(["SEX"], ["s1", "s2"], "db"), allColumnsSql(["s1", "s2"], "db"), clinicalSql(["SEX"], ["s1", "s2"], "db", "sample")]) { + assert.match(sql, /^(WITH[\s\S]*\n)?SELECT (DISTINCT )?\S*cancer_study_identifier AS s, replaceOne\(\S*_unique_id, concat\(\S*cancer_study_identifier, '_'\), ''\) AS pid/m, + "the study first, and each id stripped of its own study's prefix"); + assert.match(sql, /cancer_study_identifier IN \('s1','s2'\)/); + assert.doesNotMatch(sql, /'s1_'|'s2_'/, "no one study's prefix is assumed"); + } +}); + +test("studies are batched by size: small ones share a statement, a large one has its own, never across connections", () => { + const size: Record = { big: 60_000, a: 10_000, b: 10_000, c: 10_000, d: 100, far: 100 }; + const conn = (s: string) => s === "far" ? "c2" : "c1"; + assert.deepEqual(batchStudies(["a", "big", "b", "c", "far", "d"], conn, (s) => size[s], 25_000), [["big"], ["a", "b"], ["c", "d"], ["far"]]); + assert.deepEqual(batchStudies(["only"], conn, () => 0), [["only"]]); + assert.deepEqual(batchStudies([], conn, () => 0), []); }); test("column facts count every sample of a column, profiled or not, and type masks follow PROFILED_TYPES", () => { - const sql = columnFactsSql("s"); + const sql = columnFactsSql(["s"]); assert.match(sql, /FROM \S*\.sample_derived sd\nLEFT JOIN/, "a column profiled for nothing still has a row"); assert.match(sql, /uniqExact\(sd\.sample_unique_id\) AS samples/); assert.match(sql, /indexOf\(\['MUTATION_EXTENDED','COPY_NUMBER_ALTERATION','STRUCTURAL_VARIANT','MRNA_EXPRESSION','PROTEIN_LEVEL'\]/); diff --git a/test/oql-classes.test.ts b/test/oql-classes.test.ts index 6a64877a..98dc4724 100644 --- a/test/oql-classes.test.ts +++ b/test/oql-classes.test.ts @@ -59,7 +59,7 @@ describe("uncalled mutations are read support, not alterations", () => { assert.match(leaf("APC: TRUNC_(1400-)"), /ifNull\(mutationStatus, ''\) AS mutation_status/); }); test("the oncoprint's cells and VAF, and a plot's alteration axis, leave them out", () => { - assert.match(genomicEventsSql(["TP53"], "s1", "db"), /AND mutation_status != 'UNCALLED'/); + assert.match(genomicEventsSql(["TP53"], ["s1"], "db"), /AND mutation_status != 'UNCALLED'/); assert.match(vafSql(["TP53"], "s1", "db"), /molecularProfileId NOT LIKE '%uncalled%'/); assert.match(axisSelect({ source: "alteration", genes: ["TP53"] }, { study: "s1", db: "db", unit: "sample", cohortSql: "SELECT 1" }, "cat"), /AND mutation_status != 'UNCALLED'/); diff --git a/test/snapshots/answers/acquired.json.gz b/test/snapshots/answers/acquired.json.gz index 943f1894..ffb62331 100644 Binary files a/test/snapshots/answers/acquired.json.gz and b/test/snapshots/answers/acquired.json.gz differ diff --git a/test/snapshots/answers/actionability.json.gz b/test/snapshots/answers/actionability.json.gz index 5874b093..e7d196ca 100644 Binary files a/test/snapshots/answers/actionability.json.gz and b/test/snapshots/answers/actionability.json.gz differ diff --git a/test/snapshots/answers/attrition.json.gz b/test/snapshots/answers/attrition.json.gz index dc737cd6..e7bd223d 100644 Binary files a/test/snapshots/answers/attrition.json.gz and b/test/snapshots/answers/attrition.json.gz differ diff --git a/test/snapshots/answers/clinical.json.gz b/test/snapshots/answers/clinical.json.gz index bc8e3229..7a60a74c 100644 Binary files a/test/snapshots/answers/clinical.json.gz and b/test/snapshots/answers/clinical.json.gz differ diff --git a/test/snapshots/answers/coexpression.json.gz b/test/snapshots/answers/coexpression.json.gz index b6156f3d..f71898ac 100644 Binary files a/test/snapshots/answers/coexpression.json.gz and b/test/snapshots/answers/coexpression.json.gz differ diff --git a/test/snapshots/answers/compare.json.gz b/test/snapshots/answers/compare.json.gz index 3f159e05..f49248c9 100644 Binary files a/test/snapshots/answers/compare.json.gz and b/test/snapshots/answers/compare.json.gz differ diff --git a/test/snapshots/answers/composition.json.gz b/test/snapshots/answers/composition.json.gz index 15af34bc..0a5b926a 100644 Binary files a/test/snapshots/answers/composition.json.gz and b/test/snapshots/answers/composition.json.gz differ diff --git a/test/snapshots/answers/correlates.json.gz b/test/snapshots/answers/correlates.json.gz index 1bb35ed4..c7347575 100644 Binary files a/test/snapshots/answers/correlates.json.gz and b/test/snapshots/answers/correlates.json.gz differ diff --git a/test/snapshots/answers/course.json.gz b/test/snapshots/answers/course.json.gz index 52bfbfad..610411f1 100644 Binary files a/test/snapshots/answers/course.json.gz and b/test/snapshots/answers/course.json.gz differ diff --git a/test/snapshots/answers/cox.json.gz b/test/snapshots/answers/cox.json.gz index c4dfe883..889f9535 100644 Binary files a/test/snapshots/answers/cox.json.gz and b/test/snapshots/answers/cox.json.gz differ diff --git a/test/snapshots/answers/ctdna-chfilter.json.gz b/test/snapshots/answers/ctdna-chfilter.json.gz index ef86d6e4..d625ee29 100644 Binary files a/test/snapshots/answers/ctdna-chfilter.json.gz and b/test/snapshots/answers/ctdna-chfilter.json.gz differ diff --git a/test/snapshots/answers/ctdna.json.gz b/test/snapshots/answers/ctdna.json.gz index 31509734..3be8a40c 100644 Binary files a/test/snapshots/answers/ctdna.json.gz and b/test/snapshots/answers/ctdna.json.gz differ diff --git a/test/snapshots/answers/diffexpr.json.gz b/test/snapshots/answers/diffexpr.json.gz index a007dd04..cacd3a6b 100644 Binary files a/test/snapshots/answers/diffexpr.json.gz and b/test/snapshots/answers/diffexpr.json.gz differ diff --git a/test/snapshots/answers/exclusivity.json.gz b/test/snapshots/answers/exclusivity.json.gz index a90a112b..15ee307b 100644 Binary files a/test/snapshots/answers/exclusivity.json.gz and b/test/snapshots/answers/exclusivity.json.gz differ diff --git a/test/snapshots/answers/genome.json.gz b/test/snapshots/answers/genome.json.gz index 66bc968d..bbabbeee 100644 Binary files a/test/snapshots/answers/genome.json.gz and b/test/snapshots/answers/genome.json.gz differ diff --git a/test/snapshots/answers/germline.json.gz b/test/snapshots/answers/germline.json.gz index 9ae327a2..182ab1c4 100644 Binary files a/test/snapshots/answers/germline.json.gz and b/test/snapshots/answers/germline.json.gz differ diff --git a/test/snapshots/answers/hla.json.gz b/test/snapshots/answers/hla.json.gz index 5ce9772e..d1401689 100644 Binary files a/test/snapshots/answers/hla.json.gz and b/test/snapshots/answers/hla.json.gz differ diff --git a/test/snapshots/answers/hotspots.json.gz b/test/snapshots/answers/hotspots.json.gz index f9e4892d..c9a0146a 100644 Binary files a/test/snapshots/answers/hotspots.json.gz and b/test/snapshots/answers/hotspots.json.gz differ diff --git a/test/snapshots/answers/logistic.json.gz b/test/snapshots/answers/logistic.json.gz index 8120da89..44c2166b 100644 Binary files a/test/snapshots/answers/logistic.json.gz and b/test/snapshots/answers/logistic.json.gz differ diff --git a/test/snapshots/answers/molsubtype.json.gz b/test/snapshots/answers/molsubtype.json.gz index 397d3427..7e6096a8 100644 Binary files a/test/snapshots/answers/molsubtype.json.gz and b/test/snapshots/answers/molsubtype.json.gz differ diff --git a/test/snapshots/answers/paired.json.gz b/test/snapshots/answers/paired.json.gz index 5f905b6a..624566ed 100644 Binary files a/test/snapshots/answers/paired.json.gz and b/test/snapshots/answers/paired.json.gz differ diff --git a/test/snapshots/answers/plot.json.gz b/test/snapshots/answers/plot.json.gz index 27f3e2ce..8917aea4 100644 Binary files a/test/snapshots/answers/plot.json.gz and b/test/snapshots/answers/plot.json.gz differ diff --git a/test/snapshots/answers/pooled-paired.json.gz b/test/snapshots/answers/pooled-paired.json.gz index 4f4afde9..fb08c79c 100644 Binary files a/test/snapshots/answers/pooled-paired.json.gz and b/test/snapshots/answers/pooled-paired.json.gz differ diff --git a/test/snapshots/answers/riskmodel-validateon.json.gz b/test/snapshots/answers/riskmodel-validateon.json.gz index b0531aef..8c75a6a5 100644 Binary files a/test/snapshots/answers/riskmodel-validateon.json.gz and b/test/snapshots/answers/riskmodel-validateon.json.gz differ diff --git a/test/snapshots/answers/rmst.json.gz b/test/snapshots/answers/rmst.json.gz index 8ec5226c..ae90c63d 100644 Binary files a/test/snapshots/answers/rmst.json.gz and b/test/snapshots/answers/rmst.json.gz differ diff --git a/test/snapshots/answers/sequence.json.gz b/test/snapshots/answers/sequence.json.gz index df2e8c4c..c649e21c 100644 Binary files a/test/snapshots/answers/sequence.json.gz and b/test/snapshots/answers/sequence.json.gz differ diff --git a/test/snapshots/answers/survival.json.gz b/test/snapshots/answers/survival.json.gz index 92124f4c..7256480d 100644 Binary files a/test/snapshots/answers/survival.json.gz and b/test/snapshots/answers/survival.json.gz differ diff --git a/test/snapshots/answers/two-studies-four-views.json.gz b/test/snapshots/answers/two-studies-four-views.json.gz index 89957e88..327ddf4f 100644 Binary files a/test/snapshots/answers/two-studies-four-views.json.gz and b/test/snapshots/answers/two-studies-four-views.json.gz differ diff --git a/test/snapshots/answers/two-studies-plot.json.gz b/test/snapshots/answers/two-studies-plot.json.gz index 37a8bdeb..de2687ef 100644 Binary files a/test/snapshots/answers/two-studies-plot.json.gz and b/test/snapshots/answers/two-studies-plot.json.gz differ diff --git a/test/snapshots/answers/two-studies-survival.json.gz b/test/snapshots/answers/two-studies-survival.json.gz index 14dca690..4e18d05b 100644 Binary files a/test/snapshots/answers/two-studies-survival.json.gz and b/test/snapshots/answers/two-studies-survival.json.gz differ diff --git a/test/snapshots/diffexpr.json b/test/snapshots/diffexpr.json index aaf97a52..d622e602 100644 --- a/test/snapshots/diffexpr.json +++ b/test/snapshots/diffexpr.json @@ -84,23 +84,23 @@ "gene": "NAT1", "nA": 171, "nB": 499, - "meanA": -1.3843912280701764, + "meanA": -1.384391228070176, "meanB": 0.5428809619238478, - "effect": -1.9272721899940244, - "t": -44.23467399694273, - "p": 2.7043279892748184e-182, - "q": 1.0820557150686404e-178 + "effect": -1.927272189994024, + "t": -44.234673996942654, + "p": 2.7043279892792852e-182, + "q": 1.0820557150704277e-178 }, { "gene": "MAPT-IT1", "nA": 171, "nB": 499, - "meanA": -1.6838713450292397, + "meanA": -1.6838713450292402, "meanB": 0.20232264529058128, - "effect": -1.886193990319821, - "t": -39.81370583703942, - "p": 2.0129532020700296e-175, - "q": 6.711856960102169e-172 + "effect": -1.8861939903198215, + "t": -39.813705837039485, + "p": 2.0129532020694587e-175, + "q": 6.711856960100265e-172 }, { "gene": "CT62", @@ -109,9 +109,9 @@ "meanA": -2.0318602339181275, "meanB": 0.41401743486973996, "effect": -2.4458776687878676, - "t": -49.396461591675944, - "p": 6.399560110459782e-157, - "q": 1.8289942795694058e-153 + "t": -49.39646159167599, + "p": 6.39956011045541e-157, + "q": 1.828994279568156e-153 }, { "gene": "SLC7A13", @@ -172,12 +172,12 @@ "gene": "MAPT-AS1", "nA": 171, "nB": 499, - "meanA": -1.6475619883040897, + "meanA": -1.6475619883040895, "meanB": 0.08925931863727461, - "effect": -1.7368213069413643, - "t": -34.46869324253705, - "p": 2.7982416787569205e-144, - "q": 4.3062786942469967e-141 + "effect": -1.736821306941364, + "t": -34.46869324253701, + "p": 2.798241678757557e-144, + "q": 4.306278694247976e-141 }, { "gene": "ANKRD30A", @@ -186,9 +186,9 @@ "meanA": -1.9812163742690037, "meanB": 0.27829458917835675, "effect": -2.2595109634473602, - "t": -39.37280226605274, - "p": 5.063304961997455e-144, - "q": 7.235462790694364e-141 + "t": -39.37280226605271, + "p": 5.0633049620006246e-144, + "q": 7.235462790698894e-141 }, { "gene": "BCL11A", @@ -216,12 +216,12 @@ "gene": "PGR", "nA": 171, "nB": 499, - "meanA": -1.2939280701754385, + "meanA": -1.2939280701754383, "meanB": 0.5647364729458908, - "effect": -1.8586645431213293, - "t": -38.20350861907161, - "p": 1.2738662408655656e-139, - "q": 1.499115765573912e-136 + "effect": -1.8586645431213291, + "t": -38.20350861907157, + "p": 1.2738662408662915e-139, + "q": 1.4991157655747665e-136 }, { "gene": "CDCA7", @@ -252,13 +252,13 @@ "meanA": -1.2425690058479535, "meanB": 0.4289593186372745, "effect": -1.671528324485228, - "t": -34.674209417030276, - "p": 2.069234735993648e-137, - "q": 2.069855506414446e-134 + "t": -34.67420941703029, + "p": 2.0692347359923535e-137, + "q": 2.0698555064131514e-134 }, { "…": "1480 more", - "sha256": "522f517d2b7aef2f" + "sha256": "56f4f6f3f08baf08" } ], "grid": { @@ -2800,32 +2800,32 @@ "abs": 1e-300 }, "diffexpr:NAT1:p": { - "v": 2.7043279892748184e-182, + "v": 2.7043279892792852e-182, "rel": 0.000001, "abs": 1e-300 }, "diffexpr:NAT1:q": { - "v": 1.0820557150686404e-178, + "v": 1.0820557150704277e-178, "rel": 0.000001, "abs": 1e-300 }, "diffexpr:MAPT-IT1:p": { - "v": 2.0129532020700296e-175, + "v": 2.0129532020694587e-175, "rel": 0.000001, "abs": 1e-300 }, "diffexpr:MAPT-IT1:q": { - "v": 6.711856960102169e-172, + "v": 6.711856960100265e-172, "rel": 0.000001, "abs": 1e-300 }, "diffexpr:CT62:p": { - "v": 6.399560110459782e-157, + "v": 6.39956011045541e-157, "rel": 0.000001, "abs": 1e-300 }, "diffexpr:CT62:q": { - "v": 1.8289942795694058e-153, + "v": 1.828994279568156e-153, "rel": 0.000001, "abs": 1e-300 }, @@ -2856,7 +2856,7 @@ }, "…": { "more": 3131, - "sha256": "baa736b72024a2de" + "sha256": "e90e105b7869592d" } } }