From 0ac1fe6464e347ec84888604c677e4b3598f1ddd Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 08:25:30 +0000 Subject: [PATCH 01/65] core: the network an address belongs to, and what a report may ask for --- packages/core/src/index.ts | 1 + packages/core/src/reporting.test.ts | 75 +++++++++++++++++++++++++++ packages/core/src/reporting.ts | 80 +++++++++++++++++++++++++++++ packages/core/src/settings.test.ts | 60 +++++++++++++++++++++- packages/core/src/settings.ts | 52 +++++++++++++++++++ packages/ipdata/src/ip.test.ts | 49 +++++++++++++++++- packages/ipdata/src/ip.ts | 27 ++++++++++ 7 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/reporting.test.ts create mode 100644 packages/core/src/reporting.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c6a6f15..9327175 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,6 +15,7 @@ export * from './evaluate.js' export * from './link.js' export * from './passthrough.js' export * from './proxies.js' +export * from './reporting.js' export * from './rotation.js' export * from './secrets.js' export * from './settings.js' diff --git a/packages/core/src/reporting.test.ts b/packages/core/src/reporting.test.ts new file mode 100644 index 0000000..19d8056 --- /dev/null +++ b/packages/core/src/reporting.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest' +import { + BUCKET_MS, + KEYED_DIMENSIONS, + MAX_REPORT_BUCKETS, + MAX_REPORT_WINDOW_DAYS, + REPORT_BUCKETS, + REPORT_DIMENSIONS, + ROLLUP_DIMENSIONS, + bucketCount, + isKeyedDimension, +} from './reporting.js' + +const at = (iso: string): number => Date.parse(iso) + +describe('bucketCount', () => { + it('counts the hours a window touches, not the hours it lasts', () => { + // Twenty minutes, straddling an hour boundary: two bars on a chart. + expect(bucketCount(at('2026-09-24T10:50:00Z'), at('2026-09-24T11:10:00Z'), 'hour')).toBe(2) + // Twenty minutes inside one hour: one bar. + expect(bucketCount(at('2026-09-24T10:20:00Z'), at('2026-09-24T10:40:00Z'), 'hour')).toBe(1) + }) + + it('treats the window as half-open, so a whole hour is one bucket', () => { + expect(bucketCount(at('2026-09-24T10:00:00Z'), at('2026-09-24T11:00:00Z'), 'hour')).toBe(1) + expect(bucketCount(at('2026-09-24T10:00:00Z'), at('2026-09-24T11:00:00.001Z'), 'hour')).toBe(2) + }) + + it('counts days the same way', () => { + expect(bucketCount(at('2026-09-24T00:00:00Z'), at('2026-09-25T00:00:00Z'), 'day')).toBe(1) + expect(bucketCount(at('2026-09-24T23:59:59Z'), at('2026-09-25T00:00:01Z'), 'day')).toBe(2) + expect(bucketCount(at('2026-09-01T00:00:00Z'), at('2026-10-01T00:00:00Z'), 'day')).toBe(30) + }) + + it('answers zero for an empty or backwards window', () => { + expect(bucketCount(at('2026-09-24T10:00:00Z'), at('2026-09-24T10:00:00Z'), 'hour')).toBe(0) + expect(bucketCount(at('2026-09-24T11:00:00Z'), at('2026-09-24T10:00:00Z'), 'hour')).toBe(0) + }) + + // The two numbers the chart endpoint refuses a window with. Written out + // rather than computed from the constants, so changing either constant + // fails here first. + it('puts a year of hours inside the bucket ceiling and 400 days of hours outside it', () => { + expect(bucketCount(at('2025-09-24T00:00:00Z'), at('2026-09-24T00:00:00Z'), 'hour')).toBe(8760) + expect(MAX_REPORT_BUCKETS).toBe(2000) + expect(MAX_REPORT_WINDOW_DAYS).toBe(400) + expect(BUCKET_MS).toEqual({ hour: 3_600_000, day: 86_400_000 }) + }) +}) + +describe('the dimensions a breakdown may name', () => { + it('is the six the per-dimension rollup holds and the three that key the hourly one', () => { + expect(ROLLUP_DIMENSIONS).toEqual(['country', 'device', 'os', 'browser', 'referrer', 'target']) + expect(KEYED_DIMENSIONS).toEqual(['class', 'action', 'outcome']) + expect(REPORT_DIMENSIONS).toEqual([ + 'country', + 'device', + 'os', + 'browser', + 'referrer', + 'target', + 'class', + 'action', + 'outcome', + ]) + expect(REPORT_BUCKETS).toEqual(['hour', 'day']) + }) + + it('says which table each one is answered from', () => { + expect(isKeyedDimension('class')).toBe(true) + expect(isKeyedDimension('outcome')).toBe(true) + expect(isKeyedDimension('country')).toBe(false) + expect(isKeyedDimension('referrer')).toBe(false) + }) +}) diff --git a/packages/core/src/reporting.ts b/packages/core/src/reporting.ts new file mode 100644 index 0000000..db27eaf --- /dev/null +++ b/packages/core/src/reporting.ts @@ -0,0 +1,80 @@ +/** + * The vocabulary a report is asked in: the two bucket sizes, the dimensions a + * breakdown may name, and the bounds on a window. + * + * It is here rather than in the admin service because two files have to agree + * on the dimension names — the one that parses a request and the one that + * creates the rollup those names are keys of — and because none of it is I/O. + * + * **A window is half-open and UTC**: `from` is included, `to` is not, and + * nothing here knows what "today" means. That is deliberate. A preset like + * "last 7 days" needs a timezone; this install has no opinion about the + * operator's, and a timezone parameter on five endpoints is five places to get + * it wrong. A caller computes a preset into a pair of instants. The hourly + * grain is what leaves a whole-hour offset possible later without a schema + * change; a half-hour zone would need one. + */ + +/** The two bucket sizes a chart is drawn at. */ +export const REPORT_BUCKETS = ['hour', 'day'] as const +export type ReportBucket = (typeof REPORT_BUCKETS)[number] + +export const BUCKET_MS: Record = { + hour: 3_600_000, + day: 86_400_000, +} + +/** + * Dimensions held as rows of the per-dimension rollup, one row per value that + * actually occurred. Every one of them is open — a country list grows, a + * browser family is a plain string, and a referrer host is whatever somebody + * linked from — which is why they are rows rather than columns of a key. + */ +export const ROLLUP_DIMENSIONS = [ + 'country', + 'device', + 'os', + 'browser', + 'referrer', + 'target', +] as const +export type RollupDimension = (typeof ROLLUP_DIMENSIONS)[number] + +/** + * Dimensions that are keys of the hourly rollup itself. All three are closed + * lists whose values cannot grow without a new click record version, so keying + * by them costs a bounded number of rows per hour and per link, and a summary + * can be answered from one table. + */ +export const KEYED_DIMENSIONS = ['class', 'action', 'outcome'] as const +export type KeyedDimension = (typeof KEYED_DIMENSIONS)[number] + +export const REPORT_DIMENSIONS = [...ROLLUP_DIMENSIONS, ...KEYED_DIMENSIONS] as const +export type ReportDimension = RollupDimension | KeyedDimension + +/** Which of the two rollups answers this dimension. */ +export function isKeyedDimension(d: ReportDimension): d is KeyedDimension { + return (KEYED_DIMENSIONS as readonly string[]).includes(d) +} + +/** The longest window any report answers for. */ +export const MAX_REPORT_WINDOW_DAYS = 400 +export const MAX_REPORT_WINDOW_MS = MAX_REPORT_WINDOW_DAYS * 86_400_000 + +/** The most buckets one chart response carries. */ +export const MAX_REPORT_BUCKETS = 2000 + +/** + * How many buckets the half-open window [fromMs, toMs) touches. + * + * Counted from aligned boundaries rather than from elapsed time: twenty + * minutes either side of an hour boundary is forty minutes of elapsed time and + * two bars on a chart, and a ceiling computed the other way would let a + * response through that is one bar bigger than the ceiling. Unix time carries + * no leap seconds, so a day boundary is exact arithmetic on the epoch. + */ +export function bucketCount(fromMs: number, toMs: number, bucket: ReportBucket): number { + if (toMs <= fromMs) return 0 + const ms = BUCKET_MS[bucket] + return Math.floor((toMs - 1) / ms) - Math.floor(fromMs / ms) + 1 +} diff --git a/packages/core/src/settings.test.ts b/packages/core/src/settings.test.ts index 2bc239f..83d4a59 100644 --- a/packages/core/src/settings.test.ts +++ b/packages/core/src/settings.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest' -import { DEFAULT_TRAFFIC_SETTINGS, TrafficSettingsSchema } from './settings.js' +import { + DEFAULT_RETENTION, + DEFAULT_TRAFFIC_SETTINGS, + RetentionSettingsSchema, + TrafficSettingsSchema, + retentionNote, +} from './settings.js' describe('TrafficSettingsSchema', () => { it('accepts the defaults', () => { @@ -36,3 +42,55 @@ describe('TrafficSettingsSchema', () => { ) }) }) + +describe('RetentionSettingsSchema', () => { + it('accepts the defaults: ninety days of clicks and thirty of addresses', () => { + expect(DEFAULT_RETENTION).toEqual({ rawRetentionDays: 90, ipRetentionDays: 30 }) + expect(RetentionSettingsSchema.parse(DEFAULT_RETENTION)).toEqual(DEFAULT_RETENTION) + }) + + it('takes null for either period, meaning never', () => { + const forever = { rawRetentionDays: null, ipRetentionDays: null } + expect(RetentionSettingsSchema.parse(forever)).toEqual(forever) + }) + + it.each([ + ['zero, which reads as both forever and now', { rawRetentionDays: 0, ipRetentionDays: 30 }], + ['a negative period', { rawRetentionDays: 90, ipRetentionDays: -1 }], + ['a fraction of a day', { rawRetentionDays: 1.5, ipRetentionDays: 30 }], + ['more than ten years', { rawRetentionDays: 3651, ipRetentionDays: 30 }], + ['a period as a string', { rawRetentionDays: '90', ipRetentionDays: 30 }], + ['a field nobody knows', { rawRetentionDays: 90, ipRetentionDays: 30, keepEverything: true }], + ])('refuses %s', (_label, value) => { + expect(RetentionSettingsSchema.safeParse(value).success).toBe(false) + }) + + // Not a refusal: lowering the raw period below the IP period is a + // tightening, and a check that refused it would refuse the safe direction. + it('notes an IP period that outlives the clicks it belongs to, and refuses nothing', () => { + expect(retentionNote({ rawRetentionDays: 30, ipRetentionDays: 90 })).toBe( + 'addresses are set to be kept for 90 days but clicks for 30 days, so an address goes when its click does, after 30 days', + ) + expect( + RetentionSettingsSchema.safeParse({ rawRetentionDays: 30, ipRetentionDays: 90 }).success, + ).toBe(true) + }) + + it.each([ + ['both set the usual way round', { rawRetentionDays: 90, ipRetentionDays: 30 }], + ['clicks kept forever', { rawRetentionDays: null, ipRetentionDays: 30 }], + [ + 'addresses kept forever with clicks kept forever', + { rawRetentionDays: null, ipRetentionDays: null }, + ], + ['the same period for both', { rawRetentionDays: 30, ipRetentionDays: 30 }], + ])('has nothing to say about %s', (_label, value) => { + expect(retentionNote(value)).toBeNull() + }) + + it('notes an address kept forever while the clicks are not', () => { + expect(retentionNote({ rawRetentionDays: 90, ipRetentionDays: null })).toBe( + 'addresses are set to be kept for ever but clicks for 90 days, so an address goes when its click does, after 90 days', + ) + }) +}) diff --git a/packages/core/src/settings.ts b/packages/core/src/settings.ts index da2858b..38fc5ba 100644 --- a/packages/core/src/settings.ts +++ b/packages/core/src/settings.ts @@ -53,3 +53,55 @@ export const TrafficSettingsSchema = z } } }) + +/** The longest either retention period may be set to: ten years. */ +export const MAX_RETENTION_DAYS = 3650 +/** How long raw clicks are kept unless the admin says otherwise. */ +export const DEFAULT_RAW_RETENTION_DAYS = 90 +/** And how long the whole IP address on one is kept. */ +export const DEFAULT_IP_RETENTION_DAYS = 30 + +/** + * How long this install keeps the two things it can stop keeping. + * + * **Null is forever**, for either. Zero was rejected as the forever value + * because it reads equally as "drop everything now", and the difference + * between those two readings is the whole table. + * + * Aggregates outlive both: the hourly rollups are never dropped, because they + * are small and they are what answers a question older than the raw window. + */ +export interface RetentionSettings { + /** Days of raw clicks. Null: kept for ever. */ + rawRetentionDays: number | null + /** Days the whole address on a click is kept before it is blanked. Null: for ever. */ + ipRetentionDays: number | null +} + +export const DEFAULT_RETENTION: RetentionSettings = { + rawRetentionDays: DEFAULT_RAW_RETENTION_DAYS, + ipRetentionDays: DEFAULT_IP_RETENTION_DAYS, +} + +const RetentionDays = z.number().int().min(1).max(MAX_RETENTION_DAYS).nullable() + +export const RetentionSettingsSchema = z + .object({ rawRetentionDays: RetentionDays, ipRetentionDays: RetentionDays }) + .strict() + +/** + * What the operator should be told about a pair that is legal but does not do + * what it looks like: an IP period longer than the raw period never runs, + * because the click is gone first. + * + * It is a note and not a refusal on purpose. A CHECK that required the IP + * period to be the shorter one would refuse `raw = 10` on an install whose IP + * period is 30 — the tightening direction — and an operator reducing what they + * keep must never be the one who is refused. + */ +export function retentionNote(r: RetentionSettings): string | null { + if (r.rawRetentionDays === null) return null + if (r.ipRetentionDays !== null && r.ipRetentionDays <= r.rawRetentionDays) return null + const kept = r.ipRetentionDays === null ? 'for ever' : `for ${r.ipRetentionDays} days` + return `addresses are set to be kept ${kept} but clicks for ${r.rawRetentionDays} days, so an address goes when its click does, after ${r.rawRetentionDays} days` +} diff --git a/packages/ipdata/src/ip.test.ts b/packages/ipdata/src/ip.test.ts index d833276..7b6d1b3 100644 --- a/packages/ipdata/src/ip.test.ts +++ b/packages/ipdata/src/ip.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { addressOnly, canonicalIp, parseIp, rateKey } from './ip.js' +import { addressOnly, canonicalIp, parseIp, rateKey, truncateIp } from './ip.js' describe('parseIp', () => { it.each([ @@ -109,3 +109,50 @@ describe('rateKey', () => { expect(rateKey('2001:db8::1')).toMatch(/^6:\d+:\d+$/) }) }) + +describe('truncateIp', () => { + it.each([ + ['198.51.100.77', '198.51.100.0/24'], + ['198.51.100.0', '198.51.100.0/24'], + ['198.51.100.255', '198.51.100.0/24'], + ['192.0.2.1', '192.0.2.0/24'], + ['203.0.113.254', '203.0.113.0/24'], + ])('keeps the network and drops the host of %s', (ip, expected) => { + expect(truncateIp(ip)).toBe(expected) + }) + + it.each([ + ['2001:db8:1234:5678:9abc:def0:1234:5678', '2001:db8:1234:5678::/64'], + ['2001:db8:1234:5678::', '2001:db8:1234:5678::/64'], + ['2001:db8::1', '2001:db8::/64'], + ['::1', '::/64'], + ])('keeps the first 64 bits of %s', (ip, expected) => { + expect(truncateIp(ip)).toBe(expected) + }) + + // The address the visitor was recorded under is an IPv4 address, so what + // comes out is an IPv4 network: one visitor must not appear as two + // different kinds of thing depending on how a proxy spelled it. + it('truncates an IPv4-mapped IPv6 address as IPv4', () => { + expect(truncateIp('::ffff:198.51.100.77')).toBe('198.51.100.0/24') + }) + + // Deliberately unlike canonicalIp, which returns its input unchanged. + // Returning the input is how a whole address escapes if the parser ever + // regresses on a form it used to accept, and this function is the only + // thing between the stored column and a response. + it.each([ + ['an empty string', ''], + ['a host name', 'example.test'], + ['an address with a port', '198.51.100.77:443'], + ['a zone id', '2001:db8::1%eth0'], + ['a CIDR', '198.51.100.0/24'], + ['a sentence', 'not an address at all'], + ])('answers null for %s rather than handing it back', (_label, value) => { + expect(truncateIp(value)).toBeNull() + }) + + it('answers null for a string longer than the longest address', () => { + expect(truncateIp('1'.repeat(46))).toBeNull() + }) +}) diff --git a/packages/ipdata/src/ip.ts b/packages/ipdata/src/ip.ts index 03aa7a8..83d0b9b 100644 --- a/packages/ipdata/src/ip.ts +++ b/packages/ipdata/src/ip.ts @@ -156,3 +156,30 @@ export function rateKey(ip: string): string | null { if (!p) return null return p.v === 4 ? `4:${p.n}` : `6:${p.w[0]}:${p.w[1]}` } + +/** + * The network an address belongs to, as text: /24 for IPv4 and /64 for IPv6. + * + * Every address this install shows anybody goes through here — the click log + * and the CSV export — so that an operator can see a pattern or a bot without + * the file being a file of addresses. The raw column keeps the whole address + * until the retention pass blanks it, because classification and the abuser + * counter need it; nothing reads it back out. + * + * It is one function rather than a rule written once in each reader, and it is + * here rather than in either service, for the same reason `rateKey` is: two + * readers that disagree about the same click are worse than either answer. + * + * **Null, never the input, for anything that is not an address.** `canonicalIp` + * next door returns what it was given when it cannot parse it, which is right + * for a canonicaliser and would be exactly wrong here: it would hand back the + * whole address the moment the parser stopped recognising a form it used to + * accept. Both callers already handle null, because the retention pass blanks + * the column to an empty string. + */ +export function truncateIp(s: string): string | null { + const p = parseIp(s) + if (!p) return null + if (p.v === 4) return `${[p.n >>> 24, (p.n >>> 16) & 0xff, (p.n >>> 8) & 0xff, 0].join('.')}/24` + return `${formatV6([p.w[0], p.w[1], 0, 0])}/64` +} From 0846150128a6e56a8c55ba8d31998e3d1306dc29 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 08:27:29 +0000 Subject: [PATCH 02/65] core: pin the window bound in the unit the check compares in --- packages/core/src/reporting.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/core/src/reporting.test.ts b/packages/core/src/reporting.test.ts index 19d8056..fb9aab0 100644 --- a/packages/core/src/reporting.test.ts +++ b/packages/core/src/reporting.test.ts @@ -4,6 +4,7 @@ import { KEYED_DIMENSIONS, MAX_REPORT_BUCKETS, MAX_REPORT_WINDOW_DAYS, + MAX_REPORT_WINDOW_MS, REPORT_BUCKETS, REPORT_DIMENSIONS, ROLLUP_DIMENSIONS, @@ -45,6 +46,10 @@ describe('bucketCount', () => { expect(MAX_REPORT_BUCKETS).toBe(2000) expect(MAX_REPORT_WINDOW_DAYS).toBe(400) expect(BUCKET_MS).toEqual({ hour: 3_600_000, day: 86_400_000 }) + // The window bound in the unit the check will compare in, written out + // rather than multiplied back up, so a wrong factor is a failure here and + // not a window of 400 hours on an endpoint that says 400 days. + expect(MAX_REPORT_WINDOW_MS).toBe(34_560_000_000) }) }) From 8a440df38beda02032fd5cd5db9233774de6e156 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 08:39:45 +0000 Subject: [PATCH 03/65] core: the ten-year bound, pinned to the number it claims --- packages/core/src/reporting.ts | 7 ++++++- packages/core/src/settings.test.ts | 12 ++++++++++++ packages/ipdata/src/ip.ts | 5 +++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/core/src/reporting.ts b/packages/core/src/reporting.ts index db27eaf..20e91c5 100644 --- a/packages/core/src/reporting.ts +++ b/packages/core/src/reporting.ts @@ -19,7 +19,12 @@ export const REPORT_BUCKETS = ['hour', 'day'] as const export type ReportBucket = (typeof REPORT_BUCKETS)[number] -export const BUCKET_MS: Record = { +/** + * Readonly because `bucketCount` reads it on every call: an assignment to one + * of these two fields anywhere in the process would change every bucket count + * afterwards, and nothing would report it. + */ +export const BUCKET_MS: Readonly> = { hour: 3_600_000, day: 86_400_000, } diff --git a/packages/core/src/settings.test.ts b/packages/core/src/settings.test.ts index 83d4a59..4fc6af6 100644 --- a/packages/core/src/settings.test.ts +++ b/packages/core/src/settings.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { DEFAULT_RETENTION, DEFAULT_TRAFFIC_SETTINGS, + MAX_RETENTION_DAYS, RetentionSettingsSchema, TrafficSettingsSchema, retentionNote, @@ -49,6 +50,17 @@ describe('RetentionSettingsSchema', () => { expect(RetentionSettingsSchema.parse(DEFAULT_RETENTION)).toEqual(DEFAULT_RETENTION) }) + // The refusal below names 3651, which pins the bound only relative to + // wherever it happens to be. Both numbers here are written out, so lowering + // the ceiling — ten years to one, say — fails here rather than passing a + // suite that never mentions the constant. + it('bounds either period at ten years, and accepts exactly that', () => { + expect(MAX_RETENTION_DAYS).toBe(3650) + expect( + RetentionSettingsSchema.safeParse({ rawRetentionDays: 3650, ipRetentionDays: 3650 }).success, + ).toBe(true) + }) + it('takes null for either period, meaning never', () => { const forever = { rawRetentionDays: null, ipRetentionDays: null } expect(RetentionSettingsSchema.parse(forever)).toEqual(forever) diff --git a/packages/ipdata/src/ip.ts b/packages/ipdata/src/ip.ts index 83d0b9b..25399f6 100644 --- a/packages/ipdata/src/ip.ts +++ b/packages/ipdata/src/ip.ts @@ -174,8 +174,9 @@ export function rateKey(ip: string): string | null { * next door returns what it was given when it cannot parse it, which is right * for a canonicaliser and would be exactly wrong here: it would hand back the * whole address the moment the parser stopped recognising a form it used to - * accept. Both callers already handle null, because the retention pass blanks - * the column to an empty string. + * accept. A reader has to have an answer for null regardless of that: the + * retention pass blanks the address column to an empty string, so a click old + * enough to have been blanked arrives here as something that is not an address. */ export function truncateIp(s: string): string | null { const p = parseIp(s) From 5f4fb7ce42c63dad04db5d21873888ac3e3b5b17 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 10:29:52 +0000 Subject: [PATCH 04/65] settings: how long an install keeps what it records, read in one place --- README.md | 2 +- packages/admin/src/settings-routes.test.ts | 111 +++++++++-- packages/admin/src/settings-routes.ts | 100 +++------- packages/cli/src/commands.test.ts | 91 ++++++++- packages/cli/src/commands.ts | 132 +++++-------- packages/core/src/index.ts | 2 +- packages/core/src/settings.ts | 26 ++- packages/core/src/version.test.ts | 2 +- .../postgres/007_reporting_settings.sql | 27 +++ packages/db/src/schema-retention.test.ts | 53 +++++ packages/worker/package.json | 3 +- packages/worker/src/settings.test.ts | 183 +++++++++++++++++ packages/worker/src/settings.ts | 184 ++++++++++++++++++ 13 files changed, 741 insertions(+), 175 deletions(-) create mode 100644 packages/db/migrations/postgres/007_reporting_settings.sql create mode 100644 packages/db/src/schema-retention.test.ts create mode 100644 packages/worker/src/settings.test.ts create mode 100644 packages/worker/src/settings.ts diff --git a/README.md b/README.md index 5a66d53..3ed15ba 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ on your own infrastructure, and your click data stays yours. visitor holds stops working the moment you change the password. - **Traffic classification.** Every click is classed as human, bot, abuser, anonymous (a Tor exit), datacenter, or unknown when the IP checks could not run, from its - user-agent, the number of requests from its address in the current one-minute window, + user-agent, the number of requests from the same client in the current one-minute window, and IP data held on your server. Each class other than human has an action: flag (the default), nothing, block, or send to a safe URL. A flagged click never uses up a click cap, but a cap that is used up closes the link to it like any other. A HEAD request, diff --git a/packages/admin/src/settings-routes.test.ts b/packages/admin/src/settings-routes.test.ts index 49eba6b..3f965c3 100644 --- a/packages/admin/src/settings-routes.test.ts +++ b/packages/admin/src/settings-routes.test.ts @@ -1,4 +1,4 @@ -import { DEFAULT_TRAFFIC_SETTINGS } from '@clickmonk/core' +import { DEFAULT_RETENTION, DEFAULT_TRAFFIC_SETTINGS } from '@clickmonk/core' import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import type { FastifyInstance } from 'fastify' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' @@ -50,7 +50,10 @@ describe('every route needs a credential', () => { { name: 'PUT', method: 'PUT' as const, - payload: { actions: ALL_FLAG, safeUrl: null, abuserThreshold: 99 }, + payload: { + traffic: { actions: ALL_FLAG, safeUrl: null, abuserThreshold: 99 }, + retention: DEFAULT_RETENTION, + }, }, ])('refuses an anonymous $name /api/settings, and changes nothing', async (row) => { const r = await app.inject({ @@ -71,16 +74,21 @@ describe('every route needs a credential', () => { }) describe('reading the settings', () => { - it('reports what the redirect serves', async () => { + it('reports the retention periods and says nothing when they are the usual way round', async () => { const r = await get() expect(r.statusCode).toBe(200) - expect(r.json()).toEqual({ ...DEFAULT_TRAFFIC_SETTINGS, problem: null }) + expect(r.json()).toEqual({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: { rawRetentionDays: 90, ipRetentionDays: 30 }, + note: null, + problem: null, + }) }) it('reports the defaults, and says why, when the row was deleted by hand', async () => { await pg.query('DELETE FROM settings') const r = await get() - expect(r.json().abuserThreshold).toBe(DEFAULT_TRAFFIC_SETTINGS.abuserThreshold) + expect(r.json().traffic.abuserThreshold).toBe(DEFAULT_TRAFFIC_SETTINGS.abuserThreshold) expect(r.json().problem).toContain('no settings are stored') }) @@ -94,18 +102,40 @@ describe('reading the settings', () => { const r = await get() // The field the fixture corrupted, not one the defaults happen to match // anyway: `safeUrl` is what tells a stored row from the defaults here. - expect(r.json().safeUrl).toBe(DEFAULT_TRAFFIC_SETTINGS.safeUrl) - expect(r.json().actions).toEqual(DEFAULT_TRAFFIC_SETTINGS.actions) + expect(r.json().traffic.safeUrl).toBe(DEFAULT_TRAFFIC_SETTINGS.safeUrl) + expect(r.json().traffic.actions).toEqual(DEFAULT_TRAFFIC_SETTINGS.actions) expect(r.json().problem).toContain('invalid') }) + + // A row the retention half cannot be read from is reported as unreadable + // rather than as the defaults: the pass deletes nothing while it is in that + // state, and an operator shown "90 days" could not tell the two apart. + it('reports retention as null, and says why, when it cannot be read', async () => { + await pg.query('ALTER TABLE settings DROP CONSTRAINT settings_raw_retention_valid') + try { + await pg.query('UPDATE settings SET raw_retention_days = -5') + const r = await get() + expect(r.json().retention).toBeNull() + expect(r.json().note).toBeNull() + expect(r.json().problem).toContain('nothing is deleted until it is corrected') + } finally { + await pg.query('UPDATE settings SET raw_retention_days = 90') + await pg.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_raw_retention_valid CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650)', + ) + } + }) }) describe('writing the settings', () => { it('takes the whole object and stores it', async () => { const r = await put({ - actions: { ...ALL_FLAG, bot: 'block' }, - safeUrl: 'https://example.com/safe', - abuserThreshold: 120, + traffic: { + actions: { ...ALL_FLAG, bot: 'block' }, + safeUrl: 'https://example.com/safe', + abuserThreshold: 120, + }, + retention: DEFAULT_RETENTION, }) expect(r.statusCode).toBe(200) const stored = await pg.query<{ @@ -120,11 +150,62 @@ describe('writing the settings', () => { }) }) + it('writes both halves, and notes an address period that outlives its clicks', async () => { + const r = await put({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: { rawRetentionDays: 30, ipRetentionDays: 90 }, + }) + expect(r.statusCode).toBe(200) + expect(r.json().note).toBe( + 'addresses are set to be kept for 90 days but clicks for 30 days, so an address goes when its click does, after 30 days', + ) + const back = await get() + expect(back.json().retention).toEqual({ rawRetentionDays: 30, ipRetentionDays: 90 }) + }) + + it('refuses a body whose retention half is invalid, and writes neither half', async () => { + const r = await put({ + traffic: { ...DEFAULT_TRAFFIC_SETTINGS, abuserThreshold: 120 }, + retention: { rawRetentionDays: 0, ipRetentionDays: 30 }, + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_body') + // The traffic half was valid and is still not written: a settings write is + // one write, and half of one is how an install ends up in a state nobody + // asked for. + const back = await get() + expect(back.json().traffic.abuserThreshold).toBe(60) + expect(back.json().retention).toEqual({ rawRetentionDays: 90, ipRetentionDays: 30 }) + }) + + it('refuses a settings body with a field nobody knows', async () => { + const r = await put({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: { rawRetentionDays: 90, ipRetentionDays: 30 }, + keepEverything: true, + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_body') + }) + + it('refuses a body that leaves a half out, rather than defaulting it', async () => { + const r = await put({ traffic: DEFAULT_TRAFFIC_SETTINGS }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_body') + // Nothing was written: a refusal that had already written the traffic half + // would leave the row half-changed. + const back = await get() + expect(back.json().retention).toEqual({ rawRetentionDays: 90, ipRetentionDays: 30 }) + }) + // The row can be deleted. A write that assumed it exists would write // nothing and report success. it('writes the row back when it is missing', async () => { await pg.query('DELETE FROM settings') - const r = await put({ actions: ALL_FLAG, safeUrl: null, abuserThreshold: 90 }) + const r = await put({ + traffic: { actions: ALL_FLAG, safeUrl: null, abuserThreshold: 90 }, + retention: DEFAULT_RETENTION, + }) expect(r.statusCode).toBe(200) const stored = await pg.query<{ abuser_threshold: number }>( 'SELECT abuser_threshold FROM settings', @@ -133,6 +214,8 @@ describe('writing the settings', () => { expect((await get()).json().problem).toBeNull() }) + // Every row here is a traffic half the inner schema refuses, so each one + // proves that nesting did not stop the half's own schema from running. it.each([ [ 'the safe action with no safe URL', @@ -153,16 +236,16 @@ describe('writing the settings', () => { ], ['a threshold of zero', { actions: ALL_FLAG, safeUrl: null, abuserThreshold: 0 }], ['a threshold past the bound', { actions: ALL_FLAG, safeUrl: null, abuserThreshold: 100_001 }], - ['a partial write', { abuserThreshold: 90 }], + ['a partial traffic half', { abuserThreshold: 90 }], // There is one admin, so nothing in a body may name whose settings these // are. A strict schema is what makes that checkable rather than ignored. [ 'a field it does not know', { actions: ALL_FLAG, safeUrl: null, abuserThreshold: 60, accountId: 'a2f' }, ], - ])('refuses %s', async (_label, payload) => { + ])('refuses %s', async (_label, traffic) => { const before = await get() - const r = await put(payload) + const r = await put({ traffic, retention: DEFAULT_RETENTION }) expect(r.statusCode).toBe(400) expect((await get()).json()).toEqual(before.json()) }) diff --git a/packages/admin/src/settings-routes.ts b/packages/admin/src/settings-routes.ts index cec553c..ad0d0b2 100644 --- a/packages/admin/src/settings-routes.ts +++ b/packages/admin/src/settings-routes.ts @@ -1,91 +1,47 @@ /** - * The install-wide traffic settings. + * The install-wide settings: the traffic actions the redirect evaluates with, + * and how long this install keeps what it records. * - * Two rules the row itself teaches. **Every write upserts, in one statement**: - * the row can be deleted by hand, every reader falls back to the defaults, and - * an endpoint that assumed it exists would write nothing and report success — - * while an insert *and* an update would fire this table's statement trigger - * twice and make the redirect reload its whole snapshot for a second time. And - * **what is read is validated**, not trusted: the column checks are looser - * than `core` in places, so a row written by hand can be refused here and the - * defaults reported instead, exactly as the redirect does with the row it - * loads. + * The row is read and written by one function shared with the command line and + * the retention pass, so the fallback rule — what a missing or unreadable row + * means — exists once. What stays here is how this surface answers it. + * + * **The whole object, never a patch.** The safe action and the safe URL depend + * on each other, and a partial write is how a link ends up set to `safe` with + * nowhere to send a click. The two halves arrive nested because each is + * validated by its own strict schema. */ -import { - DEFAULT_TRAFFIC_SETTINGS, - type TrafficActions, - type TrafficSettings, - TrafficSettingsSchema, -} from '@clickmonk/core' +import { type InstallSettings, InstallSettingsSchema, retentionNote } from '@clickmonk/core' +import { readSettings, writeSettings } from '@clickmonk/worker/settings' import type { FastifyInstance } from 'fastify' import type { AdminContext } from './app.js' import { requireCredential } from './auth.js' import { readBody } from './http.js' -interface SettingsRow { - traffic_actions: TrafficActions - safe_url: string | null - abuser_threshold: number -} - -const toSettings = (r: SettingsRow): TrafficSettings => ({ - actions: r.traffic_actions, - safeUrl: r.safe_url, - abuserThreshold: r.abuser_threshold, -}) - -/** What the redirect serves, and a note when that is not what is stored. */ -export async function servedSettings( - ctx: AdminContext, -): Promise<{ settings: TrafficSettings; problem: string | null }> { - const r = await ctx.pg.query( - 'SELECT traffic_actions, safe_url, abuser_threshold FROM settings', - ) - const row = r.rows[0] - if (!row) { - return { - settings: DEFAULT_TRAFFIC_SETTINGS, - problem: 'no settings are stored; the defaults apply', - } - } - const parsed = TrafficSettingsSchema.safeParse(toSettings(row)) - if (parsed.success) return { settings: parsed.data, problem: null } - const why = parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') - return { - settings: DEFAULT_TRAFFIC_SETTINGS, - problem: `the stored settings are invalid (${why}); the defaults apply`, - } +function body(s: InstallSettings, problem: string | null): Record { + return { traffic: s.traffic, retention: s.retention, note: retentionNote(s.retention), problem } } export function registerSettingsRoutes(app: FastifyInstance, ctx: AdminContext): void { app.get('/api/settings', async (req) => { requireCredential(req) - const { settings, problem } = await servedSettings(ctx) - return { ...settings, problem } + const read = await readSettings(ctx.pg) + // Retention comes back null when the stored row cannot be read as + // retention at all, and that is reported as it is rather than as the + // defaults: the pass deletes nothing while it is in that state, and an + // operator shown "90 days" would have no way to tell. + return { + traffic: read.traffic, + retention: read.retention, + note: read.retention === null ? null : retentionNote(read.retention), + problem: read.problem, + } }) - /** - * The whole object, not a patch: the safe action and the safe URL depend on - * each other, and a partial write is how a link ends up set to `safe` with - * nowhere to send a click. `core`'s schema refuses that pair, and the - * column's own CHECK refuses it again. - */ app.put('/api/settings', async (req) => { requireCredential(req) - const next = readBody(TrafficSettingsSchema, req.body) - // One statement, because `settings` fires `config_changed` from a - // statement trigger: an insert followed by an update would make the - // redirect reload its whole snapshot twice for one write. The row can have - // been deleted by hand, so this upserts rather than assuming it is there — - // an UPDATE alone would match nothing and report success. - await ctx.pg.query( - `INSERT INTO settings (id, traffic_actions, safe_url, abuser_threshold, updated_at) - VALUES (true, $1, $2, $3, $4) - ON CONFLICT (id) DO UPDATE - SET traffic_actions = EXCLUDED.traffic_actions, safe_url = EXCLUDED.safe_url, - abuser_threshold = EXCLUDED.abuser_threshold, updated_at = EXCLUDED.updated_at`, - [JSON.stringify(next.actions), next.safeUrl, next.abuserThreshold, ctx.now()], - ) - return { ...next, problem: null } + const next = readBody(InstallSettingsSchema, req.body) + await writeSettings(ctx.pg, next, ctx.now()) + return body(next, null) }) } diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index db87303..347a509 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -206,6 +206,22 @@ describe('clickmonk settings', () => { const settings = async () => (await pg.query('SELECT traffic_actions, safe_url, abuser_threshold FROM settings')).rows[0] + const stored = async (): Promise<{ + raw_retention_days: number | null + ip_retention_days: number | null + }> => { + const r = await pg.query('SELECT raw_retention_days, ip_retention_days FROM settings') + return r.rows[0] as { raw_retention_days: number | null; ip_retention_days: number | null } + } + + /** + * The retention half put where a test wants it. Written here rather than + * left to whatever the test before it did: these tests share one row, and a + * period is exactly the kind of value a neighbouring test changes. + */ + const setRetention = (raw: number | null, ip: number | null) => + pg.query('UPDATE settings SET raw_retention_days = $1, ip_retention_days = $2', [raw, ip]) + it('shows the defaults', async () => { lines.length = 0 expect(await run('settings', 'show')).toBe(0) @@ -215,7 +231,9 @@ describe('clickmonk settings', () => { 'anonymous: flag', 'datacenter: flag', 'safe url: (none)', - 'abuser threshold: 60 clicks a minute from one address', + 'abuser threshold: 60 requests a minute from one client', + 'keep clicks: 90 days', + 'keep addresses: 30 days', ]) }) @@ -273,7 +291,7 @@ describe('clickmonk settings', () => { lines.length = 0 expect(await run('settings', 'show')).toBe(0) expect(lines[0]).toMatch( - /^note: the stored settings are invalid \(safeUrl: .+\); the defaults apply$/, + /^note: the stored traffic settings are invalid \(safeUrl: .+\); the defaults apply$/, ) expect(lines.slice(1)).toEqual([ 'bot: flag', @@ -281,7 +299,9 @@ describe('clickmonk settings', () => { 'anonymous: flag', 'datacenter: flag', 'safe url: (none)', - 'abuser threshold: 60 clicks a minute from one address', + 'abuser threshold: 60 requests a minute from one client', + 'keep clicks: 90 days', + 'keep addresses: 30 days', ]) }) @@ -296,7 +316,9 @@ describe('clickmonk settings', () => { 'anonymous: flag', 'datacenter: flag', 'safe url: (none)', - 'abuser threshold: 60 clicks a minute from one address', + 'abuser threshold: 60 requests a minute from one client', + 'keep clicks: 90 days', + 'keep addresses: 30 days', ]) }) @@ -310,6 +332,67 @@ describe('clickmonk settings', () => { }) }) + it('prints the retention periods, in days and as for ever', async () => { + await setRetention(90, null) + lines.length = 0 + expect(await run('settings', 'show')).toBe(0) + expect(lines).toContain('keep clicks: 90 days') + expect(lines).toContain('keep addresses: for ever') + expect(lines).toContain( + 'note: addresses are set to be kept for ever but clicks for 90 days, so an address goes when its click does, after 90 days', + ) + }) + + it('sets a period, and takes never for either', async () => { + await setRetention(90, 30) + expect(await run('settings', 'set', '--keep-clicks', '30', '--keep-addresses', '7')).toBe(0) + expect(await stored()).toEqual({ raw_retention_days: 30, ip_retention_days: 7 }) + expect(await run('settings', 'set', '--keep-clicks', 'never')).toBe(0) + expect(await stored()).toEqual({ raw_retention_days: null, ip_retention_days: 7 }) + }) + + it('refuses a period the schema refuses, and writes nothing', async () => { + await setRetention(90, 30) + expect(await run('settings', 'set', '--keep-clicks', '0')).toBe(2) + expect(await stored()).toEqual({ raw_retention_days: 90, ip_retention_days: 30 }) + }) + + it('refuses a period that is not a number at all, and writes nothing', async () => { + await setRetention(90, 30) + expect(await run('settings', 'set', '--keep-addresses', 'forever')).toBe(2) + expect(await stored()).toEqual({ raw_retention_days: 90, ip_retention_days: 30 }) + }) + + it('leaves the period alone when the command is about something else', async () => { + await setRetention(90, 30) + expect(await run('settings', 'set', '--abuser-threshold', '120')).toBe(0) + expect(await stored()).toEqual({ raw_retention_days: 90, ip_retention_days: 30 }) + }) + + // Printing the defaults for a row this build cannot read as retention would + // tell the operator their clicks are deleted after ninety days when nothing + // is being deleted at all. + it('says nothing is being deleted when the stored retention cannot be read', async () => { + await pg.query('ALTER TABLE settings DROP CONSTRAINT settings_raw_retention_valid') + try { + await pg.query('UPDATE settings SET raw_retention_days = -5') + lines.length = 0 + expect(await run('settings', 'show')).toBe(0) + expect(lines).toContain('keep clicks: unreadable, so nothing is being deleted') + expect(lines).toContain('keep addresses: unreadable, so nothing is being deleted') + expect(lines.some((l) => l.startsWith('keep clicks: 90'))).toBe(false) + expect(lines[0]).toMatch(/^note: the stored retention is invalid \(/) + } finally { + // The row is repaired before the constraint goes back: Postgres refuses + // to add a check a stored row already breaks, and every test after this + // one would then be running without it. + await pg.query('UPDATE settings SET raw_retention_days = 90') + await pg.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_raw_retention_valid CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650)', + ) + } + }) + it('stores a link override', async () => { lines.length = 0 expect( diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 75114f2..0ca9e6d 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -15,16 +15,14 @@ import { revokeApiKey, } from '@clickmonk/admin/keys' import { - DEFAULT_TRAFFIC_SETTINGS, + type InstallSettings, MAX_PASSWORD_LENGTH, MIN_ADMIN_PASSWORD_LENGTH, NON_HUMAN_CLASSES, - type TrafficActions, - type TrafficSettings, - TrafficSettingsSchema, isDomainUrl, normaliseHost, parseLinkInput, + retentionNote, verificationRecordName, verificationRecordValue, } from '@clickmonk/core' @@ -47,6 +45,7 @@ import { runDomainChecks, } from '@clickmonk/worker/domains' import { createLink } from '@clickmonk/worker/links' +import { readSettings, updateSettings } from '@clickmonk/worker/settings' import type { ZodError } from 'zod' export interface CliDeps { @@ -94,7 +93,8 @@ const USAGE = `usage: [--action = ...] clickmonk settings show clickmonk settings set [--action = ...] [--safe-url | --no-safe-url] - [--abuser-threshold ] + [--abuser-threshold ] [--keep-clicks |never] + [--keep-addresses |never] clickmonk ipdata update clickmonk ipdata status clickmonk admin create # the password is read from standard input @@ -366,7 +366,7 @@ async function linkAdd(args: string[], d: CliDeps): Promise { d.out(`link ${host}/${created.link.slug} ${created.link.id}`) const safe = NON_HUMAN_CLASSES.filter((c) => input.trafficActions[c] === 'safe') - if (safe.length > 0 && (await servedSettings(d)).settings.safeUrl === null) { + if (safe.length > 0 && (await readSettings(d.pg)).traffic.safeUrl === null) { d.out( `note: ${safe.join(', ')} set to safe, but no safe URL is set, so those clicks are flagged until one is (clickmonk settings set --safe-url )`, ) @@ -610,60 +610,38 @@ async function apikeyRevoke(args: string[], d: CliDeps): Promise { d.out(`key ${id} revoked`) } -function printSettings(s: TrafficSettings, d: CliDeps): void { - for (const c of NON_HUMAN_CLASSES) d.out(`${c}: ${s.actions[c]}`) - d.out(`safe url: ${s.safeUrl ?? '(none)'}`) - d.out(`abuser threshold: ${s.abuserThreshold} clicks a minute from one address`) +function printSettings(s: InstallSettings, d: CliDeps): void { + for (const c of NON_HUMAN_CLASSES) d.out(`${c}: ${s.traffic.actions[c]}`) + d.out(`safe url: ${s.traffic.safeUrl ?? '(none)'}`) + d.out(`abuser threshold: ${s.traffic.abuserThreshold} requests a minute from one client`) + d.out(`keep clicks: ${days(s.retention.rawRetentionDays)}`) + d.out(`keep addresses: ${days(s.retention.ipRetentionDays)}`) + const note = retentionNote(s.retention) + if (note) d.out(`note: ${note}`) } -interface SettingsRow { - traffic_actions: TrafficActions - safe_url: string | null - abuser_threshold: number -} - -const toSettings = (r: SettingsRow): TrafficSettings => ({ - actions: r.traffic_actions, - safeUrl: r.safe_url, - abuserThreshold: r.abuser_threshold, -}) - -/** - * The settings the redirect serves: a row that is missing (deleted by hand) - * or that core's schema refuses means the defaults there, so it does here - * too, with a note saying why. - */ -async function servedSettings(d: CliDeps): Promise<{ settings: TrafficSettings; note?: string }> { - const r = await d.pg.query( - 'SELECT traffic_actions, safe_url, abuser_threshold FROM settings', - ) - const row = r.rows[0] - if (!row) { - return { - settings: DEFAULT_TRAFFIC_SETTINGS, - note: 'note: no settings are stored; the defaults apply', - } - } - const parsed = TrafficSettingsSchema.safeParse(toSettings(row)) - if (parsed.success) return { settings: parsed.data } - const why = parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') - return { - settings: DEFAULT_TRAFFIC_SETTINGS, - note: `note: the stored settings are invalid (${why}); the defaults apply`, - } -} +const days = (n: number | null): string => (n === null ? 'for ever' : `${n} days`) async function settingsShow(d: CliDeps): Promise { - const { settings, note } = await servedSettings(d) - if (note) d.out(note) - printSettings(settings, d) + const read = await readSettings(d.pg) + if (read.problem) d.out(`note: ${read.problem}`) + if (read.retention === null) { + // Printing the defaults here would tell the operator this install is + // deleting after ninety days, when in fact it is deleting nothing. + for (const c of NON_HUMAN_CLASSES) d.out(`${c}: ${read.traffic.actions[c]}`) + d.out(`safe url: ${read.traffic.safeUrl ?? '(none)'}`) + d.out(`abuser threshold: ${read.traffic.abuserThreshold} requests a minute from one client`) + d.out('keep clicks: unreadable, so nothing is being deleted') + d.out('keep addresses: unreadable, so nothing is being deleted') + return + } + printSettings({ traffic: read.traffic, retention: read.retention }, d) } /** * Changes only what is given; the result is validated whole before anything - * is written. A row deleted by hand is first written back as the defaults, - * in the same transaction, so that concurrent writers still serialise on its - * lock. + * is written. The read, the merge and the write are the shared ones, so a + * period set here is the period the retention pass then enforces. */ async function settingsSet(args: string[], d: CliDeps): Promise { const { values } = parseArgs({ @@ -673,42 +651,36 @@ async function settingsSet(args: string[], d: CliDeps): Promise { 'safe-url': { type: 'string' }, 'no-safe-url': { type: 'boolean' }, 'abuser-threshold': { type: 'string' }, + 'keep-clicks': { type: 'string' }, + 'keep-addresses': { type: 'string' }, }, }) if (values['safe-url'] !== undefined && values['no-safe-url']) { throw new Rejected('--safe-url and --no-safe-url together') } - const client = await d.pg.connect() - try { - await client.query('BEGIN') - await client.query('INSERT INTO settings DEFAULT VALUES ON CONFLICT DO NOTHING') - const r = await client.query( - 'SELECT traffic_actions, safe_url, abuser_threshold FROM settings FOR UPDATE', - ) - const row = r.rows[0] - if (!row) throw new Error('the settings row is missing after writing it') - const current = toSettings(row) - const next = TrafficSettingsSchema.parse({ - actions: { ...current.actions, ...parseActions(values.action) }, - safeUrl: values['no-safe-url'] ? null : (values['safe-url'] ?? current.safeUrl), + // `never` is the only word either takes, and it is the stored null. A + // number is passed through as written so that core's schema is what + // refuses 0, -1 and 1.5, in one place, with one message. + const period = (given: string | undefined, current: number | null): number | null => { + if (given === undefined) return current + if (given === 'never') return null + return Number(given) + } + const next = await updateSettings(d.pg, d.now?.() ?? new Date(), (current) => ({ + traffic: { + actions: { ...current.traffic.actions, ...parseActions(values.action) }, + safeUrl: values['no-safe-url'] ? null : (values['safe-url'] ?? current.traffic.safeUrl), abuserThreshold: values['abuser-threshold'] === undefined - ? current.abuserThreshold + ? current.traffic.abuserThreshold : Number(values['abuser-threshold']), - }) - await client.query( - `UPDATE settings SET traffic_actions = $1, safe_url = $2, abuser_threshold = $3, - updated_at = now()`, - [JSON.stringify(next.actions), next.safeUrl, next.abuserThreshold], - ) - await client.query('COMMIT') - printSettings(next, d) - } catch (err) { - await client.query('ROLLBACK').catch(() => {}) - throw err - } finally { - client.release() - } + }, + retention: { + rawRetentionDays: period(values['keep-clicks'], current.retention.rawRetentionDays), + ipRetentionDays: period(values['keep-addresses'], current.retention.ipRetentionDays), + }, + })) + printSettings(next, d) } /** diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9327175..fc2db88 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,7 +2,7 @@ * The highest migration version this build understands. Every migration bumps * it; `migrate()` refuses to run against a database whose ledger is newer. */ -export const SCHEMA_VERSION = 6 +export const SCHEMA_VERSION = 7 export * from './admin-host.js' export * from './attempts.js' diff --git a/packages/core/src/settings.ts b/packages/core/src/settings.ts index 38fc5ba..b4425df 100644 --- a/packages/core/src/settings.ts +++ b/packages/core/src/settings.ts @@ -8,7 +8,7 @@ import { } from './traffic.js' /** - * Clicks per minute from one client above which it is an abuser, unless the admin + * Requests per minute from one client above which it is an abuser, unless the admin * sets another. A client is what `rateKey` counts: one IPv4 address, or one IPv6 * /64, because an IPv6 client usually holds a whole /64. */ @@ -105,3 +105,27 @@ export function retentionNote(r: RetentionSettings): string | null { const kept = r.ipRetentionDays === null ? 'for ever' : `for ${r.ipRetentionDays} days` return `addresses are set to be kept ${kept} but clicks for ${r.rawRetentionDays} days, so an address goes when its click does, after ${r.rawRetentionDays} days` } + +/** + * Everything the one settings row holds, in two halves. + * + * Two halves rather than one flat object because each half is `.strict()` and + * an intersection of two strict objects refuses every field, so a flat body + * would have to restate one of the schemas to accept the other's fields — + * which is exactly the copy this plan exists to avoid. Nested, each half is + * validated by the schema that owns it, and a problem can say which half it + * is in. + */ +export interface InstallSettings { + traffic: TrafficSettings + retention: RetentionSettings +} + +export const DEFAULT_INSTALL_SETTINGS: InstallSettings = { + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: DEFAULT_RETENTION, +} + +export const InstallSettingsSchema = z + .object({ traffic: TrafficSettingsSchema, retention: RetentionSettingsSchema }) + .strict() diff --git a/packages/core/src/version.test.ts b/packages/core/src/version.test.ts index 1bb851e..3582dd4 100644 --- a/packages/core/src/version.test.ts +++ b/packages/core/src/version.test.ts @@ -5,6 +5,6 @@ describe('SCHEMA_VERSION', () => { it('is the highest migration version this build knows about', () => { // Bumped with every migration added. packages/db's // schema-version.test.ts ties it to the migration files on disk. - expect(SCHEMA_VERSION).toBe(6) + expect(SCHEMA_VERSION).toBe(7) }) }) diff --git a/packages/db/migrations/postgres/007_reporting_settings.sql b/packages/db/migrations/postgres/007_reporting_settings.sql new file mode 100644 index 0000000..5c9bd4d --- /dev/null +++ b/packages/db/migrations/postgres/007_reporting_settings.sql @@ -0,0 +1,27 @@ +-- How long this install keeps the two things it can stop keeping. Both are +-- an operator's policy rather than a deployment's, which is why they are on +-- this row and not in the environment, and it is the same argument that put +-- the traffic actions here. +-- +-- NULL means for ever, for either. Zero was rejected as the forever value +-- because it reads equally as "drop everything now", and the difference +-- between those two readings is every click this install has. +-- +-- Only `settings` is altered. The redirect's snapshot load reads links, then +-- domains, then settings in one transaction, so this takes an exclusive lock +-- on exactly one of the tables that reload reads — and the last of them — so +-- it can block a reload but can never be half of a deadlock. +-- +-- Deliberately NOT a check that the IP period is the shorter of the two. It +-- usually is, and an IP period that outlives the clicks it belongs to simply +-- never runs, but a constraint saying so would refuse an operator who is +-- *lowering* the raw period below the IP period — the tightening direction — +-- and being refused for keeping less is the wrong way round. The API and the +-- CLI say so in a note instead. +ALTER TABLE settings + ADD COLUMN raw_retention_days integer DEFAULT 90, + ADD COLUMN ip_retention_days integer DEFAULT 30, + ADD CONSTRAINT settings_raw_retention_valid + CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650), + ADD CONSTRAINT settings_ip_retention_valid + CHECK (ip_retention_days IS NULL OR ip_retention_days BETWEEN 1 AND 3650); diff --git a/packages/db/src/schema-retention.test.ts b/packages/db/src/schema-retention.test.ts new file mode 100644 index 0000000..aa54c53 --- /dev/null +++ b/packages/db/src/schema-retention.test.ts @@ -0,0 +1,53 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { resetDatabases, testCh, testPg } from './testing.js' + +const pool = testPg() +const ch = testCh() + +beforeAll(async () => { + await resetDatabases(pool, ch) +}) + +beforeEach(async () => { + await pool.query('TRUNCATE settings') + await pool.query('INSERT INTO settings DEFAULT VALUES') +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +const set = (raw: number | null, ip: number | null) => + pool.query('UPDATE settings SET raw_retention_days = $1, ip_retention_days = $2', [raw, ip]) + +describe('postgres schema 007: retention', () => { + it('defaults to ninety days of clicks and thirty of addresses', async () => { + const r = await pool.query('SELECT raw_retention_days, ip_retention_days FROM settings') + expect(r.rows).toEqual([{ raw_retention_days: 90, ip_retention_days: 30 }]) + }) + + it('takes null for either, meaning for ever', async () => { + await set(null, null) + const r = await pool.query('SELECT raw_retention_days, ip_retention_days FROM settings') + expect(r.rows).toEqual([{ raw_retention_days: null, ip_retention_days: null }]) + }) + + it.each([ + ['zero clicks', 0, 30], + ['zero days of addresses', 90, 0], + ['a negative period', -1, 30], + ['more than ten years', 3651, 30], + ['more than ten years of addresses', 90, 3651], + ])('refuses %s', async (_label, raw, ip) => { + await expect(set(raw, ip)).rejects.toThrow(/settings_(raw|ip)_retention_valid/) + }) + + // The write gate for anything that bypasses the application is the CHECK, + // not the schema in TypeScript: hand-written SQL reaches this row. + it('accepts an IP period longer than the raw one rather than refusing it', async () => { + await set(30, 90) + const r = await pool.query('SELECT raw_retention_days, ip_retention_days FROM settings') + expect(r.rows).toEqual([{ raw_retention_days: 30, ip_retention_days: 90 }]) + }) +}) diff --git a/packages/worker/package.json b/packages/worker/package.json index 01bcd8a..7e34291 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -7,7 +7,8 @@ "exports": { ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }, "./domains": { "types": "./dist/domains.d.ts", "default": "./dist/domains.js" }, - "./links": { "types": "./dist/links.d.ts", "default": "./dist/links.js" } + "./links": { "types": "./dist/links.d.ts", "default": "./dist/links.js" }, + "./settings": { "types": "./dist/settings.d.ts", "default": "./dist/settings.js" } }, "engines": { "node": ">=22 <23" }, "scripts": { "build": "tsc -b", "build:image": "tsc -b tsconfig.build.json" }, diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts new file mode 100644 index 0000000..d5c04ef --- /dev/null +++ b/packages/worker/src/settings.test.ts @@ -0,0 +1,183 @@ +import { DEFAULT_INSTALL_SETTINGS, DEFAULT_TRAFFIC_SETTINGS } from '@clickmonk/core' +import type { Pool } from '@clickmonk/db' +import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { readSettings, updateSettings, writeSettings } from './settings.js' + +const pool: Pool = testPg() +const ch = testCh() +const NOW = new Date('2026-09-24T12:00:00.000Z') + +beforeAll(async () => { + await resetDatabases(pool, ch) +}) + +beforeEach(async () => { + await pool.query('TRUNCATE settings') + await pool.query('INSERT INTO settings DEFAULT VALUES') +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +describe('readSettings', () => { + it('reads the stored row as both halves, with nothing to report', async () => { + expect(await readSettings(pool)).toEqual({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: { rawRetentionDays: 90, ipRetentionDays: 30 }, + problem: null, + }) + }) + + it('reads a row an operator changed', async () => { + await pool.query('UPDATE settings SET raw_retention_days = 400, ip_retention_days = NULL') + const r = await readSettings(pool) + expect(r.retention).toEqual({ rawRetentionDays: 400, ipRetentionDays: null }) + expect(r.problem).toBeNull() + }) + + it('falls back to the defaults and says so when there is no row at all', async () => { + await pool.query('TRUNCATE settings') + expect(await readSettings(pool)).toEqual({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: { rawRetentionDays: 90, ipRetentionDays: 30 }, + problem: 'no settings are stored; the defaults apply', + }) + }) + + /** + * The rule this module exists to hold. A default is a safe answer when the + * thing it stands in for is an action — flag rather than block. A retention + * default is not: applying it deletes clicks. So a row this build cannot + * read as retention answers null, and the pass that reads it does nothing. + */ + it('answers null for retention it cannot read, rather than the defaults', async () => { + // Past the column CHECK, which is what hand-written SQL that dropped the + // constraint would leave behind. Written by dropping it here, because the + // constraint is doing its job and this is the case where it is not there. + await pool.query('ALTER TABLE settings DROP CONSTRAINT settings_raw_retention_valid') + try { + await pool.query('UPDATE settings SET raw_retention_days = -5') + const r = await readSettings(pool) + expect(r.retention).toBeNull() + expect(r.problem).toBe( + 'the stored retention is invalid (rawRetentionDays: Number must be greater than or equal to 1); nothing is deleted until it is corrected', + ) + // The other half still reads, because one bad column must not take the + // traffic settings down with it. + expect(r.traffic).toEqual(DEFAULT_TRAFFIC_SETTINGS) + } finally { + // In a `finally`, because a failing assertion here would otherwise leave + // this file's database without the constraint for every test after it, + // and `resetDatabases` only runs in `beforeAll`. The row is put back + // first: a constraint added over a row that still breaks it is refused, + // and then nothing after this point has it. + await pool.query('UPDATE settings SET raw_retention_days = 90') + await pool.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_raw_retention_valid CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650)', + ) + } + }) + + it('falls back to the default traffic settings without touching retention', async () => { + // Retention is moved off its defaults first, so that the assertion below + // is about the retention half surviving rather than about the defaults + // happening to match: a build that answered the defaults for both halves + // whenever either failed would agree with an unchanged fixture. + await pool.query('UPDATE settings SET raw_retention_days = 400, ip_retention_days = NULL') + await pool.query('ALTER TABLE settings DROP CONSTRAINT settings_traffic_actions_check') + try { + await pool.query(`UPDATE settings SET traffic_actions = '{"bot":"drop"}'::jsonb`) + const r = await readSettings(pool) + expect(r.traffic).toEqual(DEFAULT_TRAFFIC_SETTINGS) + expect(r.retention).toEqual({ rawRetentionDays: 400, ipRetentionDays: null }) + expect(r.problem).toMatch(/^the stored traffic settings are invalid \(/) + } finally { + await pool.query( + `UPDATE settings SET traffic_actions = + '{"bot":"flag","abuser":"flag","anonymous":"flag","datacenter":"flag"}'::jsonb`, + ) + await pool.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_traffic_actions_check CHECK (valid_traffic_actions(traffic_actions, true))', + ) + } + }) +}) + +describe('writeSettings', () => { + it('writes both halves in one statement, and writes the row back when it is gone', async () => { + await pool.query('TRUNCATE settings') + await writeSettings( + pool, + { + traffic: { ...DEFAULT_TRAFFIC_SETTINGS, abuserThreshold: 120 }, + retention: { rawRetentionDays: 7, ipRetentionDays: null }, + }, + NOW, + ) + const r = await readSettings(pool) + expect(r.traffic.abuserThreshold).toBe(120) + expect(r.retention).toEqual({ rawRetentionDays: 7, ipRetentionDays: null }) + const rows = await pool.query( + 'SELECT count(*)::int AS n, updated_at FROM settings GROUP BY updated_at', + ) + expect(rows.rows).toEqual([{ n: 1, updated_at: NOW }]) + }) +}) + +describe('updateSettings', () => { + it('changes only what the caller changed', async () => { + const next = await updateSettings(pool, NOW, (current) => ({ + ...current, + retention: { ...current.retention, ipRetentionDays: 1 }, + })) + expect(next.retention).toEqual({ rawRetentionDays: 90, ipRetentionDays: 1 }) + expect(next.traffic).toEqual(DEFAULT_TRAFFIC_SETTINGS) + expect((await readSettings(pool)).retention).toEqual({ + rawRetentionDays: 90, + ipRetentionDays: 1, + }) + }) + + it('refuses a change that does not validate, and writes nothing', async () => { + await expect( + updateSettings(pool, NOW, (current) => ({ + ...current, + retention: { ...current.retention, rawRetentionDays: 0 }, + })), + ).rejects.toThrow() + expect((await readSettings(pool)).retention).toEqual({ + rawRetentionDays: 90, + ipRetentionDays: 30, + }) + // And the client went back to the pool with nothing open. Re-reading the + // row is not enough on its own: the next read is usually served by the + // very connection that leaked, which sees its own uncommitted work and + // answers as if the rollback had happened. A second connection is what + // can see a backend left sitting inside a transaction. + const probe = testPg() + try { + const open = await probe.query<{ n: number }>( + `SELECT count(*)::int AS n FROM pg_stat_activity + WHERE datname = current_database() AND state LIKE 'idle in transaction%'`, + ) + expect(open.rows[0]?.n).toBe(0) + } finally { + await probe.end() + } + }) + + it('starts from the defaults when the row was deleted by hand', async () => { + await pool.query('TRUNCATE settings') + const next = await updateSettings(pool, NOW, (current) => ({ + ...current, + retention: { ...current.retention, rawRetentionDays: 5 }, + })) + expect(next).toEqual({ + traffic: DEFAULT_INSTALL_SETTINGS.traffic, + retention: { rawRetentionDays: 5, ipRetentionDays: 30 }, + }) + }) +}) diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts new file mode 100644 index 0000000..1bed077 --- /dev/null +++ b/packages/worker/src/settings.ts @@ -0,0 +1,184 @@ +/** + * The install's settings row: one reader, one writer, one merge. + * + * Three copies of "read the row, fall back to the defaults, say why" existed + * or were about to — the command line's, the admin API's and the retention + * pass's — and three copies of a fallback rule is where they begin to + * disagree about what an install is actually doing. This is the one. It lives + * in the worker package because that is where the other functions two + * surfaces share already live. + * + * The redirect is the deliberate exception. It reads these settings inside + * the snapshot's single REPEATABLE READ READ ONLY transaction, together with + * the domains and links, so that the configuration it serves is one + * consistent view; a pool-level helper cannot join that transaction, and + * splitting the read out of it would be a worse trade than the copy. + */ +import { + DEFAULT_INSTALL_SETTINGS, + DEFAULT_RETENTION, + DEFAULT_TRAFFIC_SETTINGS, + type InstallSettings, + InstallSettingsSchema, + type RetentionSettings, + RetentionSettingsSchema, + type TrafficActions, + type TrafficSettings, + TrafficSettingsSchema, +} from '@clickmonk/core' +import type { Pool, PoolClient } from '@clickmonk/db' + +interface SettingsRow { + traffic_actions: TrafficActions + safe_url: string | null + abuser_threshold: number + raw_retention_days: number | null + ip_retention_days: number | null +} + +const COLUMNS = 'traffic_actions, safe_url, abuser_threshold, raw_retention_days, ip_retention_days' + +export interface SettingsRead { + traffic: TrafficSettings + /** + * Null when the stored row cannot be read as retention at all. Not the + * defaults: applying a retention default deletes clicks, and it is the only + * default in this product that does. A caller that deletes things does + * nothing while this is null. + */ + retention: RetentionSettings | null + problem: string | null +} + +const why = (issues: { path: (string | number)[]; message: string }[]): string => + issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') + +/** Each half validated on its own, so one bad column cannot take the other down. */ +function parseRow(row: SettingsRow): SettingsRead { + const traffic = TrafficSettingsSchema.safeParse({ + actions: row.traffic_actions, + safeUrl: row.safe_url, + abuserThreshold: row.abuser_threshold, + }) + const retention = RetentionSettingsSchema.safeParse({ + rawRetentionDays: row.raw_retention_days, + ipRetentionDays: row.ip_retention_days, + }) + const problems: string[] = [] + if (!traffic.success) { + problems.push( + `the stored traffic settings are invalid (${why(traffic.error.issues)}); the defaults apply`, + ) + } + if (!retention.success) { + problems.push( + `the stored retention is invalid (${why(retention.error.issues)}); nothing is deleted until it is corrected`, + ) + } + return { + traffic: traffic.success ? traffic.data : DEFAULT_TRAFFIC_SETTINGS, + retention: retention.success ? retention.data : null, + problem: problems.length === 0 ? null : problems.join(' '), + } +} + +/** What this install is actually running, and anything wrong with the row it came from. */ +export async function readSettings(pg: Pool): Promise { + const r = await pg.query(`SELECT ${COLUMNS} FROM settings`) + const row = r.rows[0] + if (!row) { + return { + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: DEFAULT_RETENTION, + problem: 'no settings are stored; the defaults apply', + } + } + return parseRow(row) +} + +/** + * Writes the whole row. + * + * One statement, because `settings` fires `config_changed` from a statement + * trigger: an insert followed by an update would make the redirect reload its + * whole snapshot twice for one write. It upserts rather than assuming the row + * is there, because it can be deleted by hand and an UPDATE alone would match + * nothing and report success. + */ +export async function writeSettings(pg: Pool, next: InstallSettings, now: Date): Promise { + await pg.query( + `INSERT INTO settings (id, traffic_actions, safe_url, abuser_threshold, + raw_retention_days, ip_retention_days, updated_at) + VALUES (true, $1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE + SET traffic_actions = EXCLUDED.traffic_actions, safe_url = EXCLUDED.safe_url, + abuser_threshold = EXCLUDED.abuser_threshold, + raw_retention_days = EXCLUDED.raw_retention_days, + ip_retention_days = EXCLUDED.ip_retention_days, + updated_at = EXCLUDED.updated_at`, + [ + JSON.stringify(next.traffic.actions), + next.traffic.safeUrl, + next.traffic.abuserThreshold, + next.retention.rawRetentionDays, + next.retention.ipRetentionDays, + now, + ], + ) +} + +/** + * Read, change, validate, write — with the row locked for the whole of it, so + * two callers changing different fields cannot lose one of the changes. + * + * The row is written back as the defaults first when it is missing, in the + * same transaction, so that concurrent writers still serialise on its lock. + * That costs one extra `config_changed` on a command an operator typed, which + * is cheaper than a read-modify-write with nothing to lock. + */ +export async function updateSettings( + pg: Pool, + now: Date, + change: (current: InstallSettings) => InstallSettings, +): Promise { + const client: PoolClient = await pg.connect() + try { + await client.query('BEGIN') + await client.query('INSERT INTO settings DEFAULT VALUES ON CONFLICT DO NOTHING') + const r = await client.query(`SELECT ${COLUMNS} FROM settings FOR UPDATE`) + const row = r.rows[0] + if (!row) throw new Error('the settings row is missing after writing it') + const read = parseRow(row) + const current: InstallSettings = { + traffic: read.traffic, + // A row this build cannot read as retention starts from the defaults + // *for a deliberate write*, which is the one place that is right: the + // operator is here, changing it, and the alternative is a command that + // cannot repair the row it is complaining about. + retention: read.retention ?? DEFAULT_INSTALL_SETTINGS.retention, + } + const next = InstallSettingsSchema.parse(change(current)) + await client.query( + `UPDATE settings SET traffic_actions = $1, safe_url = $2, abuser_threshold = $3, + raw_retention_days = $4, ip_retention_days = $5, updated_at = $6`, + [ + JSON.stringify(next.traffic.actions), + next.traffic.safeUrl, + next.traffic.abuserThreshold, + next.retention.rawRetentionDays, + next.retention.ipRetentionDays, + now, + ], + ) + await client.query('COMMIT') + return next + } catch (err) { + // Every exit path rolls back before the client goes back to the pool: a + // client released inside an open transaction is a connection the next + // caller inherits mid-transaction. + await client.query('ROLLBACK').catch(() => {}) + throw err + } finally { + client.release() + } +} From 78c26c13cc1e2a78a785080c9b33a2c77b3a7c26 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 10:32:25 +0000 Subject: [PATCH 05/65] settings: pin the merge's validation on a change only core refuses --- packages/worker/src/settings.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts index d5c04ef..c8083e7 100644 --- a/packages/worker/src/settings.test.ts +++ b/packages/worker/src/settings.test.ts @@ -142,6 +142,18 @@ describe('updateSettings', () => { }) it('refuses a change that does not validate, and writes nothing', async () => { + // A safe URL with a token in its host first: the column CHECK takes it + // (printable ASCII, an http scheme) and core does not, so it is a change + // only the schema in front of the write can refuse. A period of 0 alone + // would not pin that schema at all — the column refuses it as well, so a + // build that had stopped validating would still be rejected, by Postgres. + await expect( + updateSettings(pool, NOW, (current) => ({ + ...current, + traffic: { ...current.traffic, safeUrl: 'https://{param:h}/safe' }, + })), + ).rejects.toThrow() + expect((await readSettings(pool)).traffic.safeUrl).toBeNull() await expect( updateSettings(pool, NOW, (current) => ({ ...current, From 2d9ece81a93d6045faf5715650145e487460b2a8 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 10:48:00 +0000 Subject: [PATCH 06/65] admin: expiry a frozen clock can see --- packages/admin/src/account.test.ts | 10 +++++++- packages/admin/src/auth.test.ts | 10 ++++++-- packages/cli/src/commands.test.ts | 38 ++++++++++++++++++++-------- packages/worker/src/settings.test.ts | 7 +++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/packages/admin/src/account.test.ts b/packages/admin/src/account.test.ts index 21685ac..34ec911 100644 --- a/packages/admin/src/account.test.ts +++ b/packages/admin/src/account.test.ts @@ -326,7 +326,15 @@ describe('signing in', () => { // the name is still there in front of an empty value. expect(cookie.startsWith(`${SESSION_COOKIE}=`)).toBe(true) expect(cookie.slice(SESSION_COOKIE.length + 1).length).toBeGreaterThan(20) - await pg.query("UPDATE sessions SET expires_at = now() - interval '1 day'") + // A day before the clock this app was built with, not before SQL `now()`. + // The two agree only while the real date sits near the frozen one, so a + // fixture written with `now() - interval '1 day'` stops looking expired + // the moment the real clock walks past the frozen clock by a day — and + // then this test fails every run, for a reason that has nothing to do + // with the sweep it is about. + await pg.query('UPDATE sessions SET expires_at = $1', [ + new Date(clock.now().getTime() - 86_400_000), + ]) await signInWith({ email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) const rows = await pg.query<{ n: number }>('SELECT count(*)::int AS n FROM sessions') expect(rows.rows[0]?.n).toBe(1) diff --git a/packages/admin/src/auth.test.ts b/packages/admin/src/auth.test.ts index 187d068..6b03f7d 100644 --- a/packages/admin/src/auth.test.ts +++ b/packages/admin/src/auth.test.ts @@ -505,9 +505,15 @@ describe('an API key', () => { // expired key's id, the row would pass whether the secret was compared or // not — which is what made this assertion pin nothing. const good = await keyFor(pg, 'good') - await pg.query('UPDATE api_keys SET revoked_at = now() WHERE name = $1', ['revoked']) - await pg.query("UPDATE api_keys SET expires_at = now() - interval '1 day' WHERE name = $1", [ + // Both stamps come from the clock this app was built with, not from SQL + // `now()`. The key check compares `expires_at` against that clock, so a + // fixture a day before the *database's* clock stops being in the past as + // soon as the real date walks a day past the frozen one, and this test + // then fails every run over a date rather than over the check it is about. + await pg.query('UPDATE api_keys SET revoked_at = $2 WHERE name = $1', ['revoked', clock.now()]) + await pg.query('UPDATE api_keys SET expires_at = $2 WHERE name = $1', [ 'live', + new Date(clock.now().getTime() - 86_400_000), ]) const expired = live // That live key's own id, with 43 characters of the wrong secret. diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index 347a509..fab0443 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -893,10 +893,13 @@ describe('the admin account and API keys from the CLI', () => { it('changes the password and signs every browser out', async () => { await withPassword('a decent admin password', 'admin', 'create', 'admin@example.com') const before = (await account()).rows[0]?.password_hash as string - await pg.query( - "INSERT INTO sessions (token_hash, expires_at) VALUES ($1, now() + interval '1 day')", - ['a'.repeat(64)], - ) + // Live at NOW, which is the clock every command in this block runs on. A + // day past SQL `now()` is a day past the *database's* clock, and the two + // only agree while the real date sits near the frozen one. + await pg.query('INSERT INTO sessions (token_hash, expires_at) VALUES ($1, $2)', [ + 'a'.repeat(64), + new Date(NOW.getTime() + 86_400_000), + ]) lines.length = 0 expect(await withPassword('a new decent password', 'admin', 'passwd')).toBe(0) expect(lines.join('\n')).toContain('1 session(s) signed out') @@ -911,10 +914,21 @@ describe('the admin account and API keys from the CLI', () => { // once, rather than waiting out a lock on a password that no longer exists. it('clears a standing lockout, so the new password works at once', async () => { await withPassword('a decent admin password', 'admin', 'create', 'admin@example.com') + // Both stamps from NOW, the clock this block's commands run on: a lockout + // an hour past SQL `now()` is not standing at NOW once the real date has + // walked past the frozen one, and this test would then be clearing a lock + // that was never holding anything shut. await pg.query( - `UPDATE admin_account SET failed_logins = 9, last_failed_at = now(), - locked_until = now() + interval '1 hour'`, + `UPDATE admin_account SET failed_logins = 9, last_failed_at = $1, + locked_until = $2`, + [NOW, new Date(NOW.getTime() + 3_600_000)], ) + // It really is standing, before the command that clears it: without this, + // the fixture could sit on either side of NOW and every assertion below + // would still pass. + expect( + await signIn(pg, { email: 'admin@example.com', password: 'a decent admin password' }, NOW), + ).toEqual({ ok: false, reason: 'locked', retryAfterSeconds: 3600 }) expect(await withPassword('a new decent password', 'admin', 'passwd')).toBe(0) const r = await pg.query<{ failed_logins: number @@ -970,10 +984,14 @@ describe('the admin account and API keys from the CLI', () => { NOW, ]) } - await pg.query( - "INSERT INTO sessions (token_hash, expires_at) VALUES ($1, now() + interval '1 day')", - ['b'.repeat(64)], - ) + // From NOW, like the recovery codes and the lockout above it: this test + // reads the account through `signIn(…, NOW)`, so one fixture on the + // database's clock and the rest on the frozen one is a mixture that only + // holds while the two dates are close. + await pg.query('INSERT INTO sessions (token_hash, expires_at) VALUES ($1, $2)', [ + 'b'.repeat(64), + new Date(NOW.getTime() + 86_400_000), + ]) const credentials = { email: 'admin@example.com', password: 'a decent admin password' } // Before: the password alone gets nowhere, and the lockout the lost codes // earned is standing. diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts index c8083e7..23fd846 100644 --- a/packages/worker/src/settings.test.ts +++ b/packages/worker/src/settings.test.ts @@ -169,6 +169,13 @@ describe('updateSettings', () => { // very connection that leaked, which sees its own uncommitted work and // answers as if the rollback had happened. A second connection is what // can see a backend left sitting inside a transaction. + // + // **Do not narrow this to one connection.** It counts every backend on + // this database on purpose: the leaked one cannot be addressed from here, + // because the pool hands the same client straight back to whoever asks + // next. Counting database-wide is exact under this repository's rule that + // one suite runs at a time, and nothing narrower fails when the ROLLBACK + // is removed. const probe = testPg() try { const open = await probe.query<{ n: number }>( From d1389fe91bd006afadf283aa2b429b6aecfeb8d7 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 10:51:48 +0000 Subject: [PATCH 07/65] settings: say which period a set command rewrote --- packages/cli/src/commands.test.ts | 38 ++++++++++++++++++++++ packages/cli/src/commands.ts | 46 ++++++++++++++++++-------- packages/worker/src/settings.ts | 54 +++++++++++++++++++++++++------ 3 files changed, 115 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index fab0443..1948339 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -369,6 +369,44 @@ describe('clickmonk settings', () => { expect(await stored()).toEqual({ raw_retention_days: 90, ip_retention_days: 30 }) }) + // A command about the abuser threshold rewrites an unreadable retention + // period as a side effect, because the alternative is refusing an operator + // who needs the threshold changed on exactly the install whose retention is + // broken. So it says what it did: a period learned from a line of output + // beats one learned from data that is no longer there. + it('says which period it rewrote when the stored retention could not be read', async () => { + await pg.query('ALTER TABLE settings DROP CONSTRAINT settings_raw_retention_valid') + try { + await pg.query('UPDATE settings SET raw_retention_days = -5, ip_retention_days = 7') + lines.length = 0 + expect(await run('settings', 'set', '--abuser-threshold', '120')).toBe(0) + expect(lines[0]).toBe( + 'note: the stored retention could not be read (rawRetentionDays: Number must be greater than or equal to 1), so this command has rewritten it as keep clicks 90 days, keep addresses 30 days', + ) + // **Both** periods, not only the unreadable one: retention is validated + // as one half by one strict schema, so a single bad column loses the + // other with it. The address period stored here was 7, and it is 30 + // afterwards — which is the whole reason this line has to be printed. + expect(await stored()).toEqual({ raw_retention_days: 90, ip_retention_days: 30 }) + // And the change the operator actually asked for landed. + expect(lines).toContain('abuser threshold: 120 requests a minute from one client') + } finally { + await pg.query('UPDATE settings SET raw_retention_days = 90, ip_retention_days = 30') + await pg.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_raw_retention_valid CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650)', + ) + } + }) + + // A command that changed nothing about retention says nothing about it: the + // line above is a report of a rewrite, not a banner. + it('says nothing about retention when the stored row reads fine', async () => { + await setRetention(90, 30) + lines.length = 0 + expect(await run('settings', 'set', '--abuser-threshold', '60')).toBe(0) + expect(lines.some((l) => l.includes('could not be read'))).toBe(false) + }) + // Printing the defaults for a row this build cannot read as retention would // tell the operator their clicks are deleted after ninety days when nothing // is being deleted at all. diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 0ca9e6d..b2df0d0 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -666,20 +666,38 @@ async function settingsSet(args: string[], d: CliDeps): Promise { if (given === 'never') return null return Number(given) } - const next = await updateSettings(d.pg, d.now?.() ?? new Date(), (current) => ({ - traffic: { - actions: { ...current.traffic.actions, ...parseActions(values.action) }, - safeUrl: values['no-safe-url'] ? null : (values['safe-url'] ?? current.traffic.safeUrl), - abuserThreshold: - values['abuser-threshold'] === undefined - ? current.traffic.abuserThreshold - : Number(values['abuser-threshold']), - }, - retention: { - rawRetentionDays: period(values['keep-clicks'], current.retention.rawRetentionDays), - ipRetentionDays: period(values['keep-addresses'], current.retention.ipRetentionDays), - }, - })) + // An array rather than a nullable local: the hook runs inside the call + // below, and a `let` assigned only from a callback is narrowed to its + // initialiser by the compiler. + const replaced: string[] = [] + const next = await updateSettings( + d.pg, + d.now?.() ?? new Date(), + (current) => ({ + traffic: { + actions: { ...current.traffic.actions, ...parseActions(values.action) }, + safeUrl: values['no-safe-url'] ? null : (values['safe-url'] ?? current.traffic.safeUrl), + abuserThreshold: + values['abuser-threshold'] === undefined + ? current.traffic.abuserThreshold + : Number(values['abuser-threshold']), + }, + retention: { + rawRetentionDays: period(values['keep-clicks'], current.retention.rawRetentionDays), + ipRetentionDays: period(values['keep-addresses'], current.retention.ipRetentionDays), + }, + }), + { onRetentionReplaced: (why) => replaced.push(why) }, + ) + // Said, not done quietly. This command may have been about the abuser + // threshold, and it has just rewritten how long this install keeps clicks, + // which is not a change an operator should first learn about from data that + // is no longer there. + for (const why of replaced) { + d.out( + `note: the stored retention could not be read (${why}), so this command has rewritten it as keep clicks ${days(next.retention.rawRetentionDays)}, keep addresses ${days(next.retention.ipRetentionDays)}`, + ) + } printSettings(next, d) } diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts index 1bed077..d52dfa0 100644 --- a/packages/worker/src/settings.ts +++ b/packages/worker/src/settings.ts @@ -53,8 +53,19 @@ export interface SettingsRead { const why = (issues: { path: (string | number)[]; message: string }[]): string => issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') +/** + * What `parseRow` knows and `SettingsRead` does not: why the retention half + * was refused, without the sentence built around it. `updateSettings` hands it + * to a caller that is about to overwrite the bad row, so the caller can say + * which value it could not read. + */ +interface ParsedRow { + read: SettingsRead + retentionIssues: string | null +} + /** Each half validated on its own, so one bad column cannot take the other down. */ -function parseRow(row: SettingsRow): SettingsRead { +function parseRow(row: SettingsRow): ParsedRow { const traffic = TrafficSettingsSchema.safeParse({ actions: row.traffic_actions, safeUrl: row.safe_url, @@ -70,15 +81,19 @@ function parseRow(row: SettingsRow): SettingsRead { `the stored traffic settings are invalid (${why(traffic.error.issues)}); the defaults apply`, ) } - if (!retention.success) { + const retentionIssues = retention.success ? null : why(retention.error.issues) + if (retentionIssues !== null) { problems.push( - `the stored retention is invalid (${why(retention.error.issues)}); nothing is deleted until it is corrected`, + `the stored retention is invalid (${retentionIssues}); nothing is deleted until it is corrected`, ) } return { - traffic: traffic.success ? traffic.data : DEFAULT_TRAFFIC_SETTINGS, - retention: retention.success ? retention.data : null, - problem: problems.length === 0 ? null : problems.join(' '), + read: { + traffic: traffic.success ? traffic.data : DEFAULT_TRAFFIC_SETTINGS, + retention: retention.success ? retention.data : null, + problem: problems.length === 0 ? null : problems.join(' '), + }, + retentionIssues, } } @@ -93,7 +108,7 @@ export async function readSettings(pg: Pool): Promise { problem: 'no settings are stored; the defaults apply', } } - return parseRow(row) + return parseRow(row).read } /** @@ -136,10 +151,27 @@ export async function writeSettings(pg: Pool, next: InstallSettings, now: Date): * That costs one extra `config_changed` on a command an operator typed, which * is cheaper than a read-modify-write with nothing to lock. */ +export interface UpdateSettingsHooks { + /** + * Called, before the write and inside the transaction, when the stored + * retention could not be read and the defaults stood in for it. The argument + * is why it could not be read. + * + * It exists because this substitution is the one silent data decision in + * this module: a command about something else — a threshold, a safe URL — + * rewrites a retention period the operator never mentioned, and a period + * they learn about from a deleted partition is worse than one they learn + * about from a line of output. The surface that has somewhere to say it + * says it; a caller with nowhere to say it passes no hook and is unchanged. + */ + onRetentionReplaced?: (why: string) => void +} + export async function updateSettings( pg: Pool, now: Date, change: (current: InstallSettings) => InstallSettings, + hooks: UpdateSettingsHooks = {}, ): Promise { const client: PoolClient = await pg.connect() try { @@ -148,13 +180,17 @@ export async function updateSettings( const r = await client.query(`SELECT ${COLUMNS} FROM settings FOR UPDATE`) const row = r.rows[0] if (!row) throw new Error('the settings row is missing after writing it') - const read = parseRow(row) + const { read, retentionIssues } = parseRow(row) + if (read.retention === null && retentionIssues !== null) { + hooks.onRetentionReplaced?.(retentionIssues) + } const current: InstallSettings = { traffic: read.traffic, // A row this build cannot read as retention starts from the defaults // *for a deliberate write*, which is the one place that is right: the // operator is here, changing it, and the alternative is a command that - // cannot repair the row it is complaining about. + // cannot repair the row it is complaining about. The hook above is what + // keeps it from being silent. retention: read.retention ?? DEFAULT_INSTALL_SETTINGS.retention, } const next = InstallSettingsSchema.parse(change(current)) From bd38b0a059aa6a8fd35cdadd25f46ead7fa930f5 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 11:25:01 +0000 Subject: [PATCH 08/65] settings: a default that deletes is not a default --- packages/admin/src/settings-routes.test.ts | 15 ++++++++-- packages/cli/src/commands.test.ts | 15 ++++++---- packages/cli/src/commands.ts | 28 ++++++++++++------ packages/worker/src/settings.test.ts | 30 ++++++++++++++++++-- packages/worker/src/settings.ts | 33 ++++++++++++++++------ 5 files changed, 93 insertions(+), 28 deletions(-) diff --git a/packages/admin/src/settings-routes.test.ts b/packages/admin/src/settings-routes.test.ts index 3f965c3..ac48a65 100644 --- a/packages/admin/src/settings-routes.test.ts +++ b/packages/admin/src/settings-routes.test.ts @@ -85,11 +85,20 @@ describe('reading the settings', () => { }) }) - it('reports the defaults, and says why, when the row was deleted by hand', async () => { + // The two halves answer a missing row differently on purpose. Traffic falls + // back, because the redirect has to answer the next click. Retention does + // not: a number here would be a number the pass deletes by, on an install + // whose stored period is gone and may have said "never". + it('reports the traffic defaults but no retention, and says why, when the row was deleted by hand', async () => { await pg.query('DELETE FROM settings') const r = await get() - expect(r.json().traffic.abuserThreshold).toBe(DEFAULT_TRAFFIC_SETTINGS.abuserThreshold) - expect(r.json().problem).toContain('no settings are stored') + expect(r.json()).toEqual({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: null, + note: null, + problem: + 'no settings are stored; the traffic defaults apply, and nothing is deleted until the row is written back', + }) }) // The column checks are looser than core in places, so what is stored is diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index 1948339..1defe36 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -305,20 +305,23 @@ describe('clickmonk settings', () => { ]) }) - it('shows the defaults, and says so, when the settings row is missing', async () => { + // The traffic half falls back and the retention half does not, and the + // output says both: printing "keep clicks: 90 days" for a row that is not + // there would tell the operator a period this install is not enforcing. + it('shows the traffic defaults, and no period at all, when the settings row is missing', async () => { await pg.query('DELETE FROM settings') lines.length = 0 expect(await run('settings', 'show')).toBe(0) expect(lines).toEqual([ - 'note: no settings are stored; the defaults apply', + 'note: no settings are stored; the traffic defaults apply, and nothing is deleted until the row is written back', 'bot: flag', 'abuser: flag', 'anonymous: flag', 'datacenter: flag', 'safe url: (none)', 'abuser threshold: 60 requests a minute from one client', - 'keep clicks: 90 days', - 'keep addresses: 30 days', + 'keep clicks: unknown, so nothing is being deleted', + 'keep addresses: unknown, so nothing is being deleted', ]) }) @@ -416,8 +419,8 @@ describe('clickmonk settings', () => { await pg.query('UPDATE settings SET raw_retention_days = -5') lines.length = 0 expect(await run('settings', 'show')).toBe(0) - expect(lines).toContain('keep clicks: unreadable, so nothing is being deleted') - expect(lines).toContain('keep addresses: unreadable, so nothing is being deleted') + expect(lines).toContain('keep clicks: unknown, so nothing is being deleted') + expect(lines).toContain('keep addresses: unknown, so nothing is being deleted') expect(lines.some((l) => l.startsWith('keep clicks: 90'))).toBe(false) expect(lines[0]).toMatch(/^note: the stored retention is invalid \(/) } finally { diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index b2df0d0..bd62359 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -19,6 +19,7 @@ import { MAX_PASSWORD_LENGTH, MIN_ADMIN_PASSWORD_LENGTH, NON_HUMAN_CLASSES, + type TrafficSettings, isDomainUrl, normaliseHost, parseLinkInput, @@ -610,10 +611,20 @@ async function apikeyRevoke(args: string[], d: CliDeps): Promise { d.out(`key ${id} revoked`) } +/** + * The traffic half, printed once. `settings show` prints it a second way — + * without the retention lines, when the retention half cannot be read — and + * two copies of the threshold sentence is how the two drift into disagreeing + * about what the number counts. + */ +function printTraffic(t: TrafficSettings, d: CliDeps): void { + for (const c of NON_HUMAN_CLASSES) d.out(`${c}: ${t.actions[c]}`) + d.out(`safe url: ${t.safeUrl ?? '(none)'}`) + d.out(`abuser threshold: ${t.abuserThreshold} requests a minute from one client`) +} + function printSettings(s: InstallSettings, d: CliDeps): void { - for (const c of NON_HUMAN_CLASSES) d.out(`${c}: ${s.traffic.actions[c]}`) - d.out(`safe url: ${s.traffic.safeUrl ?? '(none)'}`) - d.out(`abuser threshold: ${s.traffic.abuserThreshold} requests a minute from one client`) + printTraffic(s.traffic, d) d.out(`keep clicks: ${days(s.retention.rawRetentionDays)}`) d.out(`keep addresses: ${days(s.retention.ipRetentionDays)}`) const note = retentionNote(s.retention) @@ -628,11 +639,12 @@ async function settingsShow(d: CliDeps): Promise { if (read.retention === null) { // Printing the defaults here would tell the operator this install is // deleting after ninety days, when in fact it is deleting nothing. - for (const c of NON_HUMAN_CLASSES) d.out(`${c}: ${read.traffic.actions[c]}`) - d.out(`safe url: ${read.traffic.safeUrl ?? '(none)'}`) - d.out(`abuser threshold: ${read.traffic.abuserThreshold} requests a minute from one client`) - d.out('keep clicks: unreadable, so nothing is being deleted') - d.out('keep addresses: unreadable, so nothing is being deleted') + // "unknown" rather than "unreadable" because both states arrive here: a + // row that cannot be read as retention, and no row at all. The note + // printed above says which. + printTraffic(read.traffic, d) + d.out('keep clicks: unknown, so nothing is being deleted') + d.out('keep addresses: unknown, so nothing is being deleted') return } printSettings({ traffic: read.traffic, retention: read.retention }, d) diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts index 23fd846..cd66a11 100644 --- a/packages/worker/src/settings.test.ts +++ b/packages/worker/src/settings.test.ts @@ -38,12 +38,36 @@ describe('readSettings', () => { expect(r.problem).toBeNull() }) - it('falls back to the defaults and says so when there is no row at all', async () => { + /** + * A missing row and an unreadable one are the same answer, and it is not the + * defaults. The pass that deletes clicks reads `retention`, so a row deleted + * by hand answering `{90, 30}` would delete every click older than ninety + * days on an install whose operator had set "never" — the row that said so + * being exactly the row that is gone. Traffic still falls back, because the + * redirect has to answer the next click with something. + */ + it('answers null retention, not the defaults, when there is no row at all', async () => { await pool.query('TRUNCATE settings') expect(await readSettings(pool)).toEqual({ traffic: DEFAULT_TRAFFIC_SETTINGS, - retention: { rawRetentionDays: 90, ipRetentionDays: 30 }, - problem: 'no settings are stored; the defaults apply', + retention: null, + problem: + 'no settings are stored; the traffic defaults apply, and nothing is deleted until the row is written back', + }) + }) + + /** + * The other side of that null, and one `??` away from it: a stored pair of + * nulls is an answer — keep both for ever — and must never read as "nobody + * knows". Collapsing the two is how "never delete" becomes "delete on the + * defaults". + */ + it('answers a stored pair of nulls as for ever, which is not the absence of an answer', async () => { + await pool.query('UPDATE settings SET raw_retention_days = NULL, ip_retention_days = NULL') + expect(await readSettings(pool)).toEqual({ + traffic: DEFAULT_TRAFFIC_SETTINGS, + retention: { rawRetentionDays: null, ipRetentionDays: null }, + problem: null, }) }) diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts index d52dfa0..d473d2f 100644 --- a/packages/worker/src/settings.ts +++ b/packages/worker/src/settings.ts @@ -16,7 +16,6 @@ */ import { DEFAULT_INSTALL_SETTINGS, - DEFAULT_RETENTION, DEFAULT_TRAFFIC_SETTINGS, type InstallSettings, InstallSettingsSchema, @@ -41,10 +40,15 @@ const COLUMNS = 'traffic_actions, safe_url, abuser_threshold, raw_retention_days export interface SettingsRead { traffic: TrafficSettings /** - * Null when the stored row cannot be read as retention at all. Not the - * defaults: applying a retention default deletes clicks, and it is the only - * default in this product that does. A caller that deletes things does - * nothing while this is null. + * What this install asked to keep, or null when nobody knows what it asked + * for: the row is missing, or it cannot be read as retention. **Never the + * defaults in either case.** Applying a retention default deletes clicks, + * and it is the only default in this product that does — so a caller that + * deletes things does nothing while this is null. + * + * Null is not `{ rawRetentionDays: null, ipRetentionDays: null }`. That pair + * is an answer: keep both for ever. This is the absence of one, and the two + * are one `??` apart, which is why each has a test of its own. */ retention: RetentionSettings | null problem: string | null @@ -97,15 +101,28 @@ function parseRow(row: SettingsRow): ParsedRow { } } -/** What this install is actually running, and anything wrong with the row it came from. */ +/** + * What this install is actually running, and anything wrong with the row it + * came from. + * + * The two halves answer a missing row differently, and the difference is the + * point. **Traffic falls back to its defaults**, because the redirect has to + * answer the next click with something, and being wrong there means + * classifying a visitor a little differently. **Retention answers null**, + * because being wrong there means deleting clicks an operator asked to keep — + * including an operator who asked to keep them for ever, whose stored row + * says so and whose deleted row says nothing at all. A default that deletes + * is not a default. + */ export async function readSettings(pg: Pool): Promise { const r = await pg.query(`SELECT ${COLUMNS} FROM settings`) const row = r.rows[0] if (!row) { return { traffic: DEFAULT_TRAFFIC_SETTINGS, - retention: DEFAULT_RETENTION, - problem: 'no settings are stored; the defaults apply', + retention: null, + problem: + 'no settings are stored; the traffic defaults apply, and nothing is deleted until the row is written back', } } return parseRow(row).read From d7a803137f6578eb7b19b995b9e423604c6f303d Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 11:31:49 +0000 Subject: [PATCH 09/65] settings: the second silent rewrite, and three comments that are wrong --- README.md | 14 +++-- packages/admin/src/account.ts | 9 ++- packages/admin/src/session-routes.ts | 2 +- packages/admin/src/settings-routes.test.ts | 12 +++- packages/cli/src/commands.test.ts | 38 +++++++++++++ packages/cli/src/commands.ts | 31 ++++++++--- packages/core/src/traffic.test.ts | 15 +++-- packages/core/src/traffic.ts | 11 +++- packages/redirect/src/app.ts | 2 +- packages/worker/src/settings.test.ts | 60 ++++++++++++++++---- packages/worker/src/settings.ts | 65 +++++++++++++--------- 11 files changed, 192 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 3ed15ba..2c39f23 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,9 @@ on your own infrastructure, and your click data stays yours. - **The CLI**: `clickmonk migrate`, `clickmonk domain add|list|verify`, `clickmonk link add`, `clickmonk settings show|set`, `clickmonk ipdata status|update`, `clickmonk admin create`, `clickmonk admin passwd`, `clickmonk admin totp disable`, `clickmonk apikey create|list|revoke`. - `settings set` covers the traffic actions, the safe URL and the abuser threshold. + `settings set` covers the traffic actions, the safe URL, the abuser threshold and how + long this install keeps clicks and the addresses on them (`--keep-clicks`, + `--keep-addresses`, each taking a number of days or `never`). `admin create` and `admin passwd` read the password from standard input and from nowhere else; `admin totp disable` reads none, and is the way back in when both factors are lost. - **A Docker Compose stack** that runs all of it, and a test that restarts each service @@ -317,10 +319,12 @@ setting `"truncated": true` when there were more. `clickmonk domain list` has no `clickmonk apikey list` has the same one and says when it hit it. `GET /api/links` pages properly, with `limit` and `cursor`. -The install-wide traffic settings are `GET /api/settings` and `PUT /api/settings` — the -same three things `clickmonk settings set` covers. The `PUT` takes the whole object rather -than the fields you want to change, because the safe action and the safe URL depend on each -other and a half-written pair is how a link ends up sent to nowhere. +The install-wide settings are `GET /api/settings` and `PUT /api/settings` — the same +things `clickmonk settings set` covers, in two halves: `traffic` and `retention`. The +`PUT` takes the whole object rather than the fields you want to change, because the safe +action and the safe URL depend on each other and a half-written pair is how a link ends up +sent to nowhere. A stored retention period this build cannot read is reported as `null` +rather than as the defaults, and nothing is deleted while it reads that way. For a script, mint an API key and send it as `Authorization: Bearer …`: diff --git a/packages/admin/src/account.ts b/packages/admin/src/account.ts index f592405..3bd2f45 100644 --- a/packages/admin/src/account.ts +++ b/packages/admin/src/account.ts @@ -121,17 +121,20 @@ export async function createAccount( export async function setAccountPassword( pg: Pool, password: string, - o: { clearLockout: boolean }, + o: { clearLockout: boolean; now: Date }, ): Promise { const hash = await hashPassword(password, ADMIN_SCRYPT) + // `updated_at` from the caller's clock, not SQL `now()`: every other write in + // this module is stamped from the injected one, and a row half on each is a + // row whose timestamps cannot be compared with each other. const r = await pg.query( - `UPDATE admin_account SET password_hash = $1, updated_at = now()${ + `UPDATE admin_account SET password_hash = $1, updated_at = $2${ o.clearLockout ? `, failed_logins = 0, last_failed_at = NULL, locked_until = NULL` : '' }`, - [hash], + [hash, o.now], ) if ((r.rowCount ?? 0) === 0) throw new Error('this install has no admin account yet') return hash diff --git a/packages/admin/src/session-routes.ts b/packages/admin/src/session-routes.ts index 61a72ac..a22c106 100644 --- a/packages/admin/src/session-routes.ts +++ b/packages/admin/src/session-routes.ts @@ -340,7 +340,7 @@ export function registerSessionRoutes(app: FastifyInstance, ctx: AdminContext): // The failure count and any lockout stay: this route has been shown a // session and the current password and never the second factor, and that // count is the only bound on guesses at the factor behind a session. - await setAccountPassword(ctx.pg, body.newPassword, { clearLockout: false }) + await setAccountPassword(ctx.pg, body.newPassword, { clearLockout: false, now: ctx.now() }) const r = await ctx.pg.query('DELETE FROM sessions WHERE id <> $1', [credential.id]) return { ok: true, otherSessionsSignedOut: r.rowCount ?? 0 } }) diff --git a/packages/admin/src/settings-routes.test.ts b/packages/admin/src/settings-routes.test.ts index ac48a65..72c83e3 100644 --- a/packages/admin/src/settings-routes.test.ts +++ b/packages/admin/src/settings-routes.test.ts @@ -187,14 +187,20 @@ describe('writing the settings', () => { expect(back.json().retention).toEqual({ rawRetentionDays: 90, ipRetentionDays: 30 }) }) - it('refuses a settings body with a field nobody knows', async () => { + it('refuses a settings body with a field nobody knows, and writes nothing', async () => { + const before = await get() const r = await put({ - traffic: DEFAULT_TRAFFIC_SETTINGS, - retention: { rawRetentionDays: 90, ipRetentionDays: 30 }, + // A body whose two known halves are *changes*, so that a handler which + // ignored the unknown field instead of refusing would leave both of them + // written and the read-back below would catch it. With the halves equal + // to what is stored, a refusal and a silent acceptance look the same. + traffic: { ...DEFAULT_TRAFFIC_SETTINGS, abuserThreshold: 77 }, + retention: { rawRetentionDays: 11, ipRetentionDays: 3 }, keepEverything: true, }) expect(r.statusCode).toBe(400) expect(r.json().error).toBe('invalid_body') + expect((await get()).json()).toEqual(before.json()) }) it('refuses a body that leaves a half out, rather than defaulting it', async () => { diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index 1defe36..8f125c5 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -401,6 +401,44 @@ describe('clickmonk settings', () => { } }) + // The same rule one half along, and the harm is the same in different + // units: an unreadable traffic half is overwritten with the all-flag + // defaults by a command about a retention period, so traffic that was being + // blocked is only flagged from here on. Learning that from a line beats + // learning it from traffic that got through. + it('says so when it rewrote the traffic half it could not read', async () => { + await pg.query('ALTER TABLE settings DROP CONSTRAINT settings_traffic_actions_check') + try { + await pg.query(`UPDATE settings SET traffic_actions = '{"bot":"block"}'::jsonb`) + lines.length = 0 + expect(await run('settings', 'set', '--keep-clicks', '45')).toBe(0) + expect(lines[0]).toMatch( + /^note: the stored traffic settings could not be read \(.+\), so this command has rewritten them; what is stored now is below$/, + ) + // The rewrite happened and it is the defaults: `bot` was `block` and is + // `flag`, which is the whole reason the line has to be printed. + const stored = await pg.query<{ traffic_actions: unknown }>( + 'SELECT traffic_actions FROM settings', + ) + expect(stored.rows[0]?.traffic_actions).toEqual({ + bot: 'flag', + abuser: 'flag', + anonymous: 'flag', + datacenter: 'flag', + }) + // And the command's own change still landed. + expect(lines).toContain('keep clicks: 45 days') + } finally { + await pg.query( + `UPDATE settings SET traffic_actions = + '{"bot":"flag","abuser":"flag","anonymous":"flag","datacenter":"flag"}'::jsonb`, + ) + await pg.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_traffic_actions_check CHECK (valid_traffic_actions(traffic_actions, true))', + ) + } + }) + // A command that changed nothing about retention says nothing about it: the // line above is a report of a rewrite, not a banner. it('says nothing about retention when the stored row reads fine', async () => { diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index bd62359..55c9a59 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -510,7 +510,7 @@ async function adminPasswd(d: CliDeps): Promise { // alone: it is the way back in, and a lock over a password that no longer // exists would keep the only account out for nothing. The API route asks for // the opposite, because it never has to show the second factor. - await setAccountPassword(d.pg, password, { clearLockout: true }) + await setAccountPassword(d.pg, password, { clearLockout: true, now: d.now?.() ?? new Date() }) const r = await d.pg.query('DELETE FROM sessions') d.out(`password changed; ${r.rowCount ?? 0} session(s) signed out`) } @@ -678,10 +678,11 @@ async function settingsSet(args: string[], d: CliDeps): Promise { if (given === 'never') return null return Number(given) } - // An array rather than a nullable local: the hook runs inside the call - // below, and a `let` assigned only from a callback is narrowed to its - // initialiser by the compiler. + // Arrays rather than nullable locals: the hooks run inside the call below, + // and a `let` assigned only from a callback is narrowed to its initialiser + // by the compiler. const replaced: string[] = [] + const replacedTraffic: string[] = [] const next = await updateSettings( d.pg, d.now?.() ?? new Date(), @@ -699,12 +700,24 @@ async function settingsSet(args: string[], d: CliDeps): Promise { ipRetentionDays: period(values['keep-addresses'], current.retention.ipRetentionDays), }, }), - { onRetentionReplaced: (why) => replaced.push(why) }, + { + onRetentionReplaced: (why) => replaced.push(why), + onTrafficReplaced: (why) => replacedTraffic.push(why), + }, ) - // Said, not done quietly. This command may have been about the abuser - // threshold, and it has just rewritten how long this install keeps clicks, - // which is not a change an operator should first learn about from data that - // is no longer there. + // Both rewrites are said, not done quietly. This command may have been about + // one field and have overwritten the other half of the row on its way past, + // and neither is a change an operator should first learn about from clicks + // that are gone or from traffic that stopped being blocked. + // + // The traffic note points at the lines below rather than naming the values, + // because there are five of them and they are already about to be printed; + // the retention note names its two, because they are two. + for (const why of replacedTraffic) { + d.out( + `note: the stored traffic settings could not be read (${why}), so this command has rewritten them; what is stored now is below`, + ) + } for (const why of replaced) { d.out( `note: the stored retention could not be read (${why}), so this command has rewritten it as keep clicks ${days(next.retention.rawRetentionDays)}, keep addresses ${days(next.retention.ipRetentionDays)}`, diff --git a/packages/core/src/traffic.test.ts b/packages/core/src/traffic.test.ts index 24d481d..eb8eadb 100644 --- a/packages/core/src/traffic.test.ts +++ b/packages/core/src/traffic.test.ts @@ -23,7 +23,7 @@ const CHECKED: IpFacts = { const facts = (over: Partial = {}, ip: Partial = {}): TrafficFacts => ({ userAgent: BROWSER, head: false, - clicksThisMinute: 1, + requestsThisMinute: 1, abuserThreshold: 60, ...over, ip: { ...CHECKED, ...ip }, @@ -47,7 +47,7 @@ describe('classifyTraffic', () => { it.each([ ['a missing user-agent', facts({ userAgent: '' }), 'bot', ['ua_missing']], ['a crawler user-agent', facts({ userAgent: CRAWLER }), 'bot', ['ua_bot']], - ['one click over the threshold', facts({ clicksThisMinute: 61 }), 'abuser', ['rate']], + ['one click over the threshold', facts({ requestsThisMinute: 61 }), 'abuser', ['rate']], ['a Tor exit', facts({}, { tor: true }), 'anonymous', ['tor']], ['a hosting network', facts({}, { datacenter: true }), 'datacenter', ['datacenter']], ] as const)('classifies %s', (_label, f, cls, signals) => { @@ -55,18 +55,21 @@ describe('classifyTraffic', () => { }) it('does not call the threshold itself abuse', () => { - expect(classifyTraffic(facts({ clicksThisMinute: 60 })).class).toBe('human') + expect(classifyTraffic(facts({ requestsThisMinute: 60 })).class).toBe('human') }) it('takes the first class in order, and records every signal', () => { - const all = facts({ userAgent: CRAWLER, clicksThisMinute: 99 }, { tor: true, datacenter: true }) + const all = facts( + { userAgent: CRAWLER, requestsThisMinute: 99 }, + { tor: true, datacenter: true }, + ) expect(classifyTraffic(all)).toEqual({ class: 'bot', ruleClass: 'bot', signals: ['ua_bot', 'rate', 'tor', 'datacenter'], }) expect( - classifyTraffic(facts({ clicksThisMinute: 99 }, { tor: true, datacenter: true })).class, + classifyTraffic(facts({ requestsThisMinute: 99 }, { tor: true, datacenter: true })).class, ).toBe('abuser') expect(classifyTraffic(facts({}, { tor: true, datacenter: true })).class).toBe('anonymous') }) @@ -79,7 +82,7 @@ describe('classifyTraffic', () => { it('still classifies by user-agent and rate without IP data', () => { expect(classifyTraffic({ ...facts({ userAgent: '' }), ip: NO_IP_FACTS }).class).toBe('bot') - expect(classifyTraffic({ ...facts({ clicksThisMinute: 61 }), ip: NO_IP_FACTS }).class).toBe( + expect(classifyTraffic({ ...facts({ requestsThisMinute: 61 }), ip: NO_IP_FACTS }).class).toBe( 'abuser', ) }) diff --git a/packages/core/src/traffic.ts b/packages/core/src/traffic.ts index 67206f7..cfb53e0 100644 --- a/packages/core/src/traffic.ts +++ b/packages/core/src/traffic.ts @@ -86,8 +86,13 @@ export interface TrafficFacts { /** Already cut to the bounded length the redirect records. */ userAgent: string head: boolean - /** Requests from this address in the current minute, this one included. */ - clicksThisMinute: number + /** + * Requests from this client in the current minute, this one included. A + * client is one IPv4 address or one IPv6 /64, which is what `rateKey` + * counts — not one address, since an IPv6 client can pick a new address out + * of its /64 for every request. + */ + requestsThisMinute: number /** More than this many in a minute is an abuser. */ abuserThreshold: number ip: IpFacts @@ -115,7 +120,7 @@ export function classifyTraffic(f: TrafficFacts): Traffic { if (f.userAgent.length === 0) signals.push('ua_missing') else if (isBotUserAgent(f.userAgent)) signals.push('ua_bot') if (f.head) signals.push('head') - if (f.clicksThisMinute > f.abuserThreshold) signals.push('rate') + if (f.requestsThisMinute > f.abuserThreshold) signals.push('rate') if (f.ip.tor === true) signals.push('tor') if (f.ip.datacenter === true) signals.push('datacenter') diff --git a/packages/redirect/src/app.ts b/packages/redirect/src/app.ts index 88c07f5..9ab8be0 100644 --- a/packages/redirect/src/app.ts +++ b/packages/redirect/src/app.ts @@ -196,7 +196,7 @@ export function buildRedirectApp( userAgent: o.userAgent, head: o.head, // A monotonic clock: a wall clock stepped back would restart every count. - clicksThisMinute: deps.rate.hit(o.ip, performance.now()), + requestsThisMinute: deps.rate.hit(o.ip, performance.now()), abuserThreshold: o.snapshot.settings.abuserThreshold, ip: ipFacts, }) diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts index cd66a11..bde3f8c 100644 --- a/packages/worker/src/settings.test.ts +++ b/packages/worker/src/settings.test.ts @@ -1,10 +1,17 @@ import { DEFAULT_INSTALL_SETTINGS, DEFAULT_TRAFFIC_SETTINGS } from '@clickmonk/core' -import type { Pool } from '@clickmonk/db' -import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' +import { type Pool, createPgPool } from '@clickmonk/db' +import { TEST_PG_URL, resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' import { readSettings, updateSettings, writeSettings } from './settings.js' -const pool: Pool = testPg() +/** + * Every connection this file's pool opens reports this name to Postgres, so + * that the leaked-transaction probe below can ask about *these* connections + * and no others. Set on the connection string, because that is the only place + * it reaches every connection the pool opens, including ones it opens later. + */ +const APP_NAME = 'clickmonk-worker-settings-test' +const pool: Pool = createPgPool(`${TEST_PG_URL}?application_name=${APP_NAME}`) const ch = testCh() const NOW = new Date('2026-09-24T12:00:00.000Z') @@ -128,6 +135,39 @@ describe('readSettings', () => { ) } }) + + // Both halves at once, which nothing else covers and which is where the two + // sentences meet. Run together they read as one saying the defaults apply to + // the retention as well, which is the opposite of what the second one says. + it('reports both halves separately when neither can be read', async () => { + await pool.query('ALTER TABLE settings DROP CONSTRAINT settings_traffic_actions_check') + await pool.query('ALTER TABLE settings DROP CONSTRAINT settings_raw_retention_valid') + try { + await pool.query( + `UPDATE settings SET traffic_actions = '{"bot":"drop"}'::jsonb, raw_retention_days = -5`, + ) + const r = await readSettings(pool) + expect(r.traffic).toEqual(DEFAULT_TRAFFIC_SETTINGS) + expect(r.retention).toBeNull() + // Exact, and the separator between the two halves is the point: without + // one, "…the defaults apply" runs straight into "the stored retention is + // invalid…". + expect(r.problem).toBe( + `the stored traffic settings are invalid (actions.bot: Invalid enum value. Expected 'nothing' | 'flag' | 'block' | 'safe', received 'drop'; actions.abuser: Required; actions.anonymous: Required; actions.datacenter: Required); the defaults apply; the stored retention is invalid (rawRetentionDays: Number must be greater than or equal to 1); nothing is deleted until it is corrected`, + ) + } finally { + await pool.query( + `UPDATE settings SET raw_retention_days = 90, traffic_actions = + '{"bot":"flag","abuser":"flag","anonymous":"flag","datacenter":"flag"}'::jsonb`, + ) + await pool.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_traffic_actions_check CHECK (valid_traffic_actions(traffic_actions, true))', + ) + await pool.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_raw_retention_valid CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650)', + ) + } + }) }) describe('writeSettings', () => { @@ -194,17 +234,17 @@ describe('updateSettings', () => { // answers as if the rollback had happened. A second connection is what // can see a backend left sitting inside a transaction. // - // **Do not narrow this to one connection.** It counts every backend on - // this database on purpose: the leaked one cannot be addressed from here, - // because the pool hands the same client straight back to whoever asks - // next. Counting database-wide is exact under this repository's rule that - // one suite runs at a time, and nothing narrower fails when the ROLLBACK - // is removed. + // Scoped to this file's own connections by `application_name`, which is + // what makes it exact rather than dependent on nothing else running: the + // pool above tags every connection it opens, this probe opens its own + // untagged one, and a suite in another process cannot be counted here. const probe = testPg() try { const open = await probe.query<{ n: number }>( `SELECT count(*)::int AS n FROM pg_stat_activity - WHERE datname = current_database() AND state LIKE 'idle in transaction%'`, + WHERE datname = current_database() AND application_name = $1 + AND state LIKE 'idle in transaction%'`, + [APP_NAME], ) expect(open.rows[0]?.n).toBe(0) } finally { diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts index d473d2f..6cea3af 100644 --- a/packages/worker/src/settings.ts +++ b/packages/worker/src/settings.ts @@ -58,13 +58,14 @@ const why = (issues: { path: (string | number)[]; message: string }[]): string = issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') /** - * What `parseRow` knows and `SettingsRead` does not: why the retention half - * was refused, without the sentence built around it. `updateSettings` hands it + * What `parseRow` knows and `SettingsRead` does not: why each half was + * refused, without the sentence built around it. `updateSettings` hands both * to a caller that is about to overwrite the bad row, so the caller can say - * which value it could not read. + * which values it could not read. */ interface ParsedRow { read: SettingsRead + trafficIssues: string | null retentionIssues: string | null } @@ -80,10 +81,9 @@ function parseRow(row: SettingsRow): ParsedRow { ipRetentionDays: row.ip_retention_days, }) const problems: string[] = [] - if (!traffic.success) { - problems.push( - `the stored traffic settings are invalid (${why(traffic.error.issues)}); the defaults apply`, - ) + const trafficIssues = traffic.success ? null : why(traffic.error.issues) + if (trafficIssues !== null) { + problems.push(`the stored traffic settings are invalid (${trafficIssues}); the defaults apply`) } const retentionIssues = retention.success ? null : why(retention.error.issues) if (retentionIssues !== null) { @@ -95,8 +95,13 @@ function parseRow(row: SettingsRow): ParsedRow { read: { traffic: traffic.success ? traffic.data : DEFAULT_TRAFFIC_SETTINGS, retention: retention.success ? retention.data : null, - problem: problems.length === 0 ? null : problems.join(' '), + // Joined with a separator, not a space: both halves can be invalid at + // once, and run together the two sentences read as one that says the + // defaults apply to the retention as well — which is the opposite of + // what the second one says. + problem: problems.length === 0 ? null : problems.join('; '), }, + trafficIssues, retentionIssues, } } @@ -159,30 +164,35 @@ export async function writeSettings(pg: Pool, next: InstallSettings, now: Date): ) } +/** + * Told, before the write and inside the transaction, that a half of the stored + * row could not be read and the defaults stood in for it. The argument is why. + * + * **There are two of these, not one.** Either half of a row can be unreadable, + * and in both cases this function writes the defaults over it while the + * operator was changing something else — a threshold, a period — and says + * nothing unless the surface in front of it does. Both are the same harm in + * different units: a retention period nobody was told about is learned from + * clicks that are gone, and a traffic action nobody was told about is learned + * from traffic that was being blocked and is now only flagged. A caller with + * nowhere to print a note passes no hook and is unchanged. + */ +export interface UpdateSettingsHooks { + onRetentionReplaced?: (why: string) => void + onTrafficReplaced?: (why: string) => void +} + /** * Read, change, validate, write — with the row locked for the whole of it, so * two callers changing different fields cannot lose one of the changes. * * The row is written back as the defaults first when it is missing, in the * same transaction, so that concurrent writers still serialise on its lock. - * That costs one extra `config_changed` on a command an operator typed, which - * is cheaper than a read-modify-write with nothing to lock. + * Postgres collapses identical notifications on one channel inside a + * transaction, so that insert and the update after it reach the redirect as + * one `config_changed` and this costs no extra reload — measured, because the + * comment here used to charge for one. */ -export interface UpdateSettingsHooks { - /** - * Called, before the write and inside the transaction, when the stored - * retention could not be read and the defaults stood in for it. The argument - * is why it could not be read. - * - * It exists because this substitution is the one silent data decision in - * this module: a command about something else — a threshold, a safe URL — - * rewrites a retention period the operator never mentioned, and a period - * they learn about from a deleted partition is worse than one they learn - * about from a line of output. The surface that has somewhere to say it - * says it; a caller with nowhere to say it passes no hook and is unchanged. - */ - onRetentionReplaced?: (why: string) => void -} export async function updateSettings( pg: Pool, @@ -197,11 +207,14 @@ export async function updateSettings( const r = await client.query(`SELECT ${COLUMNS} FROM settings FOR UPDATE`) const row = r.rows[0] if (!row) throw new Error('the settings row is missing after writing it') - const { read, retentionIssues } = parseRow(row) + const { read, trafficIssues, retentionIssues } = parseRow(row) + if (trafficIssues !== null) hooks.onTrafficReplaced?.(trafficIssues) if (read.retention === null && retentionIssues !== null) { hooks.onRetentionReplaced?.(retentionIssues) } const current: InstallSettings = { + // The defaults, when the stored half could not be read — and the hook + // above is what keeps that from being silent. traffic: read.traffic, // A row this build cannot read as retention starts from the defaults // *for a deliberate write*, which is the one place that is right: the From 869c2e620314800e4ee3b067b7fe66511dd571f1 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 11:33:53 +0000 Subject: [PATCH 10/65] admin: pin the password change's stamp to the service clock --- packages/admin/src/account.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/admin/src/account.test.ts b/packages/admin/src/account.test.ts index 34ec911..086f7e1 100644 --- a/packages/admin/src/account.test.ts +++ b/packages/admin/src/account.test.ts @@ -558,6 +558,25 @@ describe('two-factor authentication', () => { } }) + // Every timestamp on this row comes from the clock the service was built + // with. One written by SQL `now()` instead cannot be compared with the + // others, and is a row whose history reads as out of order on any install + // where the two clocks differ. + it('stamps the change from the service clock, not the database one', async () => { + const cookie = await signedIn(app, pg) + clock.advance(90 * 60 * 1000) + const at = clock.now() + const r = await app.inject({ + method: 'POST', + url: '/api/password', + headers: write(cookie), + payload: { currentPassword: ADMIN_PASSWORD, newPassword: 'a new decent password' }, + }) + expect(r.statusCode).toBe(200) + const row = await pg.query<{ updated_at: Date }>('SELECT updated_at FROM admin_account') + expect(row.rows[0]?.updated_at.toISOString()).toBe(at.toISOString()) + }) + // A session cookie is not a bound. A wrong password here costs the account // exactly what a wrong password at the sign-in form costs, so a cookie taken // off a shared machine is not an oracle. From 7ad1c81a63057aa9ffa13449bbdd1d6dcb7bb280 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 12:10:14 +0000 Subject: [PATCH 11/65] settings: a write that changes what it did not read says so --- README.md | 7 +- packages/admin/src/account.test.ts | 99 +++++++++++-------- packages/admin/src/account.ts | 20 ++-- packages/admin/src/auth.test.ts | 36 +++---- packages/admin/src/domains.test.ts | 2 +- packages/admin/src/keys.test.ts | 2 +- packages/admin/src/links.test.ts | 2 +- packages/admin/src/settings-routes.test.ts | 2 +- packages/admin/src/testing.ts | 13 ++- packages/cli/src/commands.test.ts | 15 +++ packages/cli/src/commands.ts | 49 ++++----- .../postgres/007_reporting_settings.sql | 10 ++ packages/worker/src/settings.test.ts | 45 ++++++++- packages/worker/src/settings.ts | 52 +++++++--- 14 files changed, 242 insertions(+), 112 deletions(-) diff --git a/README.md b/README.md index 2c39f23..553d121 100644 --- a/README.md +++ b/README.md @@ -323,8 +323,11 @@ The install-wide settings are `GET /api/settings` and `PUT /api/settings` — th things `clickmonk settings set` covers, in two halves: `traffic` and `retention`. The `PUT` takes the whole object rather than the fields you want to change, because the safe action and the safe URL depend on each other and a half-written pair is how a link ends up -sent to nowhere. A stored retention period this build cannot read is reported as `null` -rather than as the defaults, and nothing is deleted while it reads that way. +sent to nowhere. `retention` comes back as `null` rather than as the defaults whenever +this install's kept periods are not known — the settings row is missing, or its stored +periods cannot be read — and nothing is deleted while it reads that way. A missing row is +the likelier of the two: it is one `DELETE` away, and the defaults would answer it with a +period nobody chose. For a script, mint an API key and send it as `Authorization: Bearer …`: diff --git a/packages/admin/src/account.test.ts b/packages/admin/src/account.test.ts index 086f7e1..4286329 100644 --- a/packages/admin/src/account.test.ts +++ b/packages/admin/src/account.test.ts @@ -46,16 +46,24 @@ const signInWith = (payload: Record) => describe('creating the one admin account', () => { it('refuses a second, rather than replacing the first', async () => { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) await expect( - createAccount(pg, { email: 'other@example.com', password: 'another good password' }), + createAccount(pg, { + email: 'other@example.com', + password: 'another good password', + now: clock.now(), + }), ).rejects.toThrow(AccountExistsError) const r = await pg.query<{ email: string }>('SELECT email FROM admin_account') expect(r.rows[0]?.email).toBe(ADMIN_EMAIL) }) it('stores the address lower-cased, and never the password', async () => { - await createAccount(pg, { email: ' Admin@Example.COM ', password: ADMIN_PASSWORD }) + await createAccount(pg, { + email: ' Admin@Example.COM ', + password: ADMIN_PASSWORD, + now: clock.now(), + }) const r = await pg.query<{ email: string; password_hash: string }>( 'SELECT email, password_hash FROM admin_account', ) @@ -66,7 +74,7 @@ describe('creating the one admin account', () => { describe('signing in', () => { it('answers the same refusal for an unknown address and a wrong password', async () => { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const wrongEmail = await signInWith({ email: 'nobody@example.com', password: ADMIN_PASSWORD }) const wrongPassword = await signInWith({ email: ADMIN_EMAIL, password: 'not the password' }) expect(wrongEmail.statusCode).toBe(401) @@ -86,14 +94,14 @@ describe('signing in', () => { expect(unclaimed.json().error).toBe('invalid_credentials') expect(unclaimed.body).not.toContain('no_admin') - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const wrongPassword = await signInWith({ email: ADMIN_EMAIL, password: 'not the password' }) expect(wrongPassword.statusCode).toBe(401) expect(unclaimed.json()).toEqual(wrongPassword.json()) }) it('locks the account after five failures, for longer each time', async () => { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) for (let i = 0; i < 5; i++) { const r = await signInWith({ email: ADMIN_EMAIL, password: 'wrong' }) expect(r.statusCode, `attempt ${i}`).toBe(401) @@ -130,7 +138,7 @@ describe('signing in', () => { const attempts = new AttemptCounter(2, LOGIN_ATTEMPT_WINDOW_MS) const limited = testApp(pg, clock, { loginAttempts: attempts }) try { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const attempt = (password: string) => limited.inject({ method: 'POST', @@ -172,7 +180,7 @@ describe('signing in', () => { { trustProxy: SHIPPED_TRUSTED_PROXIES }, ) try { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const attempt = (password: string, ip: string) => limited.inject({ method: 'POST', @@ -214,7 +222,7 @@ describe('signing in', () => { { trustProxy: SHIPPED_TRUSTED_PROXIES }, ) try { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const attempt = (ip: string) => limited.inject({ method: 'POST', @@ -248,7 +256,7 @@ describe('signing in', () => { const attempts = new AttemptCounter(1, LOGIN_ATTEMPT_WINDOW_MS) const limited = testApp(pg, clock, { loginAttempts: attempts }) try { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const attempt = (password: string) => limited.inject({ method: 'POST', @@ -283,7 +291,7 @@ describe('signing in', () => { it('records the session with the device and address it came from', async () => { const trusting = testApp(pg, clock, {}, { trustProxy: SHIPPED_TRUSTED_PROXIES }) try { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const signIn = (ip: string) => trusting.inject({ method: 'POST', @@ -320,7 +328,7 @@ describe('signing in', () => { }) it('removes sessions that have run out, when someone signs in', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) // Named by the constant, and carrying a token: a substring check against // the name written out passes on a cookie the helper failed to read, since // the name is still there in front of an empty value. @@ -346,7 +354,7 @@ describe('signing in', () => { // rows to be listed as live with an expiry weeks away — which is the exact // opposite of what the admin is reading that list to find out. it('treats a session idle past its bound as gone, in the sweep and in the list', async () => { - const mine = await signedIn(app, pg) + const mine = await signedIn(app, pg, clock.now()) const theirs = cookieFrom( (await signInWith({ email: ADMIN_EMAIL, password: ADMIN_PASSWORD })).headers['set-cookie'], ) @@ -412,7 +420,7 @@ describe('two-factor authentication', () => { } it('is off until a code proves the app holds the secret', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const start = await app.inject({ method: 'POST', url: '/api/totp', @@ -444,7 +452,7 @@ describe('two-factor authentication', () => { // client cannot enrol a secret it chose — and confirming without having // started is refused rather than treated as an enrolment of something. it('enrols only the secret it minted, and never one a body names', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const chosen = newTotpSecret() const smuggled = await app.inject({ method: 'POST', @@ -481,7 +489,7 @@ describe('two-factor authentication', () => { // replace it or remove it — otherwise a stolen session plus the password is // a full bypass by way of disable-then-enrol. it('needs the current factor to re-enrol or to turn off, and takes a recovery code for it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret, recoveryCodes } = await enrol(cookie) for (const [method, url] of [ @@ -541,7 +549,7 @@ describe('two-factor authentication', () => { // The password comes first on every one of these, before the second factor // is even looked at, so a wrong password never says whether a factor exists. it('needs the password again to enrol, to disable, or to replace the codes', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) for (const [method, url] of [ ['POST', '/api/totp'], ['DELETE', '/api/totp'], @@ -559,13 +567,15 @@ describe('two-factor authentication', () => { }) // Every timestamp on this row comes from the clock the service was built - // with. One written by SQL `now()` instead cannot be compared with the - // others, and is a row whose history reads as out of order on any install - // where the two clocks differ. - it('stamps the change from the service clock, not the database one', async () => { - const cookie = await signedIn(app, pg) + // with — `created_at` at the insert as well as `updated_at` at every write + // after it. One written by SQL `now()` instead cannot be compared with the + // others: on any install where the two clocks differ it reads as history out + // of order, and here it would put the change *before* the account existed. + it('stamps the row from the service clock at creation and at every change', async () => { + const made = clock.now() + const cookie = await signedIn(app, pg, made) clock.advance(90 * 60 * 1000) - const at = clock.now() + const changed = clock.now() const r = await app.inject({ method: 'POST', url: '/api/password', @@ -573,15 +583,22 @@ describe('two-factor authentication', () => { payload: { currentPassword: ADMIN_PASSWORD, newPassword: 'a new decent password' }, }) expect(r.statusCode).toBe(200) - const row = await pg.query<{ updated_at: Date }>('SELECT updated_at FROM admin_account') - expect(row.rows[0]?.updated_at.toISOString()).toBe(at.toISOString()) + const row = await pg.query<{ created_at: Date; updated_at: Date }>( + 'SELECT created_at, updated_at FROM admin_account', + ) + expect(row.rows[0]?.created_at.toISOString()).toBe(made.toISOString()) + expect(row.rows[0]?.updated_at.toISOString()).toBe(changed.toISOString()) + // And in that order, which is what a stamp from the wrong clock breaks. + expect(row.rows[0]?.created_at.getTime()).toBeLessThan( + row.rows[0]?.updated_at.getTime() as number, + ) }) // A session cookie is not a bound. A wrong password here costs the account // exactly what a wrong password at the sign-in form costs, so a cookie taken // off a shared machine is not an oracle. it('counts a wrong password behind a session, and locks the account on it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) for (let i = 0; i < 5; i++) { const r = await app.inject({ method: 'POST', @@ -631,7 +648,7 @@ describe('two-factor authentication', () => { // failures inside the window is what the lockout is for; a scatter of them // across a year is not. it('forgets failures older than the decay window, and locks on ones inside it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const mistype = () => app.inject({ method: 'POST', @@ -677,7 +694,7 @@ describe('two-factor authentication', () => { // The count and the lock are invisible everywhere else, which is half of why // the trap above went unnoticed until it sprang. it('reports the failure count and any lockout on the account', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const me = () => app.inject({ method: 'GET', url: '/api/me', headers: read(cookie) }) expect((await me()).json().failedLogins).toBe(0) expect((await me()).json().lockedUntil).toBeNull() @@ -719,7 +736,7 @@ describe('two-factor authentication', () => { const attempts = new AttemptCounter(1, LOGIN_ATTEMPT_WINDOW_MS) const limited = testApp(pg, clock, { loginAttempts: attempts }) try { - const cookie = await signedIn(limited, pg) + const cookie = await signedIn(limited, pg, clock.now()) const ask = (currentPassword: string) => limited.inject({ method: 'POST', @@ -748,7 +765,7 @@ describe('two-factor authentication', () => { // guesses. A wrong code behind a session costs what a wrong password behind a // session costs. it('counts a wrong second factor behind a session, and locks the account on it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret } = await enrol(cookie) for (let i = 0; i < 5; i++) { const r = await app.inject({ @@ -791,7 +808,7 @@ describe('two-factor authentication', () => { // and spend one of them as "the current factor" on the next request, which // is the second factor removed by someone who never held it. it('needs the current factor to replace the recovery codes', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret } = await enrol(cookie) const withoutFactor = await app.inject({ method: 'POST', @@ -820,7 +837,7 @@ describe('two-factor authentication', () => { }) it('then asks for a code at every sign-in', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret } = await enrol(cookie) const withoutCode = await signInWith({ email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) expect(withoutCode.statusCode).toBe(401) @@ -848,7 +865,7 @@ describe('two-factor authentication', () => { // A code seen over a shoulder, or read out of a proxy log, is already spent. it('accepts a code once, and refuses every step at or below it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret } = await enrol(cookie) clock.advance(60_000) const code = totpCode(secret, totpStep(clock.now().getTime())) as string @@ -871,7 +888,7 @@ describe('two-factor authentication', () => { }) it('takes each recovery code once', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { recoveryCodes } = await enrol(cookie) expect(recoveryCodes).toHaveLength(10) const code = recoveryCodes[0] as string @@ -899,7 +916,7 @@ describe('two-factor authentication', () => { }) it('refuses a recovery code this install never issued, and one from an old set', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret, recoveryCodes } = await enrol(cookie) const stranger = await signInWith({ email: ADMIN_EMAIL, @@ -938,7 +955,7 @@ describe('two-factor authentication', () => { // codes outright and enrolling a new secret are two different writes, so a // test of one says nothing about the other. it('takes the previous set of recovery codes with a fresh enrolment', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret, recoveryCodes } = await enrol(cookie) // Replacing the app: the current factor authorises it... @@ -986,7 +1003,7 @@ describe('two-factor authentication', () => { }) it('refuses a code and a recovery code in one request', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret, recoveryCodes } = await enrol(cookie) const r = await signInWith({ email: ADMIN_EMAIL, @@ -998,7 +1015,7 @@ describe('two-factor authentication', () => { }) it('takes the codes with it when it is turned off', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const { secret } = await enrol(cookie) clock.advance(60_000) const off = await app.inject({ @@ -1022,7 +1039,7 @@ describe('two-factor authentication', () => { describe('changing the password', () => { it('needs the current one, and signs every other session out', async () => { - const first = await signedIn(app, pg) + const first = await signedIn(app, pg, clock.now()) const secondLogin = await signInWith({ email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) const second = cookieFrom(secondLogin.headers['set-cookie']) @@ -1065,7 +1082,7 @@ describe('changing the password', () => { // lockout would never arrive. Clearing belongs to the command line, where a // shell is already the operator's way back in. it('does not clear the failure count when the password is changed through the API', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const wrong = () => app.inject({ method: 'POST', @@ -1104,7 +1121,7 @@ describe('changing the password', () => { }) it('refuses a new password shorter than the floor', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'POST', url: '/api/password', diff --git a/packages/admin/src/account.ts b/packages/admin/src/account.ts index 3bd2f45..40d8f67 100644 --- a/packages/admin/src/account.ts +++ b/packages/admin/src/account.ts @@ -87,16 +87,23 @@ export class AccountExistsError extends Error { /** * Creates the account. One install, one admin: a second call throws rather * than replacing the first, so a script run twice cannot take an install over. + * + * `now` stamps both timestamps rather than letting the columns default to SQL + * `now()`. The row's later writes are stamped from the caller's clock, and a + * row created on one clock and updated on another is a row whose history reads + * as out of order — `updated_at` before `created_at` — wherever the two clocks + * differ, which is every test that freezes one. */ export async function createAccount( pg: Pool, - o: { email: string; password: string }, + o: { email: string; password: string; now: Date }, ): Promise { const hash = await hashPassword(o.password, ADMIN_SCRYPT) const r = await pg.query( - `INSERT INTO admin_account (email, password_hash) VALUES ($1, $2) + `INSERT INTO admin_account (email, password_hash, created_at, updated_at) + VALUES ($1, $2, $3, $3) ON CONFLICT (id) DO NOTHING`, - [normaliseEmail(o.email), hash], + [normaliseEmail(o.email), hash, o.now], ) if ((r.rowCount ?? 0) === 0) throw new AccountExistsError() } @@ -124,9 +131,10 @@ export async function setAccountPassword( o: { clearLockout: boolean; now: Date }, ): Promise { const hash = await hashPassword(password, ADMIN_SCRYPT) - // `updated_at` from the caller's clock, not SQL `now()`: every other write in - // this module is stamped from the injected one, and a row half on each is a - // row whose timestamps cannot be compared with each other. + // `updated_at` from the caller's clock, not SQL `now()`, which is what every + // write in this module does — `createAccount` included, since it stamps both + // columns rather than letting them default. A row with one timestamp on each + // clock is a row whose history reads as out of order. const r = await pg.query( `UPDATE admin_account SET password_hash = $1, updated_at = $2${ o.clearLockout diff --git a/packages/admin/src/auth.test.ts b/packages/admin/src/auth.test.ts index 6b03f7d..46a08d1 100644 --- a/packages/admin/src/auth.test.ts +++ b/packages/admin/src/auth.test.ts @@ -61,7 +61,7 @@ describe('what the admin service answers at all', () => { // whether an admin account exists, which is the window in which the account // can still be claimed by whoever reaches the CLI first. it('answers liveness only, and says nothing about the install', async () => { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const configured = await app.inject({ method: 'GET', url: '/health' }) expect(configured.statusCode).toBe(200) expect(configured.json()).toEqual({ status: 'ok' }) @@ -81,7 +81,7 @@ describe('what the admin service answers at all', () => { // Readiness is behind the guards, where a credential has already been shown. it('reports whether an account exists only to a credential', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const me = await app.inject({ method: 'GET', url: '/api/me', headers: read(cookie) }) expect(me.statusCode).toBe(200) expect(me.json().email).toBe(ADMIN_EMAIL) @@ -99,7 +99,7 @@ describe('what the admin service answers at all', () => { // by host name, and this is the second gate: any container on the Compose // network can open a connection to this port and send whatever Host it likes. it('answers 404 for every host name but its own', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) // No empty-Host case here: `inject` substitutes `localhost:80` for one, so // it would assert nothing the `localhost` row does not already assert. // An absent Host over a real socket is the stack suite's to show. @@ -146,7 +146,7 @@ describe('what the admin service answers at all', () => { const trusting = testApp(pg, clock, {}, { trustProxy: SHIPPED_TRUSTED_PROXIES }) try { - const cookie = await signedIn(trusting, pg, ADMIN_PASSWORD) + const cookie = await signedIn(trusting, pg, clock.now(), ADMIN_PASSWORD) const me = await trusting.inject({ method: 'GET', url: '/api/me', @@ -206,7 +206,7 @@ describe('what the admin service answers at all', () => { }) it('answers its own host name with a port, as a browser sends it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'GET', url: '/api/me', @@ -243,7 +243,7 @@ describe('what the admin service answers at all', () => { }) it('never answers a CORS header, so no other origin can read it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'GET', url: '/api/me', @@ -257,7 +257,7 @@ describe('what the admin service answers at all', () => { describe('the session cookie', () => { it('is HttpOnly, Secure, SameSite=Strict and host-only', async () => { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) const r = await app.inject({ method: 'POST', url: '/api/session', @@ -283,7 +283,7 @@ describe('the session cookie', () => { // A stolen dump of `sessions` is not a way in: the row holds a digest, and // the cookie holds the token. it('stores only a digest of the token', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const token = cookie.slice(cookie.indexOf('=') + 1) const r = await pg.query<{ token_hash: string }>('SELECT token_hash FROM sessions') expect(r.rows[0]?.token_hash).toBe(hashToken(token)) @@ -291,7 +291,7 @@ describe('the session cookie', () => { }) it('is refused once its row is gone, which is what signing out does', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) expect( (await app.inject({ method: 'GET', url: '/api/me', headers: read(cookie) })).statusCode, ).toBe(200) @@ -303,7 +303,7 @@ describe('the session cookie', () => { }) it('is refused after the idle bound, and the row is gone with it', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) clock.advance(SESSION_IDLE_MS + 1000) const r = await app.inject({ method: 'GET', url: '/api/me', headers: read(cookie) }) expect(r.statusCode).toBe(401) @@ -317,7 +317,7 @@ describe('the session cookie', () => { // refusal after enough time has passed would also come from the idle bound, // which is why this walks until the first refusal and pins where it landed. it('is refused after the absolute bound however busy it was', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const step = SESSION_IDLE_MS - 1000 // Twice the absolute bound, so a session that is never ended is a loop // that finishes with nothing recorded rather than one that runs forever. @@ -340,7 +340,7 @@ describe('the session cookie', () => { }) it('refuses a token that was never issued, whatever shape it is', async () => { - await signedIn(app, pg) + await signedIn(app, pg, clock.now()) for (const value of ['', 'x', 'a'.repeat(43), 'a'.repeat(4000)]) { const r = await app.inject({ method: 'GET', @@ -354,7 +354,7 @@ describe('the session cookie', () => { describe('the cross-site write guard', () => { it('refuses a write with no Origin', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'DELETE', url: '/api/session', @@ -377,7 +377,7 @@ describe('the cross-site write guard', () => { ['a host the admin host is a prefix of', `https://${ADMIN_HOST}.evil.example.com`], ['null, as a sandboxed frame sends', 'null'], ])('refuses a write whose Origin is %s', async (_label, origin) => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'POST', url: '/api/password', @@ -397,7 +397,7 @@ describe('the cross-site write guard', () => { }) it('allows a read with no Origin, so a browser can load the API at all', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'GET', url: '/api/sessions', @@ -412,7 +412,7 @@ describe('an API key', () => { // to exist before any key does. Every test here needs one, so it is made // once, here, and none of them calls `signedIn` — that would create a second. beforeEach(async () => { - await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD }) + await createAccount(pg, { email: ADMIN_EMAIL, password: ADMIN_PASSWORD, now: clock.now() }) }) async function keyFor(pg_: typeof pg, name = 'scripting'): Promise { @@ -626,7 +626,7 @@ describe('what a body may say', () => { ['an owner', { ...change, owner: 'somebody' }], ['a password hash', { ...change, passwordHash: 'anything at all' }], ])('refuses a body carrying %s', async (_label, payload) => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'POST', url: '/api/password', @@ -647,7 +647,7 @@ describe('what a body may say', () => { }) it('answers a body that is not JSON with 400, not 500', async () => { - const cookie = await signedIn(app, pg) + const cookie = await signedIn(app, pg, clock.now()) const r = await app.inject({ method: 'POST', url: '/api/password', diff --git a/packages/admin/src/domains.test.ts b/packages/admin/src/domains.test.ts index 0a41527..6cc75e9 100644 --- a/packages/admin/src/domains.test.ts +++ b/packages/admin/src/domains.test.ts @@ -71,7 +71,7 @@ beforeEach(async () => { await pg.query('TRUNCATE domains CASCADE') resolver = new FakeResolver() app = testApp(pg, clock, { resolver: () => resolver }) - cookie = await signedIn(app, pg) + cookie = await signedIn(app, pg, clock.now()) }) afterEach(async () => { diff --git a/packages/admin/src/keys.test.ts b/packages/admin/src/keys.test.ts index e0cc410..db48ed2 100644 --- a/packages/admin/src/keys.test.ts +++ b/packages/admin/src/keys.test.ts @@ -20,7 +20,7 @@ beforeEach(async () => { await pg.query('TRUNCATE admin_account, admin_recovery_codes, sessions, api_keys') await pg.query('TRUNCATE domains CASCADE') app = testApp(pg, clock) - cookie = await signedIn(app, pg) + cookie = await signedIn(app, pg, clock.now()) }) afterEach(async () => { diff --git a/packages/admin/src/links.test.ts b/packages/admin/src/links.test.ts index b02eb82..51762e8 100644 --- a/packages/admin/src/links.test.ts +++ b/packages/admin/src/links.test.ts @@ -21,7 +21,7 @@ beforeEach(async () => { await pg.query('TRUNCATE admin_account, admin_recovery_codes, sessions, api_keys') await pg.query('TRUNCATE domains CASCADE') app = testApp(pg, clock) - cookie = await signedIn(app, pg) + cookie = await signedIn(app, pg, clock.now()) await app.inject({ method: 'POST', url: '/api/domains', diff --git a/packages/admin/src/settings-routes.test.ts b/packages/admin/src/settings-routes.test.ts index 72c83e3..593a9d1 100644 --- a/packages/admin/src/settings-routes.test.ts +++ b/packages/admin/src/settings-routes.test.ts @@ -20,7 +20,7 @@ beforeEach(async () => { await pg.query('TRUNCATE settings') await pg.query('INSERT INTO settings DEFAULT VALUES') app = testApp(pg, clock) - cookie = await signedIn(app, pg) + cookie = await signedIn(app, pg, clock.now()) }) afterEach(async () => { diff --git a/packages/admin/src/testing.ts b/packages/admin/src/testing.ts index 7ea10fe..a1420d2 100644 --- a/packages/admin/src/testing.ts +++ b/packages/admin/src/testing.ts @@ -124,12 +124,21 @@ export async function signInAs(app: FastifyInstance, password = ADMIN_PASSWORD): return cookieFrom(r.headers['set-cookie']) } -/** Creates the account and signs in; returns the cookie header a browser would hold. */ +/** + * Creates the account and signs in; returns the cookie header a browser would + * hold. + * + * `now` is not optional, and it is the clock the app under test was built + * with. A fixture that stamped the account from real time while the app stamps + * every later write from a frozen clock would write a row whose history reads + * as out of order, which is the shape this suite exists to catch elsewhere. + */ export async function signedIn( app: FastifyInstance, pg: Pool, + now: Date, password = ADMIN_PASSWORD, ): Promise { - await createAccount(pg, { email: ADMIN_EMAIL, password }) + await createAccount(pg, { email: ADMIN_EMAIL, password, now }) return signInAs(app, password) } diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index 8f125c5..995ef30 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -335,6 +335,21 @@ describe('clickmonk settings', () => { }) }) + // The third substitution, and the one that is easiest to miss because the + // command looks like it worked: before it, nothing was being deleted at all; + // after it, clicks go at ninety days. The operator asked about a threshold. + it('says so when it wrote a settings row that was not there', async () => { + await pg.query('DELETE FROM settings') + lines.length = 0 + expect(await run('settings', 'set', '--abuser-threshold', '61')).toBe(0) + expect(lines[0]).toBe( + 'note: there was no settings row, so this command has written one; nothing was being deleted before, and clicks are now kept for 90 days and addresses for 30 days', + ) + expect(await stored()).toEqual({ raw_retention_days: 90, ip_retention_days: 30 }) + // And the change the operator actually asked for landed. + expect(lines).toContain('abuser threshold: 61 requests a minute from one client') + }) + it('prints the retention periods, in days and as for ever', async () => { await setRetention(90, null) lines.length = 0 diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 55c9a59..ad59637 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -46,7 +46,7 @@ import { runDomainChecks, } from '@clickmonk/worker/domains' import { createLink } from '@clickmonk/worker/links' -import { readSettings, updateSettings } from '@clickmonk/worker/settings' +import { type SettingsSubstitution, readSettings, updateSettings } from '@clickmonk/worker/settings' import type { ZodError } from 'zod' export interface CliDeps { @@ -490,7 +490,7 @@ async function adminCreate(args: string[], d: CliDeps): Promise { } const password = await readPassword(d) try { - await createAccount(d.pg, { email, password }) + await createAccount(d.pg, { email, password, now: d.now?.() ?? new Date() }) } catch (err) { if (err instanceof AccountExistsError) throw new Rejected(err.message) throw err @@ -681,8 +681,7 @@ async function settingsSet(args: string[], d: CliDeps): Promise { // Arrays rather than nullable locals: the hooks run inside the call below, // and a `let` assigned only from a callback is narrowed to its initialiser // by the compiler. - const replaced: string[] = [] - const replacedTraffic: string[] = [] + const substituted: SettingsSubstitution[] = [] const next = await updateSettings( d.pg, d.now?.() ?? new Date(), @@ -700,28 +699,32 @@ async function settingsSet(args: string[], d: CliDeps): Promise { ipRetentionDays: period(values['keep-addresses'], current.retention.ipRetentionDays), }, }), - { - onRetentionReplaced: (why) => replaced.push(why), - onTrafficReplaced: (why) => replacedTraffic.push(why), - }, + { onSubstituted: (subs) => substituted.push(...subs) }, ) - // Both rewrites are said, not done quietly. This command may have been about - // one field and have overwritten the other half of the row on its way past, - // and neither is a change an operator should first learn about from clicks + // Every substitution is said, not done quietly, and they all arrive through + // the one report rather than as a note per case. This command may have been + // about one field and have written the rest of the row on its way past, and + // none of that is a change an operator should first learn about from clicks // that are gone or from traffic that stopped being blocked. // - // The traffic note points at the lines below rather than naming the values, - // because there are five of them and they are already about to be printed; - // the retention note names its two, because they are two. - for (const why of replacedTraffic) { - d.out( - `note: the stored traffic settings could not be read (${why}), so this command has rewritten them; what is stored now is below`, - ) - } - for (const why of replaced) { - d.out( - `note: the stored retention could not be read (${why}), so this command has rewritten it as keep clicks ${days(next.retention.rawRetentionDays)}, keep addresses ${days(next.retention.ipRetentionDays)}`, - ) + // The kept periods are named in full because there are two of them; the + // traffic settings point at the lines printed below instead, because there + // are five and they are about to be printed anyway. + const kept = `clicks are now kept for ${days(next.retention.rawRetentionDays)} and addresses for ${days(next.retention.ipRetentionDays)}` + for (const sub of substituted) { + if (sub.what === 'row') { + d.out( + `note: there was no settings row, so this command has written one; nothing was being deleted before, and ${kept}`, + ) + } else if (sub.what === 'traffic') { + d.out( + `note: the stored traffic settings could not be read (${sub.why}), so this command has rewritten them; what is stored now is below`, + ) + } else { + d.out( + `note: the stored retention could not be read (${sub.why}), so this command has rewritten it as keep clicks ${days(next.retention.rawRetentionDays)}, keep addresses ${days(next.retention.ipRetentionDays)}`, + ) + } } printSettings(next, d) } diff --git a/packages/db/migrations/postgres/007_reporting_settings.sql b/packages/db/migrations/postgres/007_reporting_settings.sql index 5c9bd4d..3b2a0c9 100644 --- a/packages/db/migrations/postgres/007_reporting_settings.sql +++ b/packages/db/migrations/postgres/007_reporting_settings.sql @@ -12,6 +12,16 @@ -- on exactly one of the tables that reload reads — and the last of them — so -- it can block a reload but can never be half of a deadlock. -- +-- The DEFAULTs backfill 90 and 30 onto the row that is already there, which is +-- safe here for one reason only: **there is no released version to upgrade +-- from.** Nothing is deployed, so no operator has ever chosen a period and no +-- existing install can be given one it did not ask for. After a first release +-- that reason is gone, and the rule that replaces it is the opposite: a +-- migration may not start deleting data an operator never chose, so a later +-- retention column arrives NULL — keep for ever — and an operator opts in. +-- A default is a fine way to answer a question; it is not a way to answer one +-- nobody was asked. +-- -- Deliberately NOT a check that the IP period is the shorter of the two. It -- usually is, and an IP period that outlives the clicks it belongs to simply -- never runs, but a constraint saying so would refuse an operator who is diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts index bde3f8c..d53d6af 100644 --- a/packages/worker/src/settings.test.ts +++ b/packages/worker/src/settings.test.ts @@ -2,7 +2,12 @@ import { DEFAULT_INSTALL_SETTINGS, DEFAULT_TRAFFIC_SETTINGS } from '@clickmonk/c import { type Pool, createPgPool } from '@clickmonk/db' import { TEST_PG_URL, resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { readSettings, updateSettings, writeSettings } from './settings.js' +import { + type SettingsSubstitution, + readSettings, + updateSettings, + writeSettings, +} from './settings.js' /** * Every connection this file's pool opens reports this name to Postgres, so @@ -263,4 +268,42 @@ describe('updateSettings', () => { retention: { rawRetentionDays: 5, ipRetentionDays: 30 }, }) }) + + /** + * The third substitution, and the one two separate callbacks could not + * express. Before this call the reader answers `retention: null` and a pass + * deletes nothing; after it the row says ninety days and the pass deletes by + * it. Nothing in the command asked for that, so it is reported. + */ + it('reports writing a row that was not there', async () => { + await pool.query('TRUNCATE settings') + const subs: SettingsSubstitution[][] = [] + await updateSettings( + pool, + NOW, + (current) => ({ + ...current, + traffic: { ...current.traffic, abuserThreshold: 61 }, + }), + { onSubstituted: (s) => subs.push(s) }, + ) + expect(subs).toEqual([[{ what: 'row', why: '' }]]) + }) + + // The report is of a substitution, not of a write: a command that started + // from the row it found substituted nothing, and a hook called every time + // would be a banner rather than a warning. + it('reports nothing when it started from the row it read', async () => { + const subs: SettingsSubstitution[][] = [] + await updateSettings( + pool, + NOW, + (current) => ({ + ...current, + traffic: { ...current.traffic, abuserThreshold: 61 }, + }), + { onSubstituted: (s) => subs.push(s) }, + ) + expect(subs).toEqual([]) + }) }) diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts index 6cea3af..b9ed4c3 100644 --- a/packages/worker/src/settings.ts +++ b/packages/worker/src/settings.ts @@ -165,21 +165,35 @@ export async function writeSettings(pg: Pool, next: InstallSettings, now: Date): } /** - * Told, before the write and inside the transaction, that a half of the stored - * row could not be read and the defaults stood in for it. The argument is why. + * One place this function had to invent the value it started from, because the + * stored row did not supply one. * - * **There are two of these, not one.** Either half of a row can be unreadable, - * and in both cases this function writes the defaults over it while the - * operator was changing something else — a threshold, a period — and says - * nothing unless the surface in front of it does. Both are the same harm in - * different units: a retention period nobody was told about is learned from - * clicks that are gone, and a traffic action nobody was told about is learned - * from traffic that was being blocked and is now only flagged. A caller with - * nowhere to print a note passes no hook and is unchanged. + * **This is an invariant, not a list of special cases.** Three times now a + * surface has had to be taught about one more silent substitution — an + * unreadable retention half, an unreadable traffic half, and a row that was + * not there at all, where the reader answers "nothing is being deleted" before + * the command and "deleted after ninety days" after it. Each was the same + * thing: **the row that was written is not derived from the row that was + * read**, and the operator was changing something else at the time. So the + * rule is stated once, here, and every case reports through it. A fourth case + * needs no new callback, and a surface that handles this one handles it. + * + * `what` says which part was invented rather than read: one half, or the whole + * row. `why` is empty for a missing row — there is nothing to explain beyond + * its absence — and carries the schema's issues otherwise. */ +export type SettingsSubstitution = + | { what: 'row'; why: '' } + | { what: 'traffic'; why: string } + | { what: 'retention'; why: string } + export interface UpdateSettingsHooks { - onRetentionReplaced?: (why: string) => void - onTrafficReplaced?: (why: string) => void + /** + * Called, before the write and inside the transaction, with every + * substitution this call made — and not called at all when it made none. + * A caller with nowhere to print them passes no hook and is unchanged. + */ + onSubstituted?: (subs: SettingsSubstitution[]) => void } /** @@ -203,15 +217,23 @@ export async function updateSettings( const client: PoolClient = await pg.connect() try { await client.query('BEGIN') - await client.query('INSERT INTO settings DEFAULT VALUES ON CONFLICT DO NOTHING') + // Whether this inserted tells us the row was not there, which is the third + // substitution and the one the earlier two callbacks could not express: + // before this statement the reader answered "nothing is being deleted", + // and after the write it answers a period. `ON CONFLICT DO NOTHING` + // reports 1 row when it inserted and 0 when the row already existed. + const ins = await client.query('INSERT INTO settings DEFAULT VALUES ON CONFLICT DO NOTHING') const r = await client.query(`SELECT ${COLUMNS} FROM settings FOR UPDATE`) const row = r.rows[0] if (!row) throw new Error('the settings row is missing after writing it') const { read, trafficIssues, retentionIssues } = parseRow(row) - if (trafficIssues !== null) hooks.onTrafficReplaced?.(trafficIssues) + const subs: SettingsSubstitution[] = [] + if ((ins.rowCount ?? 0) > 0) subs.push({ what: 'row', why: '' }) + if (trafficIssues !== null) subs.push({ what: 'traffic', why: trafficIssues }) if (read.retention === null && retentionIssues !== null) { - hooks.onRetentionReplaced?.(retentionIssues) + subs.push({ what: 'retention', why: retentionIssues }) } + if (subs.length > 0) hooks.onSubstituted?.(subs) const current: InstallSettings = { // The defaults, when the stored half could not be read — and the hook // above is what keeps that from being silent. From 29e0b56060094b45fdcab679bc56ca7ba74c97ae Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 12:19:26 +0000 Subject: [PATCH 12/65] clicks: hourly rollups, as sets rather than counters --- packages/core/src/index.ts | 2 +- packages/core/src/version.test.ts | 2 +- .../clickhouse/008_click_rollups.sql | 133 +++++++++ packages/db/src/schema-rollups.test.ts | 274 ++++++++++++++++++ 4 files changed, 409 insertions(+), 2 deletions(-) create mode 100644 packages/db/migrations/clickhouse/008_click_rollups.sql create mode 100644 packages/db/src/schema-rollups.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index fc2db88..ba82c28 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,7 +2,7 @@ * The highest migration version this build understands. Every migration bumps * it; `migrate()` refuses to run against a database whose ledger is newer. */ -export const SCHEMA_VERSION = 7 +export const SCHEMA_VERSION = 8 export * from './admin-host.js' export * from './attempts.js' diff --git a/packages/core/src/version.test.ts b/packages/core/src/version.test.ts index 3582dd4..a0ec073 100644 --- a/packages/core/src/version.test.ts +++ b/packages/core/src/version.test.ts @@ -5,6 +5,6 @@ describe('SCHEMA_VERSION', () => { it('is the highest migration version this build knows about', () => { // Bumped with every migration added. packages/db's // schema-version.test.ts ties it to the migration files on disk. - expect(SCHEMA_VERSION).toBe(7) + expect(SCHEMA_VERSION).toBe(8) }) }) diff --git a/packages/db/migrations/clickhouse/008_click_rollups.sql b/packages/db/migrations/clickhouse/008_click_rollups.sql new file mode 100644 index 0000000..09203ad --- /dev/null +++ b/packages/db/migrations/clickhouse/008_click_rollups.sql @@ -0,0 +1,133 @@ +-- Hourly rollups, written by materialized views as clicks arrive. Every report +-- but the click log and its export reads these and never the raw table. +-- +-- WHY EVERY MEASURE IS A SET AND NOT A COUNTER. A materialized view fires on +-- the block that was inserted, before any merge, so it never sees the +-- ReplacingMergeTree collapse that makes a re-shipped segment harmless in +-- `clicks`. The spool's contract says a segment will be shipped twice: the +-- worker deletes a segment after ClickHouse accepts it, and a crash in between +-- re-ships it byte for byte. A count() column would double that segment's +-- clicks here while `uniqExact(click_id)` on the raw table stayed right, and +-- the chart and the log would disagree with nobody able to say which was +-- true. uniqExact over the click id makes the second copy free. +-- +-- AND WHY VISITORS ARE A SET TOO, for a second reason: they do not sum. A +-- visitor who clicks two links is one visitor for the install and one for each +-- link, so adding the per-link numbers gives two. Merging the states gives +-- one. The same holds across any other dimension: one visitor with a human +-- click and a bot click is one visitor. +-- +-- WHY TWO TABLES. Keying one table by every dimension is a cross product, and +-- a referrer host is whatever somebody linked from: one click carrying a new +-- referrer, browser and country makes its own row, and at worst the rollup +-- grows as fast as the raw table it exists to avoid reading. So the three +-- closed dimensions — class, action and outcome, none of which can gain a +-- value without a new click record version — are keys here, and the six open +-- ones are rows of clicks_hourly_dim below. +-- +-- The state columns are named `_state` because a column called `clicks` that +-- holds an AggregateFunction is a name a query cannot alias its own output to: +-- `uniqExactMerge(clicks) AS clicks, uniqExactMergeIf(clicks, …)` resolves the +-- second `clicks` to the alias and fails with "Illegal type UInt64 of argument +-- for aggregate function with Merge suffix", which reads like a schema fault +-- and is not one. +-- +-- Nothing here is ever dropped by the retention pass. These tables are small, +-- and they are what answers a question older than the raw window. +CREATE TABLE IF NOT EXISTS clicks_hourly ( + hour DateTime('UTC'), + link_id UUID, + domain_id UUID, + traffic_class LowCardinality(String), + action LowCardinality(String), + outcome LowCardinality(String), + clicks_state AggregateFunction(uniqExact, UUID), + visitors_state AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(hour) +ORDER BY (hour, link_id, domain_id, traffic_class, action, outcome); + +-- `domain_id` is in the sort key and not merely carried: a click on no link at +-- all — an unknown slug, a domain root — has the zero link id, and its domain +-- is whichever domain was asked. Left out of the key, rows with the same hour +-- and zero link id would merge and one arbitrary domain id would win. +-- +-- `hour` leads the key because every report bounds a window first, and the +-- link comes next because per-link is the other filter every report has. + +-- `domain_id` is here although nothing reads it yet, and that is a decision +-- rather than an oversight. A per-domain breakdown is the first thing an +-- operator with several domains asks for, and adding a key column to an +-- aggregating rollup after the fact is a migration *and* a backfill of every +-- row already in it. It also costs nothing now: the only groups that can span +-- two domains are the ones on the zero link id, because every other click's +-- domain follows from its link. It is in the sort key for the same reason it is +-- on the table above — rows that differ only by domain must not merge. +CREATE TABLE IF NOT EXISTS clicks_hourly_dim ( + hour DateTime('UTC'), + link_id UUID, + domain_id UUID, + dimension LowCardinality(String), + value String, + clicks_state AggregateFunction(uniqExact, UUID), + visitors_state AggregateFunction(uniqExact, String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(hour) +ORDER BY (hour, link_id, domain_id, dimension, value); + +-- One view per dimension, all writing here. An empty value is kept rather than +-- filtered out: an empty country is a click whose address could not be looked +-- up, an empty referrer is a visitor who arrived with none, and an empty target +-- is a click that reached no target. A breakdown that silently dropped them +-- would not add up to the total beside it. + +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_mv TO clicks_hourly AS +SELECT + toStartOfHour(time) AS hour, + link_id, + domain_id, + traffic_class, + action, + outcome, + uniqExactState(click_id) AS clicks_state, + uniqExactState(visitor_id) AS visitors_state +FROM clicks +GROUP BY hour, link_id, domain_id, traffic_class, action, outcome; + +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_country_mv TO clicks_hourly_dim AS +SELECT toStartOfHour(time) AS hour, link_id, domain_id, 'country' AS dimension, country AS value, + uniqExactState(click_id) AS clicks_state, uniqExactState(visitor_id) AS visitors_state +FROM clicks GROUP BY hour, link_id, domain_id, value; + +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_device_mv TO clicks_hourly_dim AS +SELECT toStartOfHour(time) AS hour, link_id, domain_id, 'device' AS dimension, device AS value, + uniqExactState(click_id) AS clicks_state, uniqExactState(visitor_id) AS visitors_state +FROM clicks GROUP BY hour, link_id, domain_id, value; + +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_os_mv TO clicks_hourly_dim AS +SELECT toStartOfHour(time) AS hour, link_id, domain_id, 'os' AS dimension, os AS value, + uniqExactState(click_id) AS clicks_state, uniqExactState(visitor_id) AS visitors_state +FROM clicks GROUP BY hour, link_id, domain_id, value; + +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_browser_mv TO clicks_hourly_dim AS +SELECT toStartOfHour(time) AS hour, link_id, domain_id, 'browser' AS dimension, browser AS value, + uniqExactState(click_id) AS clicks_state, uniqExactState(visitor_id) AS visitors_state +FROM clicks GROUP BY hour, link_id, domain_id, value; + +-- The referrer is stored whole and rolled up by host, which is what a report +-- shows: `domain()` is ClickHouse's own host extraction, so the rollup and the +-- raw log cannot come to disagree about where a click came from. A referrer +-- that is not a URL gives an empty host, which is the same value as none. +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_referrer_mv TO clicks_hourly_dim AS +SELECT toStartOfHour(time) AS hour, link_id, domain_id, 'referrer' AS dimension, domain(referrer) AS value, + uniqExactState(click_id) AS clicks_state, uniqExactState(visitor_id) AS visitors_state +FROM clicks GROUP BY hour, link_id, domain_id, value; + +-- Which target rotation chose, which is the per-target count the product +-- promises. Empty for a click that reached no target at all. +CREATE MATERIALIZED VIEW IF NOT EXISTS clicks_hourly_target_mv TO clicks_hourly_dim AS +SELECT toStartOfHour(time) AS hour, link_id, domain_id, 'target' AS dimension, target_id AS value, + uniqExactState(click_id) AS clicks_state, uniqExactState(visitor_id) AS visitors_state +FROM clicks GROUP BY hour, link_id, domain_id, value; diff --git a/packages/db/src/schema-rollups.test.ts b/packages/db/src/schema-rollups.test.ts new file mode 100644 index 0000000..1dbf1df --- /dev/null +++ b/packages/db/src/schema-rollups.test.ts @@ -0,0 +1,274 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { resetDatabases, testCh, testPg } from './testing.js' + +const pool = testPg() +const ch = testCh() + +beforeAll(async () => { + await resetDatabases(pool, ch) +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +const LINK_A = '00000000-0000-4000-8000-0000000000a1' +const LINK_B = '00000000-0000-4000-8000-0000000000a2' +const DOMAIN = '00000000-0000-4000-8000-00000000000d' + +/** + * One click, as the shipper writes it. Fixture times are fixed rather than + * relative to now on purpose: this file drops a partition, and a fixed month + * is the only way the partition it drops is the one it inserted. + */ +const click = (over: Record = {}): Record => ({ + click_id: '01920000-0000-7000-8000-00000000000a', + time: '2026-09-24 10:15:00.000', + host: 'go.example.test', + path: '/a', + domain_id: DOMAIN, + link_id: LINK_A, + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/', + target_id: '00000000-0000-4000-8000-0000000000b1', + visitor_id: 'v1', + returning: 0, + country: 'DE', + region: '', + city: '', + geo_source: 'dbip', + device: 'desktop', + user_agent: 'ua', + referrer: 'https://blog.example.com/post', + ip: '198.51.100.77', + cap_unchecked: 0, + traffic_class: 'human', + signals: [], + action: '', + os: 'windows', + browser: 'chrome', + asn: 64500, + ...over, +}) + +const insert = (values: Record[]) => + ch.insert({ table: 'clicks', values, format: 'JSONEachRow' }) + +async function rows(query: string): Promise { + const rs = await ch.query({ query, format: 'JSONEachRow' }) + return rs.json() +} + +const totals = () => + rows<{ clicks: string; visitors: string }>( + `SELECT uniqExactMerge(clicks_state) AS clicks, uniqExactMerge(visitors_state) AS visitors + FROM clicks_hourly`, + ) + +describe('clickhouse schema 008: the hourly rollups', () => { + it('has the columns the reports read, with the states named so a query can alias its own output', async () => { + const hourly = await rows<{ name: string; type: string }>('DESCRIBE TABLE clicks_hourly') + expect(hourly.map((c) => [c.name, c.type])).toEqual([ + ['hour', "DateTime('UTC')"], + ['link_id', 'UUID'], + ['domain_id', 'UUID'], + ['traffic_class', 'LowCardinality(String)'], + ['action', 'LowCardinality(String)'], + ['outcome', 'LowCardinality(String)'], + ['clicks_state', 'AggregateFunction(uniqExact, UUID)'], + ['visitors_state', 'AggregateFunction(uniqExact, String)'], + ]) + const dim = await rows<{ name: string }>('DESCRIBE TABLE clicks_hourly_dim') + expect(dim.map((c) => c.name)).toEqual([ + 'hour', + 'link_id', + 'domain_id', + 'dimension', + 'value', + 'clicks_state', + 'visitors_state', + ]) + }) + + it('has one view per dimension and one for the hourly table', async () => { + const views = await rows<{ name: string }>( + "SELECT name FROM system.tables WHERE database = currentDatabase() AND engine = 'MaterializedView' ORDER BY name", + ) + expect(views.map((v) => v.name)).toEqual([ + 'clicks_hourly_browser_mv', + 'clicks_hourly_country_mv', + 'clicks_hourly_device_mv', + 'clicks_hourly_mv', + 'clicks_hourly_os_mv', + 'clicks_hourly_referrer_mv', + 'clicks_hourly_target_mv', + ]) + }) + + it('rolls a click up into the hour it happened in', async () => { + await insert([click()]) + expect(await totals()).toEqual([{ clicks: '1', visitors: '1' }]) + const byHour = await rows<{ hour: string; clicks: string }>( + 'SELECT toString(hour) AS hour, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly GROUP BY hour ORDER BY hour', + ) + expect(byHour).toEqual([{ hour: '2026-09-24 10:00:00', clicks: '1' }]) + }) + + /** + * The reason every measure is a set. The worker deletes a segment after + * ClickHouse accepts it; a crash in between re-ships it byte for byte. With + * a count() column this rollup would say two. + */ + it('counts a segment shipped twice once', async () => { + await insert([click()]) + expect(await totals()).toEqual([{ clicks: '1', visitors: '1' }]) + const raw = await rows<{ n: string }>('SELECT count() AS n FROM clicks') + expect(raw).toEqual([{ n: '2' }]) + }) + + it('counts one visitor on two links as one visitor for the install and one for each link', async () => { + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000000b', link_id: LINK_B, path: '/b' }), + ]) + expect(await totals()).toEqual([{ clicks: '2', visitors: '1' }]) + const perLink = await rows<{ link_id: string; clicks: string; visitors: string }>( + `SELECT link_id, uniqExactMerge(clicks_state) AS clicks, uniqExactMerge(visitors_state) AS visitors + FROM clicks_hourly GROUP BY link_id ORDER BY link_id`, + ) + expect(perLink).toEqual([ + { link_id: LINK_A, clicks: '1', visitors: '1' }, + { link_id: LINK_B, clicks: '1', visitors: '1' }, + ]) + }) + + it('counts one visitor with a human click and a bot click as one visitor', async () => { + await insert([ + click({ + click_id: '01920000-0000-7000-8000-00000000000c', + traffic_class: 'bot', + action: 'flag', + signals: ['ua_bot'], + }), + ]) + const byClass = await rows<{ traffic_class: string; clicks: string; visitors: string }>( + `SELECT traffic_class, uniqExactMerge(clicks_state) AS clicks, + uniqExactMerge(visitors_state) AS visitors + FROM clicks_hourly GROUP BY traffic_class ORDER BY traffic_class`, + ) + expect(byClass).toEqual([ + { traffic_class: 'bot', clicks: '1', visitors: '1' }, + { traffic_class: 'human', clicks: '2', visitors: '1' }, + ]) + expect(await totals()).toEqual([{ clicks: '3', visitors: '1' }]) + }) + + it('separates two clicks on no link at all by the domain they asked for', async () => { + const zero = '00000000-0000-0000-0000-000000000000' + const other = '00000000-0000-4000-8000-00000000000e' + await insert([ + click({ + click_id: '01920000-0000-7000-8000-00000000001a', + link_id: zero, + outcome: 'not_found', + step: 'resolve', + status: 404, + }), + click({ + click_id: '01920000-0000-7000-8000-00000000001b', + link_id: zero, + domain_id: other, + outcome: 'not_found', + step: 'resolve', + status: 404, + }), + ]) + const byDomain = await rows<{ domain_id: string; clicks: string }>( + `SELECT domain_id, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly + WHERE link_id = toUUID('${zero}') GROUP BY domain_id ORDER BY domain_id`, + ) + expect(byDomain).toEqual([ + { domain_id: DOMAIN, clicks: '1' }, + { domain_id: other, clicks: '1' }, + ]) + }) + + it('keeps two clicks on no link at all apart by domain in the per-dimension rollup too', async () => { + // The same pair the test above inserted, read from the other table: without + // the domain in that table's key these two rows would merge and one + // arbitrary domain would win, which is the defect the column exists for + // even though no report reads it yet. + const zero = '00000000-0000-0000-0000-000000000000' + const other = '00000000-0000-4000-8000-00000000000e' + const byDomain = await rows<{ domain_id: string; clicks: string }>( + `SELECT domain_id, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim + WHERE dimension = 'country' AND link_id = toUUID('${zero}') + GROUP BY domain_id ORDER BY domain_id`, + ) + expect(byDomain).toEqual([ + { domain_id: DOMAIN, clicks: '1' }, + { domain_id: other, clicks: '1' }, + ]) + }) + + it('rolls the referrer up by host, and an absent referrer as an empty host', async () => { + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000002a', referrer: '' }), + click({ + click_id: '01920000-0000-7000-8000-00000000002b', + referrer: 'https://blog.example.com/other', + }), + click({ click_id: '01920000-0000-7000-8000-00000000002c', referrer: 'not a url' }), + ]) + const byReferrer = await rows<{ value: string; clicks: string }>( + `SELECT value, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim + WHERE dimension = 'referrer' GROUP BY value ORDER BY value`, + ) + // These three are the only clicks in this file that override the referrer, + // and two of the three have no host: an empty string, and a value that is + // not a URL, which `domain()` also reads as no host. Every earlier click — + // the first fixture, the second link, the bot, and the 404 pair — keeps + // the default `https://blog.example.com/post`, which is six distinct + // clicks with that host against two with none. Counted, not guessed: this + // file's inserts accumulate, and the arithmetic is easy to get backwards. + expect(byReferrer).toEqual([ + { value: '', clicks: '2' }, + { value: 'blog.example.com', clicks: '6' }, + ]) + }) + + it('holds each open dimension as its own rows', async () => { + const dims = await rows<{ dimension: string }>( + 'SELECT DISTINCT dimension FROM clicks_hourly_dim ORDER BY dimension', + ) + expect(dims.map((d) => d.dimension)).toEqual([ + 'browser', + 'country', + 'device', + 'os', + 'referrer', + 'target', + ]) + }) + + /** + * The point of rollups outliving raw clicks, run against the engine rather + * than asserted: the retention pass drops a raw partition and every number + * a report shows is still there. + */ + it('keeps every number when the raw partition is dropped', async () => { + // Written out rather than read before and compared after: two reads of the + // same query agree with each other when the views never fired and both + // are empty, which is exactly the defect this test exists to catch. Eight + // distinct click ids have been inserted by this point in the file — the + // first fixture (twice, one id), the second link, the bot, the 404 pair + // and the three referrer clicks — all by the one visitor the helper + // defaults to. + expect(await totals()).toEqual([{ clicks: '8', visitors: '1' }]) + await ch.command({ query: "ALTER TABLE clicks DROP PARTITION ID '202609'" }) + expect(await rows<{ n: string }>('SELECT count() AS n FROM clicks')).toEqual([{ n: '0' }]) + expect(await totals()).toEqual([{ clicks: '8', visitors: '1' }]) + }) +}) From 4753cb30025a924342459566483361c1c5bc243b Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 12:24:30 +0000 Subject: [PATCH 13/65] clicks: pin the closed dimensions and the link in the rollup keys --- packages/db/src/schema-rollups.test.ts | 92 ++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/packages/db/src/schema-rollups.test.ts b/packages/db/src/schema-rollups.test.ts index 1dbf1df..123340a 100644 --- a/packages/db/src/schema-rollups.test.ts +++ b/packages/db/src/schema-rollups.test.ts @@ -272,3 +272,95 @@ describe('clickhouse schema 008: the hourly rollups', () => { expect(await totals()).toEqual([{ clicks: '8', visitors: '1' }]) }) }) + +/** + * The rest of the file reads whatever has accumulated in the 10:00 hour. These + * two use hours of their own instead, so they neither shift nor depend on the + * running arithmetic above, and they run after the partition drop because a + * rollup does not need the raw rows. + * + * Each inserts its clicks in ONE call on purpose. A materialized view fires + * once per inserted block and aggregates within it, and AggregatingMergeTree + * collapses equal sort keys as it writes the part — so a key column left out + * of the table shows up immediately, rather than only once a background merge + * happens to run. + */ +describe('clickhouse schema 008: the key columns that keep rows apart', () => { + const H11 = "toDateTime('2026-09-24 11:00:00', 'UTC')" + const H12 = "toDateTime('2026-09-24 12:00:00', 'UTC')" + + /** + * Class, action and outcome are keys rather than rows of the per-dimension + * table, which only holds if each of them keeps rows apart on its own. Two of + * these four clicks differ from a third only in their outcome, and two only + * in their action, so dropping either column from the key merges a pair and + * an arbitrary value wins. + */ + it('keeps two clicks apart when they differ only in outcome, and only in action', async () => { + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000003a', time: '2026-09-24 11:05:00.000' }), + click({ + click_id: '01920000-0000-7000-8000-00000000003b', + time: '2026-09-24 11:05:00.000', + outcome: 'capped', + }), + click({ + click_id: '01920000-0000-7000-8000-00000000003c', + time: '2026-09-24 11:05:00.000', + traffic_class: 'bot', + action: 'flag', + signals: ['ua_bot'], + }), + click({ + click_id: '01920000-0000-7000-8000-00000000003d', + time: '2026-09-24 11:05:00.000', + traffic_class: 'bot', + action: 'nothing', + signals: ['ua_bot'], + }), + ]) + const byOutcome = await rows<{ outcome: string; clicks: string }>( + `SELECT outcome, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly + WHERE hour = ${H11} GROUP BY outcome ORDER BY outcome`, + ) + expect(byOutcome).toEqual([ + { outcome: 'capped', clicks: '1' }, + { outcome: 'target', clicks: '3' }, + ]) + const byAction = await rows<{ action: string; clicks: string }>( + `SELECT action, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly + WHERE hour = ${H11} GROUP BY action ORDER BY action`, + ) + expect(byAction).toEqual([ + { action: '', clicks: '2' }, + { action: 'flag', clicks: '1' }, + { action: 'nothing', clicks: '1' }, + ]) + }) + + /** + * The link is a key of the per-dimension table too, and every report that + * breaks a single link down by country or device reads it that way. These two + * clicks differ only in their link, so without the column in the key they + * merge into one row under whichever link won. + */ + it('keeps two clicks apart in the per-dimension rollup when they differ only in link', async () => { + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000004a', time: '2026-09-24 12:05:00.000' }), + click({ + click_id: '01920000-0000-7000-8000-00000000004b', + time: '2026-09-24 12:05:00.000', + link_id: LINK_B, + path: '/b', + }), + ]) + const perLink = await rows<{ link_id: string; clicks: string }>( + `SELECT link_id, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim + WHERE dimension = 'country' AND hour = ${H12} GROUP BY link_id ORDER BY link_id`, + ) + expect(perLink).toEqual([ + { link_id: LINK_A, clicks: '1' }, + { link_id: LINK_B, clicks: '1' }, + ]) + }) +}) From 1cd403f69838b6473d6e587ff33aa6ad520c5e45 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 12:49:04 +0000 Subject: [PATCH 14/65] clicks: pin a rollup key where a merge can collapse it --- .../clickhouse/008_click_rollups.sql | 25 ++- packages/db/src/schema-rollups.test.ts | 202 ++++++++++++++++-- 2 files changed, 199 insertions(+), 28 deletions(-) diff --git a/packages/db/migrations/clickhouse/008_click_rollups.sql b/packages/db/migrations/clickhouse/008_click_rollups.sql index 09203ad..c1142df 100644 --- a/packages/db/migrations/clickhouse/008_click_rollups.sql +++ b/packages/db/migrations/clickhouse/008_click_rollups.sql @@ -34,6 +34,18 @@ -- -- Nothing here is ever dropped by the retention pass. These tables are small, -- and they are what answers a question older than the raw window. +-- +-- WHY THE KEY BELOW IS WHAT IT IS. `hour` leads it because every report bounds +-- a window first, and the link comes next because per-link is the other filter +-- every report has. Every one of the six columns is in the key rather than +-- merely carried, and a column left out of it is not a smaller breakdown but a +-- wrong one: AggregatingMergeTree merges rows that share a sort key, so two +-- rows differing only in the missing column become one, its value taken from +-- whichever row the merge read first, while the total over the table stays +-- right. `domain_id` needs saying twice over, because a click on no link at +-- all — an unknown slug, a domain root — has the zero link id and its domain +-- is whichever domain was asked, so without it in the key those rows merge +-- across domains. CREATE TABLE IF NOT EXISTS clicks_hourly ( hour DateTime('UTC'), link_id UUID, @@ -48,14 +60,13 @@ ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(hour) ORDER BY (hour, link_id, domain_id, traffic_class, action, outcome); --- `domain_id` is in the sort key and not merely carried: a click on no link at --- all — an unknown slug, a domain root — has the zero link id, and its domain --- is whichever domain was asked. Left out of the key, rows with the same hour --- and zero link id would merge and one arbitrary domain id would win. +-- The same holds for the key below, and `dimension` is the column it is +-- easiest to think safe: the six views write one table, and a click with no +-- country, no referrer and no target writes an empty value under three +-- different dimensions in the same hour for the same link. Those three rows +-- differ in nothing but `dimension`, so leaving it out of the key does not +-- blur a breakdown — it makes two of the six dimensions disappear. -- --- `hour` leads the key because every report bounds a window first, and the --- link comes next because per-link is the other filter every report has. - -- `domain_id` is here although nothing reads it yet, and that is a decision -- rather than an oversight. A per-domain breakdown is the first thing an -- operator with several domains asks for, and adding a key column to an diff --git a/packages/db/src/schema-rollups.test.ts b/packages/db/src/schema-rollups.test.ts index 123340a..ddc14c0 100644 --- a/packages/db/src/schema-rollups.test.ts +++ b/packages/db/src/schema-rollups.test.ts @@ -81,15 +81,19 @@ describe('clickhouse schema 008: the hourly rollups', () => { ['clicks_state', 'AggregateFunction(uniqExact, UUID)'], ['visitors_state', 'AggregateFunction(uniqExact, String)'], ]) - const dim = await rows<{ name: string }>('DESCRIBE TABLE clicks_hourly_dim') - expect(dim.map((c) => c.name)).toEqual([ - 'hour', - 'link_id', - 'domain_id', - 'dimension', - 'value', - 'clicks_state', - 'visitors_state', + const dim = await rows<{ name: string; type: string }>('DESCRIBE TABLE clicks_hourly_dim') + expect(dim.map((c) => [c.name, c.type])).toEqual([ + ['hour', "DateTime('UTC')"], + ['link_id', 'UUID'], + ['domain_id', 'UUID'], + ['dimension', 'LowCardinality(String)'], + // A referrer host is whatever somebody linked from, so `value` is a plain + // String: the dictionary a LowCardinality column keeps would grow without + // a bound here, which is the same reason these six dimensions are rows + // rather than keys of the table above. + ['value', 'String'], + ['clicks_state', 'AggregateFunction(uniqExact, UUID)'], + ['visitors_state', 'AggregateFunction(uniqExact, String)'], ]) }) @@ -226,13 +230,15 @@ describe('clickhouse schema 008: the hourly rollups', () => { `SELECT value, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim WHERE dimension = 'referrer' GROUP BY value ORDER BY value`, ) - // These three are the only clicks in this file that override the referrer, - // and two of the three have no host: an empty string, and a value that is - // not a URL, which `domain()` also reads as no host. Every earlier click — - // the first fixture, the second link, the bot, and the 404 pair — keeps - // the default `https://blog.example.com/post`, which is six distinct - // clicks with that host against two with none. Counted, not guessed: this - // file's inserts accumulate, and the arithmetic is easy to get backwards. + // The three clicks above are the only ones in the file that override the + // referrer. Two of them have no host — an empty string, and a value that is + // not a URL, which `domain()` also reads as no host — and the third is a + // second path on the default host. The five earlier clicks (the first + // fixture, the second link, the bot, and the 404 pair) all keep the default + // `https://blog.example.com/post`, so that host has those five plus this + // test's third: six, against the two with none. Counted, not guessed — + // this file's inserts accumulate and the arithmetic is easy to get + // backwards. expect(byReferrer).toEqual([ { value: '', clicks: '2' }, { value: 'blog.example.com', clicks: '6' }, @@ -279,11 +285,16 @@ describe('clickhouse schema 008: the hourly rollups', () => { * running arithmetic above, and they run after the partition drop because a * rollup does not need the raw rows. * - * Each inserts its clicks in ONE call on purpose. A materialized view fires - * once per inserted block and aggregates within it, and AggregatingMergeTree - * collapses equal sort keys as it writes the part — so a key column left out - * of the table shows up immediately, rather than only once a background merge - * happens to run. + * Each inserts its clicks in ONE call, which is what makes these two + * deterministic: a materialized view fires once per inserted block and + * aggregates within it, and AggregatingMergeTree collapses equal sort keys as + * it writes the part, so a column missing from the key shows up here without + * waiting on a merge. + * + * That reaches only the columns one block can vary. Rows in two parts stay + * apart until the parts are merged, whatever the key, so a column these + * fixtures vary across insert calls is not tested here at all — which is what + * the describe below forces with OPTIMIZE. */ describe('clickhouse schema 008: the key columns that keep rows apart', () => { const H11 = "toDateTime('2026-09-24 11:00:00', 'UTC')" @@ -364,3 +375,152 @@ describe('clickhouse schema 008: the key columns that keep rows apart', () => { ]) }) }) + +/** + * The sort keys themselves, which nothing above reaches: a column can be + * missing from a key for as long as no two rows that differ only in it ever sit + * in one part. `OPTIMIZE TABLE … FINAL` forces that merge, synchronously, and + * against the committed schema it changes no number — so every expectation here + * is the same before and after it. + * + * These clicks are in February rather than the September the rest of the file + * accumulates in, because parts in different partitions never merge together: + * the only rows the forced merge can collapse are the ones these tests wrote. + */ +describe('clickhouse schema 008: the sort keys, across a merge', () => { + const FEB = 'toYYYYMM(hour) = 202602' + + /** A click that reached no target, from an address no database knew, with no referrer. */ + const nothingToShow = (clickId: string, time: string) => + click({ + click_id: clickId, + time, + outcome: 'blocked', + step: 'traffic', + status: 403, + destination: '', + target_id: '', + country: '', + referrer: '', + traffic_class: 'bot', + action: 'block', + signals: ['ua_bot'], + }) + + const dimsAt = (hour: string) => + rows<{ dimension: string; value: string; clicks: string }>( + `SELECT dimension, value, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim + WHERE hour = toDateTime('${hour}', 'UTC') GROUP BY dimension, value ORDER BY dimension`, + ) + + /** + * What that one click must read as in every dimension. The value is asserted + * and not only the dimension name: a view whose SELECT alias does not match a + * target column is created without complaint and writes that column's + * default, so a dimension can be present and empty for a reason that has + * nothing to do with the click. + */ + const ONE_CLICK_EVERY_DIMENSION = [ + { dimension: 'browser', value: 'chrome', clicks: '1' }, + { dimension: 'country', value: '', clicks: '1' }, + { dimension: 'device', value: 'desktop', clicks: '1' }, + { dimension: 'os', value: 'windows', clicks: '1' }, + { dimension: 'referrer', value: '', clicks: '1' }, + { dimension: 'target', value: '', clicks: '1' }, + ] + + /** + * The hour, the link and the class: the three columns a report filters or + * groups on before anything else. Each of these four clicks differs from the + * first in exactly one of them, and each is inserted on its own so that no + * two of them are ever in one part until the merge. The total is asserted + * beside the breakdowns because that is the shape of the bug: a key column + * missing from the table leaves the total right and destroys every breakdown + * under it, so a report would answer 4 and then account for it wrongly. + */ + it('keeps the hour, the link and the class apart when a merge collapses the parts', async () => { + const base = { time: '2026-02-01 01:05:00.000', action: 'nothing' } + await insert([click({ ...base, click_id: '01920000-0000-7000-8000-00000000005a' })]) + await insert([ + click({ + ...base, + click_id: '01920000-0000-7000-8000-00000000005b', + time: '2026-02-01 02:05:00.000', + }), + ]) + await insert([ + click({ + ...base, + click_id: '01920000-0000-7000-8000-00000000005c', + link_id: LINK_B, + path: '/b', + }), + ]) + await insert([ + click({ + ...base, + click_id: '01920000-0000-7000-8000-00000000005d', + traffic_class: 'bot', + signals: ['ua_bot'], + }), + ]) + await ch.command({ query: 'OPTIMIZE TABLE clicks_hourly FINAL' }) + + // `AS at` rather than `AS hour`: an alias that repeats the column's name + // shadows the column for the WHERE beside it, and this query would fail + // with "Illegal type String of argument of function toYYYYMM" — the same + // trap the state columns carry their `_state` suffix to avoid. + const byHour = await rows<{ at: string; clicks: string }>( + `SELECT toString(hour) AS at, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly + WHERE ${FEB} GROUP BY hour ORDER BY hour`, + ) + expect(byHour).toEqual([ + { at: '2026-02-01 01:00:00', clicks: '3' }, + { at: '2026-02-01 02:00:00', clicks: '1' }, + ]) + const byLink = await rows<{ link_id: string; clicks: string }>( + `SELECT link_id, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly + WHERE ${FEB} GROUP BY link_id ORDER BY link_id`, + ) + expect(byLink).toEqual([ + { link_id: LINK_A, clicks: '3' }, + { link_id: LINK_B, clicks: '1' }, + ]) + const byClass = await rows<{ traffic_class: string; clicks: string }>( + `SELECT traffic_class, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly + WHERE ${FEB} GROUP BY traffic_class ORDER BY traffic_class`, + ) + expect(byClass).toEqual([ + { traffic_class: 'bot', clicks: '1' }, + { traffic_class: 'human', clicks: '3' }, + ]) + const feb = await rows<{ clicks: string }>( + `SELECT uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly WHERE ${FEB}`, + ) + expect(feb).toEqual([{ clicks: '4' }]) + }) + + /** + * An empty value is a value, for each of the three dimensions that can hold + * one. A view that filtered them out would leave a breakdown that no longer + * adds up to the total beside it, and the click below is the one that has all + * three at once. + */ + it('keeps an empty country, referrer and target as values of their own', async () => { + await insert([nothingToShow('01920000-0000-7000-8000-00000000006a', '2026-02-01 03:05:00.000')]) + expect(await dimsAt('2026-02-01 03:00:00')).toEqual(ONE_CLICK_EVERY_DIMENSION) + }) + + /** + * And `dimension` is a key column, which the click above is also what proves: + * its country, referrer and target rows share an hour, a link, a domain and + * an empty value, so they differ in nothing else. Without `dimension` in the + * key a merge folds the three into one and two of the six dimensions are + * gone — not a blurred breakdown but a missing report. + */ + it('keeps the six dimensions apart when three of them share an empty value', async () => { + await insert([nothingToShow('01920000-0000-7000-8000-00000000007a', '2026-02-01 04:05:00.000')]) + await ch.command({ query: 'OPTIMIZE TABLE clicks_hourly_dim FINAL' }) + expect(await dimsAt('2026-02-01 04:00:00')).toEqual(ONE_CLICK_EVERY_DIMENSION) + }) +}) From ad429ada9583de3e9e941d3be407cda544b1ec46 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 12:53:19 +0000 Subject: [PATCH 15/65] clicks: pin the partition the merge tests rely on --- packages/db/src/schema-rollups.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/db/src/schema-rollups.test.ts b/packages/db/src/schema-rollups.test.ts index ddc14c0..152b3eb 100644 --- a/packages/db/src/schema-rollups.test.ts +++ b/packages/db/src/schema-rollups.test.ts @@ -386,10 +386,32 @@ describe('clickhouse schema 008: the key columns that keep rows apart', () => { * These clicks are in February rather than the September the rest of the file * accumulates in, because parts in different partitions never merge together: * the only rows the forced merge can collapse are the ones these tests wrote. + * **That isolation is the monthly partition key**, so these tests depend on it + * as much as on the sort key they are named for — which is why the first test + * here pins the partition expression, rather than leaving a partition change to + * surface as a sort-key test failing for a reason its name does not describe. */ describe('clickhouse schema 008: the sort keys, across a merge', () => { const FEB = 'toYYYYMM(hour) = 202602' + /** + * The expression, read from the engine, and not a row count that would follow + * from it: a count can come out right under a different partitioning, and + * whatever a rollup is partitioned by has to be derived from `hour` alone for + * a report's window to read whole months rather than every part in the table. + */ + it('partitions both rollup tables by the month of the hour', async () => { + const keys = await rows<{ name: string; partition_key: string }>( + `SELECT name, partition_key FROM system.tables + WHERE database = currentDatabase() AND name IN ('clicks_hourly', 'clicks_hourly_dim') + ORDER BY name`, + ) + expect(keys).toEqual([ + { name: 'clicks_hourly', partition_key: 'toYYYYMM(hour)' }, + { name: 'clicks_hourly_dim', partition_key: 'toYYYYMM(hour)' }, + ]) + }) + /** A click that reached no target, from an address no database knew, with no referrer. */ const nothingToShow = (clickId: string, time: string) => click({ From cd3ed6dcef473e66d7adf70162f10dde9fccfdc6 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 12:54:39 +0000 Subject: [PATCH 16/65] clicks: the merge tests do not depend on the partition, say so --- packages/db/src/schema-rollups.test.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/db/src/schema-rollups.test.ts b/packages/db/src/schema-rollups.test.ts index 152b3eb..29c724c 100644 --- a/packages/db/src/schema-rollups.test.ts +++ b/packages/db/src/schema-rollups.test.ts @@ -384,21 +384,23 @@ describe('clickhouse schema 008: the key columns that keep rows apart', () => { * is the same before and after it. * * These clicks are in February rather than the September the rest of the file - * accumulates in, because parts in different partitions never merge together: - * the only rows the forced merge can collapse are the ones these tests wrote. - * **That isolation is the monthly partition key**, so these tests depend on it - * as much as on the sort key they are named for — which is why the first test - * here pins the partition expression, rather than leaving a partition change to - * surface as a sort-key test failing for a reason its name does not describe. + * accumulates in, so the forced merge cannot reach a row another test wrote. + * Two independent things keep them apart and either alone would do: a different + * monthly partition, because parts in different partitions never merge; and + * `hour` at the head of both sort keys. The partition is the one that reasoning + * names, so the first test below pins the expression on its own. A change to it + * then fails a test whose name is about partitioning, and the merge tests go on + * saying only what their own names say. */ describe('clickhouse schema 008: the sort keys, across a merge', () => { const FEB = 'toYYYYMM(hour) = 202602' /** - * The expression, read from the engine, and not a row count that would follow - * from it: a count can come out right under a different partitioning, and - * whatever a rollup is partitioned by has to be derived from `hour` alone for - * a report's window to read whole months rather than every part in the table. + * The expression, read from the engine, rather than a row count that would + * follow from it: a count comes out right under any partitioning at all. What + * has to hold is that the expression is derived from `hour` and nothing else, + * because that is what lets a report bounding a window skip the partitions + * outside it instead of reading the table. */ it('partitions both rollup tables by the month of the hour', async () => { const keys = await rows<{ name: string; partition_key: string }>( From 56756fc14a0723c2d72af624144fd84b8c08a1d4 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 13:24:47 +0000 Subject: [PATCH 17/65] clicks: a dimension nobody can confuse with its default --- packages/db/src/schema-rollups.test.ts | 253 +++++++++++++++++-------- 1 file changed, 172 insertions(+), 81 deletions(-) diff --git a/packages/db/src/schema-rollups.test.ts b/packages/db/src/schema-rollups.test.ts index 29c724c..e621062 100644 --- a/packages/db/src/schema-rollups.test.ts +++ b/packages/db/src/schema-rollups.test.ts @@ -383,24 +383,31 @@ describe('clickhouse schema 008: the key columns that keep rows apart', () => { * against the committed schema it changes no number — so every expectation here * is the same before and after it. * - * These clicks are in February rather than the September the rest of the file - * accumulates in, so the forced merge cannot reach a row another test wrote. - * Two independent things keep them apart and either alone would do: a different - * monthly partition, because parts in different partitions never merge; and - * `hour` at the head of both sort keys. The partition is the one that reasoning - * names, so the first test below pins the expression on its own. A change to it - * then fails a test whose name is about partitioning, and the merge tests go on - * saying only what their own names say. + * `OPTIMIZE … FINAL` merges every partition of the table, and within a partition + * it merges every row — not only the rows the calling test wrote. It never + * merges across partitions, so a month of its own is what keeps one test's rows + * from being folded into another's, and reading only the hours a test inserted + * is what keeps another test's rows out of its answer. February pins the hourly + * key, March what the views write, April the `dimension` column and May the + * `hour` column of the per-dimension key. + * + * Sharing a month would make a test pass or fail on what a neighbour happened + * to write. The first version of this block did share one, and the accident was + * not harmless: it was the only thing pinning `hour` on the per-dimension key, + * under a test whose name says nothing about it. */ describe('clickhouse schema 008: the sort keys, across a merge', () => { - const FEB = 'toYYYYMM(hour) = 202602' + const HOURLY_KEYS = 'hour, link_id, domain_id, traffic_class, action, outcome' + const DIM_KEYS = 'hour, link_id, domain_id, dimension, value' + + const at = (hour: string) => `toDateTime('${hour}', 'UTC')` /** - * The expression, read from the engine, rather than a row count that would - * follow from it: a count comes out right under any partitioning at all. What - * has to hold is that the expression is derived from `hour` and nothing else, - * because that is what lets a report bounding a window skip the partitions - * outside it instead of reading the table. + * The partition expression, read from the engine rather than inferred from a + * row count that comes out right under any partitioning at all. What has to + * hold is that it is derived from `hour` and nothing else, because that is + * what lets a report bounding a window skip the partitions outside it instead + * of reading the table — and it is what gives each test below its own month. */ it('partitions both rollup tables by the month of the hour', async () => { const keys = await rows<{ name: string; partition_key: string }>( @@ -414,53 +421,33 @@ describe('clickhouse schema 008: the sort keys, across a merge', () => { ]) }) - /** A click that reached no target, from an address no database knew, with no referrer. */ - const nothingToShow = (clickId: string, time: string) => - click({ - click_id: clickId, - time, - outcome: 'blocked', - step: 'traffic', - status: 403, - destination: '', - target_id: '', - country: '', - referrer: '', - traffic_class: 'bot', - action: 'block', - signals: ['ua_bot'], - }) - - const dimsAt = (hour: string) => - rows<{ dimension: string; value: string; clicks: string }>( - `SELECT dimension, value, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim - WHERE hour = toDateTime('${hour}', 'UTC') GROUP BY dimension, value ORDER BY dimension`, - ) - /** - * What that one click must read as in every dimension. The value is asserted - * and not only the dimension name: a view whose SELECT alias does not match a - * target column is created without complaint and writes that column's - * default, so a dimension can be present and empty for a reason that has - * nothing to do with the click. + * The key as a list, in order, beside the tests that show what each column in + * it does. Order is not a detail a merge can reveal — any permutation of the + * same columns keeps exactly the same rows apart — but it decides which + * prefixes ClickHouse can skip data on, and every report bounds a window + * before it filters anything else. */ - const ONE_CLICK_EVERY_DIMENSION = [ - { dimension: 'browser', value: 'chrome', clicks: '1' }, - { dimension: 'country', value: '', clicks: '1' }, - { dimension: 'device', value: 'desktop', clicks: '1' }, - { dimension: 'os', value: 'windows', clicks: '1' }, - { dimension: 'referrer', value: '', clicks: '1' }, - { dimension: 'target', value: '', clicks: '1' }, - ] + it('orders both rollup tables by the window first and the link second', async () => { + const keys = await rows<{ name: string; sorting_key: string }>( + `SELECT name, sorting_key FROM system.tables + WHERE database = currentDatabase() AND name IN ('clicks_hourly', 'clicks_hourly_dim') + ORDER BY name`, + ) + expect(keys).toEqual([ + { name: 'clicks_hourly', sorting_key: HOURLY_KEYS }, + { name: 'clicks_hourly_dim', sorting_key: DIM_KEYS }, + ]) + }) /** - * The hour, the link and the class: the three columns a report filters or - * groups on before anything else. Each of these four clicks differs from the - * first in exactly one of them, and each is inserted on its own so that no - * two of them are ever in one part until the merge. The total is asserted - * beside the breakdowns because that is the shape of the bug: a key column - * missing from the table leaves the total right and destroys every breakdown - * under it, so a report would answer 4 and then account for it wrongly. + * The hour, the link and the class: three of the columns a report filters or + * groups on. Each of these four clicks differs from the first in exactly one + * of them, and each is inserted on its own so that no two of them are ever in + * one part until the merge. The total is asserted beside the breakdowns to say + * what the failure looks like: a column missing from the key leaves the total + * right and the breakdown *by that column* wrong, so a report answers 4 and + * then accounts for the 4 under one hour, or one link, or one class. */ it('keeps the hour, the link and the class apart when a merge collapses the parts', async () => { const base = { time: '2026-02-01 01:05:00.000', action: 'nothing' } @@ -490,13 +477,17 @@ describe('clickhouse schema 008: the sort keys, across a merge', () => { ]) await ch.command({ query: 'OPTIMIZE TABLE clicks_hourly FINAL' }) + // The two hours this test inserted, and nothing else: a month-wide filter + // would read whatever a later test adds to February and pass only while + // this test ran first. + const mine = `hour IN (${at('2026-02-01 01:00:00')}, ${at('2026-02-01 02:00:00')})` // `AS at` rather than `AS hour`: an alias that repeats the column's name - // shadows the column for the WHERE beside it, and this query would fail - // with "Illegal type String of argument of function toYYYYMM" — the same - // trap the state columns carry their `_state` suffix to avoid. + // shadows the column for the WHERE beside it, and a `toYYYYMM(hour)` there + // would fail with "Illegal type String of argument" — the same trap the + // state columns carry their `_state` suffix to avoid. const byHour = await rows<{ at: string; clicks: string }>( `SELECT toString(hour) AS at, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly - WHERE ${FEB} GROUP BY hour ORDER BY hour`, + WHERE ${mine} GROUP BY hour ORDER BY hour`, ) expect(byHour).toEqual([ { at: '2026-02-01 01:00:00', clicks: '3' }, @@ -504,7 +495,7 @@ describe('clickhouse schema 008: the sort keys, across a merge', () => { ]) const byLink = await rows<{ link_id: string; clicks: string }>( `SELECT link_id, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly - WHERE ${FEB} GROUP BY link_id ORDER BY link_id`, + WHERE ${mine} GROUP BY link_id ORDER BY link_id`, ) expect(byLink).toEqual([ { link_id: LINK_A, clicks: '3' }, @@ -512,39 +503,139 @@ describe('clickhouse schema 008: the sort keys, across a merge', () => { ]) const byClass = await rows<{ traffic_class: string; clicks: string }>( `SELECT traffic_class, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly - WHERE ${FEB} GROUP BY traffic_class ORDER BY traffic_class`, + WHERE ${mine} GROUP BY traffic_class ORDER BY traffic_class`, ) expect(byClass).toEqual([ { traffic_class: 'bot', clicks: '1' }, { traffic_class: 'human', clicks: '3' }, ]) - const feb = await rows<{ clicks: string }>( - `SELECT uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly WHERE ${FEB}`, + const total = await rows<{ clicks: string }>( + `SELECT uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly WHERE ${mine}`, + ) + expect(total).toEqual([{ clicks: '4' }]) + }) + + const TARGET = '00000000-0000-4000-8000-0000000000b1' + + /** + * A click shipped from a version 1 spool record — no class, no signals, no + * action, an empty OS and browser, ASN 0 — whose address was in no geo + * database, that arrived with no referrer, and that reached no target because + * the link had expired and its fallback URL was served instead. Every open + * dimension it carries is therefore empty but the device, which is a closed + * list in every record version. + */ + const nothingKnown = (clickId: string, time: string) => + click({ + click_id: clickId, + time, + outcome: 'expired', + step: 'limits', + destination: 'https://example.com/backup', + target_id: '', + country: '', + geo_source: '', + referrer: '', + os: '', + browser: '', + traffic_class: '', + action: '', + signals: [], + asn: 0, + }) + + const dimsAt = (hour: string) => + rows<{ dimension: string; value: string; clicks: string }>( + `SELECT dimension, value, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim + WHERE hour = ${at(hour)} GROUP BY dimension, value ORDER BY dimension, value`, ) - expect(feb).toEqual([{ clicks: '4' }]) + + /** + * Every dimension carries two distinct values here, and for five of the six + * one of the two is the empty string. That pairing is what makes a view's + * output checkable at all: with one value per dimension, a view that wrote a + * constant — or whose SELECT aliased its value to a name the target table + * does not have, which ClickHouse creates without complaint and then fills + * with the column's default — produces exactly what a correct view produces. + * Two values, one of them not the default, is the least that tells them + * apart, and it is why the referrer was the only dimension the first version + * of this file pinned. + * + * The device pair is 'desktop' and 'ios' rather than a value and an empty + * one, because the device is a closed list that no record version leaves + * blank. + */ + it('writes each dimension its own values, not a constant and not a default', async () => { + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000006a', time: '2026-03-01 05:05:00.000' }), + click({ + click_id: '01920000-0000-7000-8000-00000000006b', + time: '2026-03-01 05:05:00.000', + device: 'ios', + }), + nothingKnown('01920000-0000-7000-8000-00000000006c', '2026-03-01 05:05:00.000'), + ]) + expect(await dimsAt('2026-03-01 05:00:00')).toEqual([ + { dimension: 'browser', value: '', clicks: '1' }, + { dimension: 'browser', value: 'chrome', clicks: '2' }, + { dimension: 'country', value: '', clicks: '1' }, + { dimension: 'country', value: 'DE', clicks: '2' }, + { dimension: 'device', value: 'desktop', clicks: '2' }, + { dimension: 'device', value: 'ios', clicks: '1' }, + { dimension: 'os', value: '', clicks: '1' }, + { dimension: 'os', value: 'windows', clicks: '2' }, + { dimension: 'referrer', value: '', clicks: '1' }, + { dimension: 'referrer', value: 'blog.example.com', clicks: '2' }, + { dimension: 'target', value: '', clicks: '1' }, + { dimension: 'target', value: TARGET, clicks: '2' }, + ]) }) /** - * An empty value is a value, for each of the three dimensions that can hold - * one. A view that filtered them out would leave a breakdown that no longer - * adds up to the total beside it, and the click below is the one that has all - * three at once. + * `dimension` is a key column, and the click above is what shows why: five of + * its six rows hold an empty value in the same hour for the same link and + * domain, so they differ in nothing but `dimension`. Without it in the key a + * merge folds those five into one and four of the six dimensions are gone — + * not a blurred breakdown but four missing reports. */ - it('keeps an empty country, referrer and target as values of their own', async () => { - await insert([nothingToShow('01920000-0000-7000-8000-00000000006a', '2026-02-01 03:05:00.000')]) - expect(await dimsAt('2026-02-01 03:00:00')).toEqual(ONE_CLICK_EVERY_DIMENSION) + it('keeps the dimensions apart when five of them share an empty value', async () => { + await insert([nothingKnown('01920000-0000-7000-8000-00000000007a', '2026-04-01 06:05:00.000')]) + await ch.command({ query: 'OPTIMIZE TABLE clicks_hourly_dim FINAL' }) + expect(await dimsAt('2026-04-01 06:00:00')).toEqual([ + { dimension: 'browser', value: '', clicks: '1' }, + { dimension: 'country', value: '', clicks: '1' }, + { dimension: 'device', value: 'desktop', clicks: '1' }, + { dimension: 'os', value: '', clicks: '1' }, + { dimension: 'referrer', value: '', clicks: '1' }, + { dimension: 'target', value: '', clicks: '1' }, + ]) }) /** - * And `dimension` is a key column, which the click above is also what proves: - * its country, referrer and target rows share an hour, a link, a domain and - * an empty value, so they differ in nothing else. Without `dimension` in the - * key a merge folds the three into one and two of the six dimensions are - * gone — not a blurred breakdown but a missing report. + * And `hour` is a key column of the per-dimension table too, which no test + * above says: the two clicks here are identical but for their hour, each in + * its own insert call so that only the merge can bring them together. Without + * `hour` in that key the two collapse into one row and a chart over the + * window reports the whole day at whichever hour the merge kept. */ - it('keeps the six dimensions apart when three of them share an empty value', async () => { - await insert([nothingToShow('01920000-0000-7000-8000-00000000007a', '2026-02-01 04:05:00.000')]) + it('keeps two hours apart in the per-dimension rollup across a merge', async () => { + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000008a', time: '2026-05-01 07:05:00.000' }), + ]) + await insert([ + click({ click_id: '01920000-0000-7000-8000-00000000008b', time: '2026-05-01 08:05:00.000' }), + ]) await ch.command({ query: 'OPTIMIZE TABLE clicks_hourly_dim FINAL' }) - expect(await dimsAt('2026-02-01 04:00:00')).toEqual(ONE_CLICK_EVERY_DIMENSION) + // The month, not the two hours: this test is about which hours exist, so a + // filter naming them could not see a row that landed on a third. May holds + // nothing but this test's two clicks, so the month is still its own fixture. + const byHour = await rows<{ at: string; clicks: string }>( + `SELECT toString(hour) AS at, uniqExactMerge(clicks_state) AS clicks FROM clicks_hourly_dim + WHERE dimension = 'country' AND toYYYYMM(hour) = 202605 GROUP BY hour ORDER BY hour`, + ) + expect(byHour).toEqual([ + { at: '2026-05-01 07:00:00', clicks: '1' }, + { at: '2026-05-01 08:00:00', clicks: '1' }, + ]) }) }) From a736aa11b0fac71a525be02a2deb1834cf332d50 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 13:38:35 +0000 Subject: [PATCH 18/65] admin: what a window of clicks adds up to --- docker-compose.yml | 12 +- packages/admin/package.json | 1 + packages/admin/src/app.ts | 31 ++- packages/admin/src/config.test.ts | 19 +- packages/admin/src/config.ts | 16 ++ packages/admin/src/index.ts | 25 ++- packages/admin/src/reports.test.ts | 299 +++++++++++++++++++++++++++++ packages/admin/src/reports.ts | 279 +++++++++++++++++++++++++++ packages/admin/src/testing.ts | 5 + packages/db/src/clients.ts | 3 + pnpm-lock.yaml | 3 + 11 files changed, 687 insertions(+), 6 deletions(-) create mode 100644 packages/admin/src/reports.test.ts create mode 100644 packages/admin/src/reports.ts diff --git a/docker-compose.yml b/docker-compose.yml index 954bacb..6bf79b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,12 +42,22 @@ services: # network, and only for the host name CLICKMONK_ADMIN_HOST names. Unset, it # starts and answers 503 on every route rather than refusing to boot — an # install that never sets it must still serve its links. + # + # Started after ClickHouse but not gated on its health, the way the redirect + # depends on Postgres: the reports read it, and a report answers 503 while it + # is down rather than keeping the rest of this API from starting. admin: <<: *app command: ["node", "packages/admin/dist/index.js"] - depends_on: [postgres] + depends_on: [postgres, clickhouse] environment: CLICKMONK_POSTGRES_URL: postgres://clickmonk:${POSTGRES_PASSWORD:?set it in .env}@postgres:5432/clickmonk + # Reports read ClickHouse directly. The same four values the worker has; + # this service only ever reads. + CLICKMONK_CLICKHOUSE_URL: http://clickhouse:8123 + CLICKMONK_CLICKHOUSE_USER: clickmonk + CLICKMONK_CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?set it in .env} + CLICKMONK_CLICKHOUSE_DB: clickmonk CLICKMONK_ADMIN_HOST: ${CLICKMONK_ADMIN_HOST:-} CLICKMONK_TRUSTED_PROXIES: ${CLICKMONK_TRUSTED_PROXIES:-uniquelocal,loopback} CLICKMONK_DNS_SERVERS: ${CLICKMONK_DNS_SERVERS:-} diff --git a/packages/admin/package.json b/packages/admin/package.json index 4ee725c..35ec71e 100644 --- a/packages/admin/package.json +++ b/packages/admin/package.json @@ -12,6 +12,7 @@ "engines": { "node": ">=22 <23" }, "scripts": { "build": "tsc -b", "build:image": "tsc -b tsconfig.build.json" }, "dependencies": { + "@clickhouse/client": "^1.23.1", "@clickmonk/core": "workspace:*", "@clickmonk/db": "workspace:*", "@clickmonk/ipdata": "workspace:*", diff --git a/packages/admin/src/app.ts b/packages/admin/src/app.ts index 61574b8..9e6a461 100644 --- a/packages/admin/src/app.ts +++ b/packages/admin/src/app.ts @@ -1,5 +1,5 @@ import { AttemptCounter, ConcurrencyGate, newSlug, normaliseHost } from '@clickmonk/core' -import type { Pool } from '@clickmonk/db' +import type { ClickHouseClient, Pool } from '@clickmonk/db' import { MAX_IP_LENGTH, addressOnly, canonicalIp } from '@clickmonk/ipdata' import type { DomainResolver } from '@clickmonk/worker/domains' import Fastify, { @@ -12,6 +12,7 @@ import { registerDomainRoutes } from './domains.js' import { HttpError, MAX_BODY_BYTES, securityHeaders } from './http.js' import { registerKeyRoutes } from './keys.js' import { registerLinkRoutes } from './links.js' +import { registerReportRoutes } from './reports.js' import { registerSessionRoutes } from './session-routes.js' import { registerSettingsRoutes } from './settings-routes.js' @@ -26,6 +27,19 @@ export const PASSWORD_CHECKS_IN_FLIGHT = 2 * bound must not have to reach back into this file for the number. */ export const DNS_CHECKS_IN_FLIGHT = 2 +/** + * Report queries this process runs at once, and exports. + * + * A credential is not a bound: an authenticated caller — or a leaked API key — + * can loop a report endpoint, and each pass is a scan of a window they chose. + * Past these the answer is a refusal with `retry-after`, never a queued query, + * for the same reason the on-demand DNS check refuses rather than queues. + * Exports get their own, smaller gate because an export is the one request + * that holds a ClickHouse result open for as long as the client takes to read + * it. + */ +export const REPORT_QUERIES_IN_FLIGHT = 2 +export const EXPORTS_IN_FLIGHT = 1 export interface AdminDeps { pg: Pool @@ -62,6 +76,16 @@ export interface AdminDeps { loginAttempts?: AttemptCounter passwordGate?: ConcurrencyGate checkGate?: ConcurrencyGate + /** + * Where the reports read from. Optional here and required in `loadConfig`: + * required in configuration so an install with a typo fails at boot with the + * variable named rather than answering 503 for ever, and optional here so + * that the 503 path is reachable from a test, and so that a suite about + * links does not have to build a ClickHouse client to test a link. + */ + ch?: ClickHouseClient + reportGate?: ConcurrencyGate + exportGate?: ConcurrencyGate /** * Where a link with no slug of its own gets one. `newSlug` unless a caller * says otherwise, and nothing in the product says otherwise: it is here so @@ -84,6 +108,8 @@ export interface AdminContext extends AdminDeps { passwordGate: ConcurrencyGate /** How many on-demand DNS checks this process runs at once. */ checkGate: ConcurrencyGate + reportGate: ConcurrencyGate + exportGate: ConcurrencyGate slugSource: () => string } @@ -129,6 +155,8 @@ export function buildAdminApp( deps.loginAttempts ?? new AttemptCounter(LOGIN_ATTEMPT_LIMIT, LOGIN_ATTEMPT_WINDOW_MS), passwordGate: deps.passwordGate ?? new ConcurrencyGate(PASSWORD_CHECKS_IN_FLIGHT), checkGate: deps.checkGate ?? new ConcurrencyGate(DNS_CHECKS_IN_FLIGHT), + reportGate: deps.reportGate ?? new ConcurrencyGate(REPORT_QUERIES_IN_FLIGHT), + exportGate: deps.exportGate ?? new ConcurrencyGate(EXPORTS_IN_FLIGHT), slugSource: deps.slugSource ?? newSlug, } @@ -243,6 +271,7 @@ export function buildAdminApp( registerDomainRoutes(app, ctx) registerLinkRoutes(app, ctx) registerSettingsRoutes(app, ctx) + registerReportRoutes(app, ctx) return app } diff --git a/packages/admin/src/config.test.ts b/packages/admin/src/config.test.ts index de97b36..007b86e 100644 --- a/packages/admin/src/config.test.ts +++ b/packages/admin/src/config.test.ts @@ -1,7 +1,13 @@ import { describe, expect, it } from 'vitest' import { loadConfig } from './config.js' -const base = { CLICKMONK_POSTGRES_URL: 'postgres://u:p@db:5432/clickmonk' } +const base = { + CLICKMONK_POSTGRES_URL: 'postgres://u:p@db:5432/clickmonk', + CLICKMONK_CLICKHOUSE_URL: 'http://clickhouse:8123', + CLICKMONK_CLICKHOUSE_USER: 'clickmonk', + CLICKMONK_CLICKHOUSE_PASSWORD: 'a clickhouse password', + CLICKMONK_CLICKHOUSE_DB: 'clickmonk', +} describe('loadConfig', () => { // The whole object, not a subset: a field this service starts with and @@ -9,6 +15,12 @@ describe('loadConfig', () => { it('applies the documented defaults', () => { expect(loadConfig(base)).toEqual({ postgresUrl: 'postgres://u:p@db:5432/clickmonk', + ch: { + url: 'http://clickhouse:8123', + username: 'clickmonk', + password: 'a clickhouse password', + database: 'clickmonk', + }, adminHost: null, port: 9100, trustedProxies: ['127.0.0.1'], @@ -16,6 +28,11 @@ describe('loadConfig', () => { }) }) + it('refuses to start without the store the reports read', () => { + const { CLICKMONK_CLICKHOUSE_URL: _omitted, ...rest } = base + expect(() => loadConfig(rest)).toThrow(/CLICKMONK_CLICKHOUSE_URL/) + }) + it('names every missing or bad variable at once', () => { expect(() => loadConfig({ CLICKMONK_ADMIN_PORT: 'http' })).toThrow( /CLICKMONK_POSTGRES_URL[\s\S]*CLICKMONK_ADMIN_PORT/, diff --git a/packages/admin/src/config.ts b/packages/admin/src/config.ts index b846d86..168541c 100644 --- a/packages/admin/src/config.ts +++ b/packages/admin/src/config.ts @@ -1,4 +1,5 @@ import { AdminHostSchema, TrustedProxiesSchema, formatConfigError } from '@clickmonk/core' +import type { ChConfig } from '@clickmonk/db' import { isResolverAddress } from '@clickmonk/worker/domains' import { z } from 'zod' @@ -24,6 +25,14 @@ const DnsServers = z const Schema = z.object({ CLICKMONK_POSTGRES_URL: z.string().url(), + // Reports read ClickHouse. Required rather than optional: an install with a + // typo here would otherwise answer 503 on every report for the rest of its + // life, and the first person to notice would be whoever opened the + // dashboard. Failing at boot names the variable. + CLICKMONK_CLICKHOUSE_URL: z.string().url(), + CLICKMONK_CLICKHOUSE_USER: z.string().min(1), + CLICKMONK_CLICKHOUSE_PASSWORD: z.string(), + CLICKMONK_CLICKHOUSE_DB: z.string().min(1), // The shared schema, so this service and the redirect cannot come to read // the same variable differently. Empty means "not configured": every route // answers 503 until an operator names the host, rather than this service @@ -41,6 +50,7 @@ const Schema = z.object({ export interface AdminConfig { postgresUrl: string + ch: ChConfig /** Null when unset: every route answers 503 until an operator names the host. */ adminHost: string | null port: number @@ -54,6 +64,12 @@ export function loadConfig(env: NodeJS.ProcessEnv): AdminConfig { const e = r.data return { postgresUrl: e.CLICKMONK_POSTGRES_URL, + ch: { + url: e.CLICKMONK_CLICKHOUSE_URL, + username: e.CLICKMONK_CLICKHOUSE_USER, + password: e.CLICKMONK_CLICKHOUSE_PASSWORD, + database: e.CLICKMONK_CLICKHOUSE_DB, + }, adminHost: e.CLICKMONK_ADMIN_HOST === '' ? null : e.CLICKMONK_ADMIN_HOST, port: e.CLICKMONK_ADMIN_PORT, trustedProxies: e.CLICKMONK_TRUSTED_PROXIES, diff --git a/packages/admin/src/index.ts b/packages/admin/src/index.ts index 677953e..a2ed44b 100644 --- a/packages/admin/src/index.ts +++ b/packages/admin/src/index.ts @@ -1,4 +1,4 @@ -import { createPgPool } from '@clickmonk/db' +import { createChClient, createPgPool } from '@clickmonk/db' import { buildAdminApp } from './app.js' import { loadConfig } from './config.js' @@ -19,9 +19,19 @@ const pg = createPgPool(config.postgresUrl, { connectTimeoutMs: 5_000, }) +// One client, built here rather than per request: it is an HTTP client with a +// connection pool of its own, and building one per request would open a +// connection per report. A bounded request timeout for the same reason the +// pool has one — a query that never answers must not hold a request open for +// ever — and it is longer than the server-side bound each query carries, so +// that a query ClickHouse itself refuses comes back as ClickHouse's error and +// not as a client-side abort with nothing in it. +const ch = createChClient({ ...config.ch, requestTimeoutMs: 30_000 }) + const app = buildAdminApp( { pg, + ch, adminHost: config.adminHost, dnsServers: config.dnsServers, }, @@ -42,8 +52,17 @@ async function shutdown(signal: string): Promise { stopping = true app.log.info({ signal }, 'draining') try { - await app.close() - await pg.end() + // All three, whatever any one of them does. Closed in sequence, a server + // that fails to drain leaves the pool and the ClickHouse client open, and + // the process exits holding its sockets. `allSettled` never rejects, so a + // failure is reported from the results rather than from the catch below — + // which still stands, because a throw before the first await is possible. + for (const closed of await Promise.allSettled([app.close(), pg.end(), ch.close()])) { + if (closed.status === 'rejected') { + console.error('shutdown error', closed.reason) + process.exitCode = 1 + } + } } catch (err) { console.error('shutdown error', err) process.exitCode = 1 diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts new file mode 100644 index 0000000..c0ebc92 --- /dev/null +++ b/packages/admin/src/reports.test.ts @@ -0,0 +1,299 @@ +import { ConcurrencyGate } from '@clickmonk/core' +import { type ClickHouseClient, createChClient } from '@clickmonk/db' +import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' +import type { FastifyInstance } from 'fastify' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { HttpError } from './http.js' +import { HOUR_MS, parseWindow } from './reports.js' +import { ADMIN_HOST, clockFrom, read, signedIn, testApp } from './testing.js' + +const pool = testPg() +const ch = testCh() +const clock = clockFrom(new Date('2026-09-24T12:00:00.000Z')) +let app: FastifyInstance +let cookie = '' + +const LINK_A = '00000000-0000-4000-8000-0000000000a1' +const LINK_B = '00000000-0000-4000-8000-0000000000a2' +const DOMAIN = '00000000-0000-4000-8000-00000000000d' + +/** One click as the shipper writes it, at a fixed instant inside the window below. */ +const click = (over: Record = {}): Record => ({ + click_id: '01920000-0000-7000-8000-00000000000a', + time: '2026-09-24 10:15:00.000', + host: 'go.example.test', + path: '/a', + domain_id: DOMAIN, + link_id: LINK_A, + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/', + target_id: '00000000-0000-4000-8000-0000000000b1', + visitor_id: 'v1', + returning: 0, + country: 'DE', + region: '', + city: '', + geo_source: 'dbip', + device: 'desktop', + user_agent: 'ua', + referrer: 'https://blog.example.com/post', + ip: '198.51.100.77', + cap_unchecked: 0, + traffic_class: 'human', + signals: [], + action: '', + os: 'windows', + browser: 'chrome', + asn: 64500, + ...over, +}) + +const insert = (values: Record[]) => + ch.insert({ table: 'clicks', values, format: 'JSONEachRow' }) + +/** The window every test below asks for, unless it is about the window itself. */ +const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' + +const summary = (on: FastifyInstance, query = WINDOW) => + on.inject({ method: 'GET', url: `/api/reports/summary?${query}`, headers: read(cookie) }) + +beforeAll(async () => { + await resetDatabases(pool, ch) + await insert([ + click(), + click({ click_id: '01920000-0000-7000-8000-00000000000b', visitor_id: 'v2', country: 'US' }), + // The same visitor as the first, on another link: one visitor, two clicks. + click({ click_id: '01920000-0000-7000-8000-00000000000c', link_id: LINK_B, path: '/b' }), + // A bot, blocked, in the same hour, by the same visitor as the first. It + // reached no target, so its target id is empty — which is also the only + // empty dimension value in these fixtures. + click({ + click_id: '01920000-0000-7000-8000-00000000000d', + traffic_class: 'bot', + action: 'block', + outcome: 'blocked', + step: 'classify', + status: 403, + signals: ['ua_bot'], + destination: '', + target_id: '', + }), + // An hour later, and outside the narrow window one test asks for. + click({ + click_id: '01920000-0000-7000-8000-00000000000e', + time: '2026-09-24 11:30:00.000', + visitor_id: 'v3', + }), + ]) +}) + +beforeEach(async () => { + app = testApp(pool, clock, { ch }) + cookie = await signedIn(app, pool, clock.now()) +}) + +afterEach(async () => { + await app.close() + await pool.query('TRUNCATE admin_account, admin_recovery_codes, sessions, api_keys') +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +describe('GET /api/reports/summary', () => { + it('counts the window, by click and by visitor', async () => { + const r = await summary(app) + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z' }, + link: null, + clicks: 5, + visitors: 3, + byClass: { human: 4, bot: 1 }, + byAction: { '': 4, block: 1 }, + byOutcome: { target: 4, blocked: 1 }, + newestHour: '2026-09-24T11:00:00.000Z', + }) + }) + + // Three clicks by two visitors, one of whom also clicked the other link: + // the per-link visitor count is not a share of the install-wide one. + it('counts one link when asked for one', async () => { + const r = await summary(app, `${WINDOW}&link=${LINK_B}`) + expect(r.json().clicks).toBe(1) + expect(r.json().visitors).toBe(1) + expect(r.json().link).toBe(LINK_B) + }) + + it('counts whole hours and says which hours it counted', async () => { + const r = await summary(app, 'from=2026-09-24T10:30:00.000Z&to=2026-09-24T11:10:00.000Z') + // Asked for 10:30 to 11:10; counted 10:00 to 12:00, which is both hours. + expect(r.json().window).toEqual({ + from: '2026-09-24T10:00:00.000Z', + to: '2026-09-24T12:00:00.000Z', + }) + expect(r.json().clicks).toBe(5) + }) + + it('answers an empty window with zeroes rather than nothing', async () => { + const r = await summary(app, 'from=2020-01-01T00:00:00.000Z&to=2020-01-02T00:00:00.000Z') + expect(r.statusCode).toBe(200) + expect(r.json().clicks).toBe(0) + expect(r.json().visitors).toBe(0) + expect(r.json().byClass).toEqual({}) + }) + + it.each([ + ['no window at all', ''], + ['only a start', 'from=2026-09-24T00:00:00.000Z'], + ['a start that is not a time', 'from=yesterday&to=2026-09-25T00:00:00.000Z'], + ['a time with no zone', 'from=2026-09-24T00:00:00&to=2026-09-25T00:00:00.000Z'], + ['a backwards window', 'from=2026-09-25T00:00:00.000Z&to=2026-09-24T00:00:00.000Z'], + ['an empty window', 'from=2026-09-24T00:00:00.000Z&to=2026-09-24T00:00:00.000Z'], + ['a link that is not an id', `${WINDOW}&link=nope`], + ['a field nobody knows', `${WINDOW}&account=1`], + ])('refuses %s', async (_label, query) => { + const r = await summary(app, query) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_query') + }) + + it('refuses a window longer than four hundred days, and says how long it may be', async () => { + const r = await summary(app, 'from=2025-01-01T00:00:00.000Z&to=2026-09-25T00:00:00.000Z') + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('window_too_long') + expect(r.json().message).toContain('400 days') + }) + + it('needs a credential', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/reports/summary?${WINDOW}`, + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + }) + + it('answers an API key, because a report is a read a script may make', async () => { + // A key is minted through the API by the signed-in session, then used. + const made = await app.inject({ + method: 'POST', + url: '/api/keys', + headers: { host: ADMIN_HOST, origin: `https://${ADMIN_HOST}`, cookie }, + payload: { name: 'reporting' }, + }) + const key = made.json().key as string + const r = await app.inject({ + method: 'GET', + url: `/api/reports/summary?${WINDOW}`, + headers: { host: ADMIN_HOST, authorization: `Bearer ${key}` }, + }) + expect(r.statusCode).toBe(200) + expect(r.json().clicks).toBe(5) + }) + + it('refuses when every report slot is taken, and says when to come back', async () => { + const full = new ConcurrencyGate(0) + const other = testApp(pool, clock, { ch, reportGate: full }) + try { + const r = await summary(other) + expect(r.statusCode).toBe(429) + expect(r.json().error).toBe('too_many_reports') + expect(r.headers['retry-after']).toBe('1') + } finally { + await other.close() + } + }) + + // One slot, two reports in sequence: the second is answered only because the + // first gave its slot back. With the shipped two slots a leak would take + // three requests to show, so the gate is shrunk to the one that proves it. + it('gives a slot back, so the next report is answered', async () => { + const one = new ConcurrencyGate(1) + const other = testApp(pool, clock, { ch, reportGate: one }) + try { + for (const attempt of ['first', 'second']) { + const r = await summary(other) + expect(r.statusCode, attempt).toBe(200) + } + } finally { + await other.close() + } + }) +}) + +// The route cannot reach this: `from` is a datetime in the schema, so an +// unreadable date is a 400 before the parser sees it. The parser is exported +// and the chart and the log will call it too, so the bound lives in the +// function and is checked there — the same rule as every other bound a later +// caller could reach directly. +describe('parseWindow', () => { + it('refuses a date it cannot read, rather than returning a window of NaN', () => { + // The guard for this is `!(toMs > fromMs)`. Written the natural way round, + // as `toMs <= fromMs`, this call returns a window of NaN: every later check + // compares with `>` and every comparison with NaN is false, so the length + // bound and the bucket ceiling both pass it through. + let thrown: unknown + try { + parseWindow({ from: 'yesterday', to: '2026-09-25T00:00:00.000Z' }, { alignMs: HOUR_MS }) + } catch (err) { + thrown = err + } + expect(thrown).toBeInstanceOf(HttpError) + expect((thrown as HttpError).status).toBe(400) + expect((thrown as HttpError).code).toBe('invalid_query') + }) +}) + +describe('when ClickHouse is not there', () => { + let dead: ClickHouseClient + let deadApp: FastifyInstance + + beforeEach(() => { + // A port nothing listens on, in the range no service in this repository + // binds. The client does not connect until it is asked to. + dead = createChClient({ + url: 'http://127.0.0.1:1', + username: 'clickmonk', + password: 'clickmonk', + database: 'clickmonk_test', + requestTimeoutMs: 2000, + }) + deadApp = testApp(pool, clock, { ch: dead }) + }) + + afterEach(async () => { + await deadApp.close() + await dead.close() + }) + + it('answers a report 503 and names the store, not an internal error', async () => { + const r = await summary(deadApp) + expect(r.statusCode).toBe(503) + expect(r.json().error).toBe('reporting_unavailable') + // Nothing about the failure itself: a connection refused to a named host + // and port is an address this service is not obliged to hand out. + expect(r.json().message).not.toContain('127.0.0.1') + }) + + it('still answers everything that does not read it', async () => { + const r = await deadApp.inject({ method: 'GET', url: '/api/links', headers: read(cookie) }) + expect(r.statusCode).toBe(200) + }) + + it('answers a report 503 when there is no client at all', async () => { + const none = testApp(pool, clock) + try { + const r = await summary(none) + expect(r.statusCode).toBe(503) + expect(r.json().error).toBe('reporting_unavailable') + } finally { + await none.close() + } + }) +}) diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts new file mode 100644 index 0000000..a12f18b --- /dev/null +++ b/packages/admin/src/reports.ts @@ -0,0 +1,279 @@ +/** + * Reports: what the rollups say about a window. + * + * Everything here is a read behind the credential this service already has — + * a session or an API key, because a nightly export is a script and not a + * browser — and none of it is a write, so the cross-site rule that guards a + * write does not apply. + * + * **A credential is not a bound.** A caller who has one, or a key that has + * leaked, can loop any of these, and each pass is a scan of a window they + * chose. So a window is bounded, a page is bounded, and at most two of these + * run in this process at once; past that the answer is a refusal with + * `retry-after` rather than a queued query. The same argument as the + * on-demand DNS check, for the same reason. + * + * **A report counts whole hours.** The rollups are hourly, so `from` is + * floored to the hour and `to` is raised to the next one, and the response + * says which window was counted rather than echoing the one that was asked + * for. The click log reads raw clicks and uses the window exactly as given; + * the two can therefore differ over a ragged window, which is why both of them + * say what they counted. + */ +import { type ConcurrencyGate, MAX_REPORT_WINDOW_DAYS, MAX_REPORT_WINDOW_MS } from '@clickmonk/core' +import type { ClickHouseClient } from '@clickmonk/db' +import type { FastifyInstance, FastifyRequest } from 'fastify' +import { z } from 'zod' +import type { AdminContext } from './app.js' +import { requireCredential } from './auth.js' +import { fail } from './http.js' + +/** The server-side bound on one report query. Shorter than the client's own. */ +export const CH_MAX_EXECUTION_SECONDS = 25 + +/** Exported because the chart aligns to its own bucket and the log aligns to nothing. */ +export const HOUR_MS = 3_600_000 + +/** + * The three fields every report query has. Spread into each route's own + * schema rather than shared as a base object, so that each route stays + * `.strict()` — a query parameter nobody knows is a 400 and not something + * quietly ignored, which is what makes "no request field can widen what is + * read" checkable on a surface that has nothing to widen to yet. + */ +export const WINDOW_FIELDS = { + from: z.string().datetime({ offset: true }), + to: z.string().datetime({ offset: true }), + link: z.string().uuid().optional(), +} + +export interface ReportWindow { + fromMs: number + toMs: number + /** The same two instants in ClickHouse's own text form. */ + from: string + to: string + linkId: string | null +} + +const chTime = (ms: number): string => new Date(ms).toISOString().replace('T', ' ').replace('Z', '') + +/** + * A window from a parsed query. `alignMs` floors the start and raises the end + * to a multiple of itself — an hour for the summary, the bucket size for the + * chart, because that is the grain each of them can actually answer. The log + * passes null and gets the instants as they were sent. + * + * The bound is on the window that will actually be read, so it is measured + * after alignment: a caller must not be able to buy an extra bucket of scan by + * asking for a ragged window one millisecond inside the ceiling. + * + * The two bounds do not compose, either: four hundred days of hourly buckets + * is 9,600 of them against a ceiling of 2,000, and the ceiling bites first + * from about 84 days on. Whatever asks for buckets checks its own count; what + * this function owes it is a window made of numbers. + */ +export function parseWindow( + q: { from: string; to: string; link?: string }, + o: { alignMs: number | null }, +): ReportWindow { + const rawFrom = Date.parse(q.from) + const rawTo = Date.parse(q.to) + const align = o.alignMs + const fromMs = align === null ? rawFrom : Math.floor(rawFrom / align) * align + const toMs = align === null ? rawTo : Math.ceil(rawTo / align) * align + // `!(toMs > fromMs)` and not `toMs <= fromMs`: every comparison with NaN is + // false, so the second form lets an unparseable date through, and NaN then + // survives the alignment, the length check and the bucket ceiling — each of + // which compares with `>` and is therefore also false. The double negative is + // the guard. Do not simplify it. + if (!(toMs > fromMs)) { + return fail(400, 'invalid_query', 'to: must be after from') + } + if (toMs - fromMs > MAX_REPORT_WINDOW_MS) { + return fail(400, 'window_too_long', `a window may be at most ${MAX_REPORT_WINDOW_DAYS} days`) + } + return { + fromMs, + toMs, + from: chTime(fromMs), + to: chTime(toMs), + linkId: q.link ?? null, + } +} + +/** A parsed query, or a 400 that names the field. Query strings, not bodies. */ +export function readQuery(schema: z.ZodType, query: unknown): T { + const r = schema.safeParse(query ?? {}) + if (r.success) return r.data + const issues = r.error.issues + .map((i) => `${i.path.join('.') || 'query'}: ${i.message}`) + .join('; ') + return fail(400, 'invalid_query', issues) +} + +/** + * The client, or a 503. Not having one at all and not being able to reach one + * are the same answer to a caller: reporting is not available on this install + * right now. They are different lines in the log. + */ +export function requireCh(ctx: AdminContext, req: FastifyRequest): ClickHouseClient { + if (!ctx.ch) { + req.log.error('a report was asked for but this service has no clickhouse client') + // `return fail(…)`, never a bare call. `fail` is a const arrow with no + // annotation on the variable, so TypeScript does not treat the call as + // never-returning and does not narrow `ctx.ch` away from undefined; a bare + // call leaves the line below as `ClickHouseClient | undefined`. The link + // routes carry a comment saying the same thing, for the same reason. + return fail(503, 'reporting_unavailable', 'reporting is not available on this install') + } + return ctx.ch +} + +/** + * One query, as rows. + * + * Every value is a bound parameter; the only thing built into the text is a + * clause that is present or absent, never a value. A failure is a 503 and a + * server-side log line: an unreachable or overloaded store is the install's + * state rather than a fault in the request, and `{"error":"internal"}` would + * tell the operator nothing they could act on. Nothing ClickHouse said reaches + * the caller — an error body can carry a table name, a column, or the address + * of the server. + */ +export async function chRows(o: { + ch: ClickHouseClient + req: FastifyRequest + query: string + params: Record +}): Promise { + try { + const rs = await o.ch.query({ + query: o.query, + query_params: o.params, + format: 'JSONEachRow', + clickhouse_settings: { max_execution_time: CH_MAX_EXECUTION_SECONDS }, + }) + return await rs.json() + } catch (err) { + o.req.log.error({ err }, 'clickhouse query failed') + return fail(503, 'reporting_unavailable', 'reporting is not available on this install') + } +} + +/** Takes a slot, or refuses. Always gives it back. */ +export async function withSlot( + gate: ConcurrencyGate, + o: { code: string; message: string }, + fn: () => Promise, +): Promise { + if (!gate.tryEnter()) { + return fail(429, o.code, o.message, { 'retry-after': '1' }) + } + try { + return await fn() + } finally { + gate.leave() + } +} + +/** UInt64 comes back as a decimal string; every count here fits a double many times over. */ +const count = (v: string | number | undefined): number => Number(v ?? 0) + +/** The window clause every report shares, with the link filter only when there is one. */ +export function windowClause(w: ReportWindow, column = 'hour'): string { + const parts = [`${column} >= {from:DateTime64(3,'UTC')}`, `${column} < {to:DateTime64(3,'UTC')}`] + if (w.linkId !== null) parts.push('link_id = {link:UUID}') + return parts.join(' AND ') +} + +export function windowParams(w: ReportWindow): Record { + return { from: w.from, to: w.to, ...(w.linkId === null ? {} : { link: w.linkId }) } +} + +const SummaryQuery = z.object({ ...WINDOW_FIELDS }).strict() + +export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): void { + /** + * The numbers at the top of a dashboard: how many clicks, how many people, + * and how they broke down by the three closed dimensions. + * + * Three queries rather than one. The totals cannot be summed from the + * grouped rows — a click belongs to exactly one (class, action, outcome) + * triple so its count can be added up, but a visitor appears under as many + * triples as they have clicks, and adding those gives a number larger than + * the number of people. The third is the freshness of the whole install, + * which is not about this window at all. + */ + app.get('/api/reports/summary', async (req) => { + requireCredential(req) + const q = readQuery(SummaryQuery, req.query) + const w = parseWindow(q, { alignMs: HOUR_MS }) + const ch = requireCh(ctx, req) + return withSlot( + ctx.reportGate, + { code: 'too_many_reports', message: 'too many reports at once; try again' }, + async () => { + const where = windowClause(w) + const params = windowParams(w) + const grouped = await chRows<{ + traffic_class: string + action: string + outcome: string + clicks: string + }>({ + ch, + req, + query: `SELECT traffic_class, action, outcome, uniqExactMerge(clicks_state) AS clicks + FROM clicks_hourly WHERE ${where} + GROUP BY traffic_class, action, outcome`, + params, + }) + const [totals] = await chRows<{ clicks: string; visitors: string }>({ + ch, + req, + query: `SELECT uniqExactMerge(clicks_state) AS clicks, + uniqExactMerge(visitors_state) AS visitors + FROM clicks_hourly WHERE ${where}`, + params, + }) + const [newest] = await chRows<{ newest: string }>({ + ch, + req, + query: 'SELECT toString(max(hour)) AS newest FROM clicks_hourly', + params: {}, + }) + + const add = (into: Record, key: string, n: number): void => { + into[key] = (into[key] ?? 0) + n + } + const byClass: Record = {} + const byAction: Record = {} + const byOutcome: Record = {} + for (const row of grouped) { + const n = count(row.clicks) + add(byClass, row.traffic_class, n) + add(byAction, row.action, n) + add(byOutcome, row.outcome, n) + } + + return { + window: { from: new Date(w.fromMs).toISOString(), to: new Date(w.toMs).toISOString() }, + link: w.linkId, + clicks: count(totals?.clicks), + visitors: count(totals?.visitors), + byClass, + byAction, + byOutcome, + // Empty on an install with no clicks at all: ClickHouse answers the + // zero of the type for max() over nothing, which is not an hour + // anybody had. + newestHour: + !newest?.newest || newest.newest.startsWith('1970') + ? null + : `${newest.newest.replace(' ', 'T')}.000Z`, + } + }, + ) + }) +} diff --git a/packages/admin/src/testing.ts b/packages/admin/src/testing.ts index a1420d2..43ff1ad 100644 --- a/packages/admin/src/testing.ts +++ b/packages/admin/src/testing.ts @@ -60,6 +60,11 @@ export const SHIPPED_TRUSTED_PROXIES = ['uniquelocal', 'loopback'] * `trustProxy: false` by default, because most suites are not about forwarded * headers and a false here keeps them reading what they were sent. A suite * that is about them passes `SHIPPED_TRUSTED_PROXIES`. + * + * A suite that reads a report passes `{ ch }` in `extra`; one that does not, + * does not, and the report routes then answer 503 — which is a real answer + * this service gives and has a test of its own. Nothing ClickHouse is + * re-exported from here: each suite builds its own client with `testCh()`. */ export function testApp( pg: Pool, diff --git a/packages/db/src/clients.ts b/packages/db/src/clients.ts index 4adfb99..637167e 100644 --- a/packages/db/src/clients.ts +++ b/packages/db/src/clients.ts @@ -12,6 +12,8 @@ export interface ChConfig { database: string /** Passed straight through to the client's own logger, e.g. ClickHouseLogLevel.OFF to silence it. */ logLevel?: ClickHouseLogLevel + /** Client-side bound on one query, in milliseconds. */ + requestTimeoutMs?: number } export interface PgPoolOptions { @@ -50,5 +52,6 @@ export function createChClient(cfg: ChConfig): ClickHouseClient { password: cfg.password, database: cfg.database, ...(cfg.logLevel !== undefined ? { log: { level: cfg.logLevel } } : {}), + ...(cfg.requestTimeoutMs !== undefined ? { request_timeout: cfg.requestTimeoutMs } : {}), }) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 42dcc22..49108b9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: packages/admin: dependencies: + '@clickhouse/client': + specifier: ^1.23.1 + version: 1.23.1 '@clickmonk/core': specifier: workspace:* version: link:../core From eb46e0bc64df97ae25dfab52d243ba09ac877950 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 13:43:22 +0000 Subject: [PATCH 19/65] admin: the log is what tells a missing store from an unreachable one --- packages/admin/src/reports.test.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index c0ebc92..25c8314 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -286,12 +286,30 @@ describe('when ClickHouse is not there', () => { expect(r.statusCode).toBe(200) }) - it('answers a report 503 when there is no client at all', async () => { - const none = testApp(pool, clock) + // The answer is the same 503 as an unreachable store, so the status alone + // pins nothing: with no guard at all the query is attempted on nothing, the + // TypeError is caught where a ClickHouse failure is caught, and the caller + // sees the same body. The log is the difference, and it is the difference an + // operator reads — so the log is what this asserts. + it('answers a report 503 when there is no client at all, and says which it was', async () => { + const lines: string[] = [] + const none = testApp(pool, clock, { + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) try { const r = await summary(none) expect(r.statusCode).toBe(503) expect(r.json().error).toBe('reporting_unavailable') + expect(lines.filter((l) => l.includes('has no clickhouse client')).length).toBe(1) + // And nothing was asked of a store that is not there. + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) } finally { await none.close() } From 8e558e48fe2d8cf95ee694489cef87cb4009b153 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 13:47:40 +0000 Subject: [PATCH 20/65] admin: a refused connection is not a test failure --- packages/admin/src/reports.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 25c8314..aae77ca 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -1,3 +1,4 @@ +import { ClickHouseLogLevel } from '@clickhouse/client' import { ConcurrencyGate } from '@clickmonk/core' import { type ClickHouseClient, createChClient } from '@clickmonk/db' import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' @@ -257,12 +258,18 @@ describe('when ClickHouse is not there', () => { beforeEach(() => { // A port nothing listens on, in the range no service in this repository // binds. The client does not connect until it is asked to. + // + // Its own logger is off, as the shipper's unreachable client is: the + // refused connection is what these tests arrange, so printing it on every + // green run teaches a reader to skim the stderr a real failure appears in. + // What this service logged is asserted below, not printed. dead = createChClient({ url: 'http://127.0.0.1:1', username: 'clickmonk', password: 'clickmonk', database: 'clickmonk_test', requestTimeoutMs: 2000, + logLevel: ClickHouseLogLevel.OFF, }) deadApp = testApp(pool, clock, { ch: dead }) }) @@ -290,7 +297,10 @@ describe('when ClickHouse is not there', () => { // pins nothing: with no guard at all the query is attempted on nothing, the // TypeError is caught where a ClickHouse failure is caught, and the caller // sees the same body. The log is the difference, and it is the difference an - // operator reads — so the log is what this asserts. + // operator reads — so the log is what this asserts. **Do not turn this into + // a status code**: answering the two cases differently would tell a caller + // which of them it was, which is the one thing this surface set out not to + // say, so the log line is the claim and the log line is what pins it. it('answers a report 503 when there is no client at all, and says which it was', async () => { const lines: string[] = [] const none = testApp(pool, clock, { From a9a2c8804caf136428d75057fbf592d38b3864c0 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 14:17:20 +0000 Subject: [PATCH 21/65] admin: the end of a window, and the guards nothing was watching --- packages/admin/src/config.test.ts | 21 ++++++- packages/admin/src/reports.test.ts | 90 +++++++++++++++++++++++++++++- packages/admin/src/reports.ts | 28 +++++++--- packages/db/src/clients.test.ts | 40 ++++++++++++- test/stack/compose-config.test.ts | 24 ++++++++ 5 files changed, 189 insertions(+), 14 deletions(-) diff --git a/packages/admin/src/config.test.ts b/packages/admin/src/config.test.ts index 007b86e..32d2e35 100644 --- a/packages/admin/src/config.test.ts +++ b/packages/admin/src/config.test.ts @@ -28,9 +28,24 @@ describe('loadConfig', () => { }) }) - it('refuses to start without the store the reports read', () => { - const { CLICKMONK_CLICKHOUSE_URL: _omitted, ...rest } = base - expect(() => loadConfig(rest)).toThrow(/CLICKMONK_CLICKHOUSE_URL/) + // All four, one row each: the comment on them claims that an install with a + // typo fails at boot with the variable named, and three of the four were + // otherwise free to become optional with that claim still written down. + it.each([ + 'CLICKMONK_CLICKHOUSE_URL', + 'CLICKMONK_CLICKHOUSE_USER', + 'CLICKMONK_CLICKHOUSE_PASSWORD', + 'CLICKMONK_CLICKHOUSE_DB', + ])('refuses to start without %s, which the reports read', (name) => { + const rest: Record = { ...base } + delete rest[name] + expect(() => loadConfig(rest)).toThrow(new RegExp(name)) + }) + + // The password is the one that needs saying: empty is a value an operator may + // have set, and it is not the same as the variable being absent above. + it('takes an empty ClickHouse password, which is not the same as none', () => { + expect(loadConfig({ ...base, CLICKMONK_CLICKHOUSE_PASSWORD: '' }).ch.password).toBe('') }) it('names every missing or bad variable at once', () => { diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index aae77ca..6034a83 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -5,7 +5,7 @@ import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import type { FastifyInstance } from 'fastify' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' import { HttpError } from './http.js' -import { HOUR_MS, parseWindow } from './reports.js' +import { HOUR_MS, newestHourOrNull, parseWindow } from './reports.js' import { ADMIN_HOST, clockFrom, read, signedIn, testApp } from './testing.js' const pool = testPg() @@ -87,7 +87,22 @@ beforeAll(async () => { time: '2026-09-24 11:30:00.000', visitor_id: 'v3', }), + // In the hour that *starts* at the aligned end of the window below, which + // that window must not count: half-open is the only reason two adjacent + // reports do not both claim this click. Its own visitor, so an inclusive + // upper bound moves the visitor count as well as the clicks. + click({ + click_id: '01920000-0000-7000-8000-00000000000f', + time: '2026-09-25 00:30:00.000', + visitor_id: 'v4', + }), ]) + // The same segment again, byte for byte: the worker deletes a spool segment + // only after ClickHouse accepts it, so a crash in between ships it twice. + // Every number a report gives is a set over click ids, so the second copy + // must change none of them — the invariant the whole rollup design rests on, + // and one no fixture of distinct ids can see. + await insert([click()]) }) beforeEach(async () => { @@ -117,10 +132,25 @@ describe('GET /api/reports/summary', () => { byClass: { human: 4, bot: 1 }, byAction: { '': 4, block: 1 }, byOutcome: { target: 4, blocked: 1 }, - newestHour: '2026-09-24T11:00:00.000Z', + // The whole table, not this window: the click half an hour past the end + // of it is the newest hour the install has. + newestHour: '2026-09-25T00:00:00.000Z', }) }) + // The other end of the same clause, and the end that goes unnoticed: an + // inclusive `to` makes two adjacent reports both count the click above, so a + // month read as twelve windows counts eleven hours twice. + it('ends the window before the hour it names', async () => { + const r = await summary(app, 'from=2026-09-24T23:00:00.000Z&to=2026-09-25T00:00:00.000Z') + expect(r.statusCode).toBe(200) + expect(r.json().clicks).toBe(0) + expect(r.json().visitors).toBe(0) + // And the hour it stops before does hold a click, so the zero above is the + // bound rather than an empty table. + expect(r.json().newestHour).toBe('2026-09-25T00:00:00.000Z') + }) + // Three clicks by two visitors, one of whom also clicked the other link: // the per-link visitor count is not a share of the install-wide one. it('counts one link when asked for one', async () => { @@ -251,6 +281,31 @@ describe('parseWindow', () => { }) }) +describe('the newest hour the install holds', () => { + it('is null rather than the epoch when the rollup holds nothing', () => { + expect(newestHourOrNull('1970-01-01 00:00:00')).toBeNull() + expect(newestHourOrNull('')).toBeNull() + expect(newestHourOrNull(undefined)).toBeNull() + expect(newestHourOrNull('2026-09-24 11:00:00')).toBe('2026-09-24T11:00:00.000Z') + }) + + // The claim that branch rests on, asked of the store rather than assumed: an + // aggregate with no GROUP BY answers exactly one row however empty the range, + // and max() over no rows is the zero of the column's type. It is also why the + // summary's totals need no fallback of their own. + it('is the epoch, in one row, when ClickHouse is asked over no rows at all', async () => { + const rs = await ch.query({ + query: `SELECT toString(max(hour)) AS newest, count() AS seen + FROM clicks_hourly WHERE hour >= '2099-01-01 00:00:00'`, + format: 'JSONEachRow', + }) + const rows = await rs.json<{ newest: string; seen: string }>() + expect(rows.length).toBe(1) + expect(rows[0]?.newest).toBe('1970-01-01 00:00:00') + expect(rows[0]?.seen).toBe('0') + }) +}) + describe('when ClickHouse is not there', () => { let dead: ClickHouseClient let deadApp: FastifyInstance @@ -288,6 +343,37 @@ describe('when ClickHouse is not there', () => { expect(r.json().message).not.toContain('127.0.0.1') }) + // A refusal has to be more than a status code: the window bound exists to + // stop a scan, so a bound checked after the query would be no bound at all. + // Against a store that cannot answer, anything that reaches it is a 503 — so + // a 400 here says the refusal came first, and the log says nothing was asked. + it('refuses a window too long to read without reading anything', async () => { + const lines: string[] = [] + const logging = testApp(pool, clock, { + ch: dead, + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + const r = await logging.inject({ + method: 'GET', + url: '/api/reports/summary?from=2025-01-01T00:00:00.000Z&to=2026-09-25T00:00:00.000Z', + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('window_too_long') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + } finally { + await logging.close() + } + }) + it('still answers everything that does not read it', async () => { const r = await deadApp.inject({ method: 'GET', url: '/api/links', headers: read(cookie) }) expect(r.statusCode).toBe(200) diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index a12f18b..33dee2b 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -180,6 +180,21 @@ export async function withSlot( /** UInt64 comes back as a decimal string; every count here fits a double many times over. */ const count = (v: string | number | undefined): number => Number(v ?? 0) +/** + * The newest hour the rollup holds, as an instant — or null when it holds none. + * + * `max()` with no `GROUP BY` answers one row whatever the range, and over no + * rows that row is the zero of the column's type: `1970-01-01 00:00:00`, which + * is not an hour any install had. An empty install must say "nothing yet" + * rather than name the epoch, so the sentinel is read here. A separate function + * because the branch is otherwise reachable only from a suite that owns the + * whole table. + */ +export function newestHourOrNull(text: string | undefined): string | null { + if (!text || text.startsWith('1970')) return null + return `${text.replace(' ', 'T')}.000Z` +} + /** The window clause every report shares, with the link filter only when there is one. */ export function windowClause(w: ReportWindow, column = 'hour'): string { const parts = [`${column} >= {from:DateTime64(3,'UTC')}`, `${column} < {to:DateTime64(3,'UTC')}`] @@ -260,18 +275,17 @@ export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): v return { window: { from: new Date(w.fromMs).toISOString(), to: new Date(w.toMs).toISOString() }, link: w.linkId, + // The `?.` on both of these is the type of an array index under + // `noUncheckedIndexedAccess`, not a guard against a missing row: an + // aggregate with no `GROUP BY` answers exactly one row even over an + // empty range, which the suite pins directly. The zeroes an empty + // window gives back are ClickHouse's, not a fallback of ours. clicks: count(totals?.clicks), visitors: count(totals?.visitors), byClass, byAction, byOutcome, - // Empty on an install with no clicks at all: ClickHouse answers the - // zero of the type for max() over nothing, which is not an hour - // anybody had. - newestHour: - !newest?.newest || newest.newest.startsWith('1970') - ? null - : `${newest.newest.replace(' ', 'T')}.000Z`, + newestHour: newestHourOrNull(newest?.newest), } }, ) diff --git a/packages/db/src/clients.test.ts b/packages/db/src/clients.test.ts index 775498f..d0ee89a 100644 --- a/packages/db/src/clients.test.ts +++ b/packages/db/src/clients.test.ts @@ -1,6 +1,7 @@ +import { ClickHouseLogLevel } from '@clickhouse/client' import { afterAll, describe, expect, it } from 'vitest' -import { createPgPool } from './clients.js' -import { TEST_PG_URL, testPg } from './testing.js' +import { createChClient, createPgPool } from './clients.js' +import { TEST_CH, TEST_PG_URL, testPg } from './testing.js' const admin = testPg() @@ -28,3 +29,38 @@ describe('createPgPool', () => { } }) }) + +describe('createChClient', () => { + /** + * What the bound does, not what it is spelled. The library ignores an option + * name it does not know without a word — `requestTimeout` for + * `request_timeout` reads as a configured bound and is none — so the check + * is a query that outlives the bound and has to come back as a failure + * rather than as an answer. The ceiling is well under the query's own length + * and unrelated to the timeout, so it cannot pass by arithmetic. + */ + it('gives up on a query that outlives the timeout it was given', async () => { + const ch = createChClient({ + ...TEST_CH, + requestTimeoutMs: 300, + logLevel: ClickHouseLogLevel.OFF, + }) + const started = Date.now() + try { + await expect(ch.query({ query: 'SELECT sleep(3)', format: 'JSONEachRow' })).rejects.toThrow() + expect(Date.now() - started).toBeLessThan(2000) + } finally { + await ch.close() + } + }) + + it('leaves a query alone when it is given no timeout at all', async () => { + const ch = createChClient({ ...TEST_CH, logLevel: ClickHouseLogLevel.OFF }) + try { + const rs = await ch.query({ query: 'SELECT sleep(1) AS slept', format: 'JSONEachRow' }) + expect((await rs.json<{ slept: number }>()).length).toBe(1) + } finally { + await ch.close() + } + }) +}) diff --git a/test/stack/compose-config.test.ts b/test/stack/compose-config.test.ts index f77e405..e0df008 100644 --- a/test/stack/compose-config.test.ts +++ b/test/stack/compose-config.test.ts @@ -145,6 +145,30 @@ describe('what the stack publishes', () => { } }) + // The admin API reads ClickHouse directly, and its loader makes all four of + // these required, so a missing one is not a report that degrades: it is a + // container that exits at boot and an install with no admin interface. The + // values are the worker's, because the two services read one store, and the + // password is interpolated rather than written here — a literal would be a + // credential in this file and would also stop being the store's password the + // first time an operator changed theirs. + it('gives the admin API the four values its reports read ClickHouse with', () => { + const block = serviceBlock(cfg, 'admin') + for (const [name, value] of [ + ['CLICKMONK_CLICKHOUSE_URL', 'http://clickhouse:8123'], + ['CLICKMONK_CLICKHOUSE_USER', 'clickmonk'], + ['CLICKMONK_CLICKHOUSE_PASSWORD', 'y'], + ['CLICKMONK_CLICKHOUSE_DB', 'clickmonk'], + ]) { + // Optionally quoted, as the admin-host rows above are: Compose quotes a + // value YAML would otherwise read as something other than a string, and + // the one-character password in this suite's env file is one of them. + const line = new RegExp(`\\n {6}${name}: "?${value}"?\\n`) + expect(block, name).toMatch(line) + expect(serviceBlock(cfg, 'worker'), `${name} on the worker`).toMatch(line) + } + }) + it('tells the redirect to believe a forwarded address only from the stack’s own network', () => { expect(cfg).toContain('CLICKMONK_TRUSTED_PROXIES: uniquelocal,loopback') }) From 7721f34ae15592c527d2dbed12a221505870efed Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 14:18:24 +0000 Subject: [PATCH 22/65] admin: a bad link is a refusal, not a store error --- packages/admin/src/app.ts | 8 +++++--- packages/admin/src/reports.test.ts | 19 +++++++++++++++++++ packages/admin/src/reports.ts | 21 ++++++++++++++++----- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/packages/admin/src/app.ts b/packages/admin/src/app.ts index 9e6a461..b247285 100644 --- a/packages/admin/src/app.ts +++ b/packages/admin/src/app.ts @@ -34,9 +34,11 @@ export const DNS_CHECKS_IN_FLIGHT = 2 * can loop a report endpoint, and each pass is a scan of a window they chose. * Past these the answer is a refusal with `retry-after`, never a queued query, * for the same reason the on-demand DNS check refuses rather than queues. - * Exports get their own, smaller gate because an export is the one request - * that holds a ClickHouse result open for as long as the client takes to read - * it. + * + * The export gate is declared here and nothing takes it yet: the endpoint it + * bounds does not exist. It is separate from the report gate, and smaller, + * because an export holds a ClickHouse result open for as long as its client + * takes to read it, which is not a bound this process can set. */ export const REPORT_QUERIES_IN_FLIGHT = 2 export const EXPORTS_IN_FLIGHT = 1 diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 6034a83..5c0c55c 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -279,6 +279,25 @@ describe('parseWindow', () => { expect((thrown as HttpError).status).toBe(400) expect((thrown as HttpError).code).toBe('invalid_query') }) + + // The link's shape belongs here for the same reason the length does. Bound + // into a query as a UUID, a link that is not one comes back from the store as + // a failure, so a caller reaching this function without the route's schema + // would be told reporting is unavailable when what is wrong is their field. + it('refuses a link that is not an id, whatever schema the caller came through', () => { + let thrown: unknown + try { + parseWindow( + { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z', link: 'nope' }, + { alignMs: HOUR_MS }, + ) + } catch (err) { + thrown = err + } + expect(thrown).toBeInstanceOf(HttpError) + expect((thrown as HttpError).status).toBe(400) + expect((thrown as HttpError).code).toBe('invalid_query') + }) }) describe('the newest hour the install holds', () => { diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 33dee2b..34442d8 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -8,10 +8,10 @@ * * **A credential is not a bound.** A caller who has one, or a key that has * leaked, can loop any of these, and each pass is a scan of a window they - * chose. So a window is bounded, a page is bounded, and at most two of these - * run in this process at once; past that the answer is a refusal with - * `retry-after` rather than a queued query. The same argument as the - * on-demand DNS check, for the same reason. + * chose. So a window is bounded — in length, and to the grain the rollups + * answer at — and at most two of these run in this process at once; past that + * the answer is a refusal with `retry-after` rather than a queued query. The + * same argument as the on-demand DNS check, for the same reason. * * **A report counts whole hours.** The rollups are hourly, so `from` is * floored to the hour and `to` is raised to the next one, and the response @@ -41,10 +41,13 @@ export const HOUR_MS = 3_600_000 * quietly ignored, which is what makes "no request field can widen what is * read" checkable on a surface that has nothing to widen to yet. */ +/** A link filter is a link id and nothing else. Checked here and in the parser. */ +const LinkId = z.string().uuid() + export const WINDOW_FIELDS = { from: z.string().datetime({ offset: true }), to: z.string().datetime({ offset: true }), - link: z.string().uuid().optional(), + link: LinkId.optional(), } export interface ReportWindow { @@ -93,6 +96,14 @@ export function parseWindow( if (toMs - fromMs > MAX_REPORT_WINDOW_MS) { return fail(400, 'window_too_long', `a window may be at most ${MAX_REPORT_WINDOW_DAYS} days`) } + // The shape of the link, here and not only in a route's schema, for the same + // reason the length is here: this value is bound into a query as a UUID, so a + // caller who reaches this function without that schema turns a bad request + // into a store error — a 503 that says reporting is unavailable when what is + // actually wrong is the one field the caller sent. + if (q.link !== undefined && !LinkId.safeParse(q.link).success) { + return fail(400, 'invalid_query', 'link: must be a link id') + } return { fromMs, toMs, From 037aea22524fcc5515370ed6eb927f5f70b9143e Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 14:32:24 +0000 Subject: [PATCH 23/65] admin: clicks by hour and by day, with the quiet hours in them --- packages/admin/src/reports.test.ts | 179 +++++++++++++++++++++++++++++ packages/admin/src/reports.ts | 102 +++++++++++++++- 2 files changed, 280 insertions(+), 1 deletion(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 5c0c55c..ed4c945 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -258,6 +258,151 @@ describe('GET /api/reports/summary', () => { }) }) +describe('GET /api/reports/timeseries', () => { + it('gives one bucket an hour, including the hours with nothing in them', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T09:00:00.000Z&to=2026-09-24T13:00:00.000Z&bucket=hour', + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: { from: '2026-09-24T09:00:00.000Z', to: '2026-09-24T13:00:00.000Z' }, + link: null, + bucket: 'hour', + buckets: [ + { at: '2026-09-24T09:00:00.000Z', clicks: 0, visitors: 0 }, + { at: '2026-09-24T10:00:00.000Z', clicks: 4, visitors: 2 }, + { at: '2026-09-24T11:00:00.000Z', clicks: 1, visitors: 1 }, + { at: '2026-09-24T12:00:00.000Z', clicks: 0, visitors: 0 }, + ], + }) + }) + + it('gives one bucket a day, and counts a visitor across the hours of that day once', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=day', + headers: read(cookie), + }) + expect(r.json().buckets).toEqual([ + // Five clicks by three visitors: v1 clicked in both hours and is one + // visitor for the day, which summing the two hourly numbers would not + // give. + { at: '2026-09-24T00:00:00.000Z', clicks: 5, visitors: 3 }, + ]) + }) + + // The bucket the window's end names is the one it stops before, the same way + // the summary's is: with `<= to` this day chart would carry a second bucket + // holding the click at 2026-09-25 00:30, and two adjacent charts drawn a day + // apart would each show it. + it('stops before the bucket its end names', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=day', + headers: read(cookie), + }) + expect(r.json().buckets).toHaveLength(1) + // And the day it stops before does hold a click, so the single bucket is + // the bound rather than an empty table beyond it. + const next = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-25T00:00:00.000Z&to=2026-09-26T00:00:00.000Z&bucket=day', + headers: read(cookie), + }) + expect(next.json().buckets).toEqual([ + { at: '2026-09-25T00:00:00.000Z', clicks: 1, visitors: 1 }, + ]) + }) + + it('aligns a day chart to whole days, and says so', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T10:30:00.000Z&to=2026-09-24T11:30:00.000Z&bucket=day', + headers: read(cookie), + }) + expect(r.json().window).toEqual({ + from: '2026-09-24T00:00:00.000Z', + to: '2026-09-25T00:00:00.000Z', + }) + expect(r.json().buckets).toHaveLength(1) + }) + + it('counts one link when asked for one', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/reports/timeseries?from=2026-09-24T10:00:00.000Z&to=2026-09-24T11:00:00.000Z&bucket=hour&link=${LINK_B}`, + headers: read(cookie), + }) + expect(r.json().buckets).toEqual([{ at: '2026-09-24T10:00:00.000Z', clicks: 1, visitors: 1 }]) + }) + + it('refuses more buckets than one response carries, and says to ask for days', async () => { + const r = await app.inject({ + method: 'GET', + // A hundred days of hours: 2,400 buckets, past the ceiling of 2,000. + url: '/api/reports/timeseries?from=2026-06-16T00:00:00.000Z&to=2026-09-24T00:00:00.000Z&bucket=hour', + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('too_many_buckets') + expect(r.json().message).toContain('bucket=day') + }) + + it('takes a hundred days of days, which is inside the ceiling', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-06-16T00:00:00.000Z&to=2026-09-24T00:00:00.000Z&bucket=day', + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + const buckets = r.json().buckets as { at: string; clicks: number; visitors: number }[] + expect(buckets).toHaveLength(100) + expect(buckets[0]?.at).toBe('2026-06-16T00:00:00.000Z') + expect(buckets[99]?.at).toBe('2026-09-23T00:00:00.000Z') + // Every fixture click is on the 24th, which is the instant this window + // ends at, so every one of the hundred buckets is a real zero rather than + // a row that was not returned. + expect(buckets.reduce((n, b) => n + b.clicks, 0)).toBe(0) + }) + + it.each([ + ['no bucket at all', 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z'], + [ + 'a bucket nobody has', + 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=week', + ], + [ + 'a field nobody knows', + 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=hour&tz=CET', + ], + ])('refuses %s', async (_label, query) => { + const r = await app.inject({ + method: 'GET', + url: `/api/reports/timeseries?${query}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_query') + }) + + it('refuses when every report slot is taken', async () => { + const other = testApp(pool, clock, { ch, reportGate: new ConcurrencyGate(0) }) + try { + const r = await other.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=hour', + headers: read(cookie), + }) + expect(r.statusCode).toBe(429) + expect(r.json().error).toBe('too_many_reports') + } finally { + await other.close() + } + }) +}) + // The route cannot reach this: `from` is a datetime in the schema, so an // unreadable date is a 400 before the parser sees it. The parser is exported // and the chart and the log will call it too, so the bound lives in the @@ -393,6 +538,40 @@ describe('when ClickHouse is not there', () => { } }) + // The same claim for the chart's own ceiling, and it needs making separately: + // a hundred days is well inside the four hundred the window bound allows, so + // the only thing that can refuse 2,400 hourly buckets is the bucket count. A + // count taken after the query would be no ceiling at all — the scan it exists + // to stop would already have happened — and against a store that cannot + // answer, anything that reaches it is a 503, so the 400 is what says the + // count came first. + it('refuses too many buckets without reading anything', async () => { + const lines: string[] = [] + const logging = testApp(pool, clock, { + ch: dead, + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + const r = await logging.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-06-16T00:00:00.000Z&to=2026-09-24T00:00:00.000Z&bucket=hour', + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('too_many_buckets') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + } finally { + await logging.close() + } + }) + it('still answers everything that does not read it', async () => { const r = await deadApp.inject({ method: 'GET', url: '/api/links', headers: read(cookie) }) expect(r.statusCode).toBe(200) diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 34442d8..e8d9f7c 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -20,7 +20,16 @@ * the two can therefore differ over a ragged window, which is why both of them * say what they counted. */ -import { type ConcurrencyGate, MAX_REPORT_WINDOW_DAYS, MAX_REPORT_WINDOW_MS } from '@clickmonk/core' +import { + BUCKET_MS, + type ConcurrencyGate, + MAX_REPORT_BUCKETS, + MAX_REPORT_WINDOW_DAYS, + MAX_REPORT_WINDOW_MS, + REPORT_BUCKETS, + type ReportBucket, + bucketCount, +} from '@clickmonk/core' import type { ClickHouseClient } from '@clickmonk/db' import type { FastifyInstance, FastifyRequest } from 'fastify' import { z } from 'zod' @@ -219,6 +228,18 @@ export function windowParams(w: ReportWindow): Record { const SummaryQuery = z.object({ ...WINDOW_FIELDS }).strict() +/** + * How each bucket size is grouped. A module constant per bucket rather than + * anything built from the request: the value that reaches the query text is + * one of exactly two strings written here. + */ +const BUCKET_EXPR: Record = { + hour: 'hour', + day: 'toStartOfDay(hour)', +} + +const TimeseriesQuery = z.object({ ...WINDOW_FIELDS, bucket: z.enum(REPORT_BUCKETS) }).strict() + export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): void { /** * The numbers at the top of a dashboard: how many clicks, how many people, @@ -301,4 +322,83 @@ export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): v }, ) }) + + /** + * The chart: one row per bucket, with the empty ones filled in. + * + * Filled here rather than by ClickHouse's `WITH FILL`, because the loop that + * fills is then the same loop whose count decided whether this window was + * allowed at all — a response cannot come back longer than the ceiling that + * let it through. The other way round, the ceiling is in this file and the + * filling is in a query, and nothing holds them to each other. + * + * The bucket count is checked here and not by `parseWindow`, because the two + * bounds do not compose: four hundred days is inside the window bound and + * 9,600 hourly buckets, nearly five times the ceiling. The window bound alone + * would let that through. + * + * A visitor is merged per bucket and never summed across buckets: somebody + * who clicked at ten and at eleven is two hourly visitors and one daily one, + * and both of those are the right answer to their own question. + */ + app.get('/api/reports/timeseries', async (req) => { + requireCredential(req) + const q = readQuery(TimeseriesQuery, req.query) + const step = BUCKET_MS[q.bucket] + const w = parseWindow(q, { alignMs: step }) + const buckets = bucketCount(w.fromMs, w.toMs, q.bucket) + if (buckets > MAX_REPORT_BUCKETS) { + // `return fail(…)` rather than a bare call, for the reason `requireCh` + // spells out: `fail` is a const arrow, so TypeScript does not treat a + // bare call as ending the path. + return fail( + 400, + 'too_many_buckets', + `that window is ${buckets} buckets and at most ${MAX_REPORT_BUCKETS} are returned; ask for bucket=day or a shorter window`, + ) + } + const ch = requireCh(ctx, req) + return withSlot( + ctx.reportGate, + { code: 'too_many_reports', message: 'too many reports at once; try again' }, + async () => { + // The alias is `at` and not `hour`: an alias that shadows the column + // would be what the `WHERE` below reads, and a string where a DateTime + // belongs fails with an illegal-type error. + const rows = await chRows<{ at: string; clicks: string; visitors: string }>({ + ch, + req, + query: `SELECT toString(${BUCKET_EXPR[q.bucket]}) AS at, + uniqExactMerge(clicks_state) AS clicks, + uniqExactMerge(visitors_state) AS visitors + FROM clicks_hourly WHERE ${windowClause(w)} + GROUP BY at ORDER BY at`, + params: windowParams(w), + }) + // Keyed by the same text ClickHouse returned, so nothing has to agree + // about how to format an instant twice. `toString` of a DateTime gives + // nineteen characters and `chTime` gives milliseconds too, so the key + // is the sliced form of ours. + const found = new Map(rows.map((row) => [row.at, row])) + const out: { at: string; clicks: number; visitors: number }[] = [] + // `at < w.toMs`, matching the half-open clause the query ran with: the + // bucket the window's end names belongs to the next window, and filling + // it here would draw a bar two adjacent charts both claim. + for (let at = w.fromMs; at < w.toMs; at += step) { + const row = found.get(chTime(at).slice(0, 19)) + out.push({ + at: new Date(at).toISOString(), + clicks: count(row?.clicks), + visitors: count(row?.visitors), + }) + } + return { + window: { from: new Date(w.fromMs).toISOString(), to: new Date(w.toMs).toISOString() }, + link: w.linkId, + bucket: q.bucket, + buckets: out, + } + }, + ) + }) } From 4e4ef21b9bd36aa4fddaa24e5726f3a3ba9f08fc Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 14:36:06 +0000 Subject: [PATCH 24/65] admin: a visitor in two hours of one day, or nothing sums differently --- packages/admin/src/reports.test.ts | 34 ++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index ed4c945..0791eb0 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -81,11 +81,14 @@ beforeAll(async () => { destination: '', target_id: '', }), - // An hour later, and outside the narrow window one test asks for. + // An hour later, and outside the narrow window one test asks for. The same + // visitor as the first, which is what makes a day's visitor count something + // other than the sum of its hours: two visitors at ten and one at eleven, + // and two for the day. With a third visitor here the two arithmetics agree + // and neither the chart nor the summary would notice being summed. click({ click_id: '01920000-0000-7000-8000-00000000000e', time: '2026-09-24 11:30:00.000', - visitor_id: 'v3', }), // In the hour that *starts* at the aligned end of the window below, which // that window must not count: half-open is the only reason two adjacent @@ -128,7 +131,9 @@ describe('GET /api/reports/summary', () => { window: { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z' }, link: null, clicks: 5, - visitors: 3, + // Two, not three: the click an hour later is the same visitor as the + // first, so the window's visitors are not its hours' visitors added up. + visitors: 2, byClass: { human: 4, bot: 1 }, byAction: { '': 4, block: 1 }, byOutcome: { target: 4, blocked: 1 }, @@ -286,10 +291,10 @@ describe('GET /api/reports/timeseries', () => { headers: read(cookie), }) expect(r.json().buckets).toEqual([ - // Five clicks by three visitors: v1 clicked in both hours and is one - // visitor for the day, which summing the two hourly numbers would not - // give. - { at: '2026-09-24T00:00:00.000Z', clicks: 5, visitors: 3 }, + // Five clicks by two visitors. One of them clicked in both hours and is + // one visitor for the day, so the day's number is not the two hourly + // numbers added up: those are 2 and 1, and this is 2. + { at: '2026-09-24T00:00:00.000Z', clicks: 5, visitors: 2 }, ]) }) @@ -387,6 +392,21 @@ describe('GET /api/reports/timeseries', () => { expect(r.json().error).toBe('invalid_query') }) + // The status alone does not pin the enum. With a plain string in its place an + // unknown bucket is still a 400, from the wrong guard: no bucket size is found + // for a name nobody has, the alignment becomes NaN, and the caller is told + // their `to` is not after their `from` — about a field they got right. So what + // pins the enum is the field the refusal names. + it('names the bucket when the bucket is one nobody has', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=week', + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().message).toContain('bucket') + }) + it('refuses when every report slot is taken', async () => { const other = testApp(pool, clock, { ch, reportGate: new ConcurrencyGate(0) }) try { From 64ba1f9b2fc03a25c3ca955d7f09ec7be9152f7a Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 14:39:48 +0000 Subject: [PATCH 25/65] admin: the chart asks for a credential of its own --- packages/admin/src/reports.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 0791eb0..dd4640e 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -407,6 +407,20 @@ describe('GET /api/reports/timeseries', () => { expect(r.json().message).toContain('bucket') }) + // Its own test and not the summary's: the credential is asked for per route + // here, with no hook over all of them, so a route that forgot to ask would be + // an unauthenticated read of every click the install has and the summary's + // test would still be green. + it('needs a credential', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=hour', + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + }) + it('refuses when every report slot is taken', async () => { const other = testApp(pool, clock, { ch, reportGate: new ConcurrencyGate(0) }) try { From 06fcd231b875dc2522fedba7643e321c36070d45 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 15:05:11 +0000 Subject: [PATCH 26/65] admin: a refusal that reads nothing, and a body asserted whole --- packages/admin/src/reports.test.ts | 172 +++++++++++++++++++++-------- 1 file changed, 126 insertions(+), 46 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index dd4640e..7b9a0dc 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -156,13 +156,28 @@ describe('GET /api/reports/summary', () => { expect(r.json().newestHour).toBe('2026-09-25T00:00:00.000Z') }) - // Three clicks by two visitors, one of whom also clicked the other link: - // the per-link visitor count is not a share of the install-wide one. + // One click by a visitor who also clicked the other link, so the per-link + // visitor count is not a share of the install-wide one. + // + // Asserted whole, and this is the response to assert whole: the one above + // carries `link: null`, which an echo replaced by a constant would still + // answer. A filter the response does not really carry is the worst of the + // three answers — the numbers are for one link and the body says so about + // nothing in particular. it('counts one link when asked for one', async () => { const r = await summary(app, `${WINDOW}&link=${LINK_B}`) - expect(r.json().clicks).toBe(1) - expect(r.json().visitors).toBe(1) - expect(r.json().link).toBe(LINK_B) + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z' }, + link: LINK_B, + clicks: 1, + visitors: 1, + byClass: { human: 1 }, + byAction: { '': 1 }, + byOutcome: { target: 1 }, + // The whole install's freshness, which a link filter does not narrow. + newestHour: '2026-09-25T00:00:00.000Z', + }) }) it('counts whole hours and says which hours it counted', async () => { @@ -290,12 +305,21 @@ describe('GET /api/reports/timeseries', () => { url: '/api/reports/timeseries?from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z&bucket=day', headers: read(cookie), }) - expect(r.json().buckets).toEqual([ - // Five clicks by two visitors. One of them clicked in both hours and is - // one visitor for the day, so the day's number is not the two hourly - // numbers added up: those are 2 and 1, and this is 2. - { at: '2026-09-24T00:00:00.000Z', clicks: 5, visitors: 2 }, - ]) + expect(r.statusCode).toBe(200) + // Whole, because this is the response where `bucket` is off the value the + // hourly test above asserts: an echo replaced by the constant `'hour'` + // answers that one correctly and this one wrongly. + expect(r.json()).toEqual({ + window: { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z' }, + link: null, + bucket: 'day', + buckets: [ + // Five clicks by two visitors. One of them clicked in both hours and is + // one visitor for the day, so the day's number is not the two hourly + // numbers added up: those are 2 and 1, and this is 2. + { at: '2026-09-24T00:00:00.000Z', clicks: 5, visitors: 2 }, + ], + }) }) // The bucket the window's end names is the one it stops before, the same way @@ -334,13 +358,21 @@ describe('GET /api/reports/timeseries', () => { expect(r.json().buckets).toHaveLength(1) }) + // Whole for the same reason the summary's link test is whole: the only other + // body this block asserts entirely carries `link: null`. it('counts one link when asked for one', async () => { const r = await app.inject({ method: 'GET', url: `/api/reports/timeseries?from=2026-09-24T10:00:00.000Z&to=2026-09-24T11:00:00.000Z&bucket=hour&link=${LINK_B}`, headers: read(cookie), }) - expect(r.json().buckets).toEqual([{ at: '2026-09-24T10:00:00.000Z', clicks: 1, visitors: 1 }]) + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: { from: '2026-09-24T10:00:00.000Z', to: '2026-09-24T11:00:00.000Z' }, + link: LINK_B, + bucket: 'hour', + buckets: [{ at: '2026-09-24T10:00:00.000Z', clicks: 1, visitors: 1 }], + }) }) it('refuses more buckets than one response carries, and says to ask for days', async () => { @@ -355,6 +387,32 @@ describe('GET /api/reports/timeseries', () => { expect(r.json().message).toContain('bucket=day') }) + // The ceiling itself, at the two points either side of it. The tests around + // this one reach 2,400 and 100, so the comparison could be written `>=` — one + // bucket tighter than the number the message promises — and nothing would + // notice. Both windows are written out rather than computed from + // MAX_REPORT_BUCKETS: a bound derived from the constant it is testing moves + // when the constant does and goes on passing. + it('takes the most buckets it will return, and refuses one more', async () => { + // 2026-07-02 16:00 to 2026-09-24 00:00 is 2,000 hours: 704 left in July, + // 744 in August, 552 in September. + const at = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-07-02T16:00:00.000Z&to=2026-09-24T00:00:00.000Z&bucket=hour', + headers: read(cookie), + }) + expect(at.statusCode).toBe(200) + expect(at.json().buckets).toHaveLength(2000) + // One hour earlier is 2,001. + const over = await app.inject({ + method: 'GET', + url: '/api/reports/timeseries?from=2026-07-02T15:00:00.000Z&to=2026-09-24T00:00:00.000Z&bucket=hour', + headers: read(cookie), + }) + expect(over.statusCode).toBe(400) + expect(over.json().error).toBe('too_many_buckets') + }) + it('takes a hundred days of days, which is inside the ceiling', async () => { const r = await app.inject({ method: 'GET', @@ -532,6 +590,38 @@ describe('when ClickHouse is not there', () => { await dead.close() }) + /** + * An app on the unreachable client, with its error log captured. + * + * Every refusal that has to come *before* a query is asked of this app. + * Against a store that cannot answer, anything reaching it is a 503, so a + * refusal's own status is what says nothing was asked — and the log line the + * query failure would have written is asserted absent as well, because a + * guard moved after the query would otherwise leave only the status to tell + * the two apart. + */ + const onDeadWithLog = async ( + fn: (on: FastifyInstance, lines: string[]) => Promise, + ): Promise => { + const lines: string[] = [] + const on = testApp(pool, clock, { + ch: dead, + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + await fn(on, lines) + } finally { + await on.close() + } + } + it('answers a report 503 and names the store, not an internal error', async () => { const r = await summary(deadApp) expect(r.statusCode).toBe(503) @@ -546,20 +636,8 @@ describe('when ClickHouse is not there', () => { // Against a store that cannot answer, anything that reaches it is a 503 — so // a 400 here says the refusal came first, and the log says nothing was asked. it('refuses a window too long to read without reading anything', async () => { - const lines: string[] = [] - const logging = testApp(pool, clock, { - ch: dead, - log: { - level: 'error', - stream: { - write(line: string) { - lines.push(line) - }, - }, - }, - }) - try { - const r = await logging.inject({ + await onDeadWithLog(async (on, lines) => { + const r = await on.inject({ method: 'GET', url: '/api/reports/summary?from=2025-01-01T00:00:00.000Z&to=2026-09-25T00:00:00.000Z', headers: read(cookie), @@ -567,9 +645,25 @@ describe('when ClickHouse is not there', () => { expect(r.statusCode).toBe(400) expect(r.json().error).toBe('window_too_long') expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) - } finally { - await logging.close() - } + }) + }) + + // The credential is asked for route by route, with no hook over all of them, + // and the 401 tests pin that each route asks — not where it asks. Moved below + // the query, an unauthenticated request takes a report slot and scans the + // window it chose before being told it was never allowed to ask, which is the + // scan the credential is there to stop. Both routes, because each one asks + // for its own and a rule one route knows is not inherited by its neighbour. + it.each([ + ['the summary', `/api/reports/summary?${WINDOW}`], + ['the chart', `/api/reports/timeseries?${WINDOW}&bucket=hour`], + ])('refuses %s without a credential before reading anything', async (_label, url) => { + await onDeadWithLog(async (on, lines) => { + const r = await on.inject({ method: 'GET', url, headers: { host: ADMIN_HOST } }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + }) }) // The same claim for the chart's own ceiling, and it needs making separately: @@ -580,20 +674,8 @@ describe('when ClickHouse is not there', () => { // answer, anything that reaches it is a 503, so the 400 is what says the // count came first. it('refuses too many buckets without reading anything', async () => { - const lines: string[] = [] - const logging = testApp(pool, clock, { - ch: dead, - log: { - level: 'error', - stream: { - write(line: string) { - lines.push(line) - }, - }, - }, - }) - try { - const r = await logging.inject({ + await onDeadWithLog(async (on, lines) => { + const r = await on.inject({ method: 'GET', url: '/api/reports/timeseries?from=2026-06-16T00:00:00.000Z&to=2026-09-24T00:00:00.000Z&bucket=hour', headers: read(cookie), @@ -601,9 +683,7 @@ describe('when ClickHouse is not there', () => { expect(r.statusCode).toBe(400) expect(r.json().error).toBe('too_many_buckets') expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) - } finally { - await logging.close() - } + }) }) it('still answers everything that does not read it', async () => { From 47fc1efba7a3e17dd6919e6828677a9038ee00c7 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 15:09:09 +0000 Subject: [PATCH 27/65] admin: comments that describe the chart that was built --- packages/admin/src/reports.test.ts | 33 ++++++++++++++++----- packages/admin/src/reports.ts | 46 +++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 7b9a0dc..6a8ab93 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -296,6 +296,9 @@ describe('GET /api/reports/timeseries', () => { { at: '2026-09-24T11:00:00.000Z', clicks: 1, visitors: 1 }, { at: '2026-09-24T12:00:00.000Z', clicks: 0, visitors: 0 }, ], + // The whole install's newest hour, past the end of this window: the two + // zero buckets above are quiet hours, and this is what says so. + newestHour: '2026-09-25T00:00:00.000Z', }) }) @@ -319,13 +322,22 @@ describe('GET /api/reports/timeseries', () => { // numbers added up: those are 2 and 1, and this is 2. { at: '2026-09-24T00:00:00.000Z', clicks: 5, visitors: 2 }, ], + newestHour: '2026-09-25T00:00:00.000Z', }) }) - // The bucket the window's end names is the one it stops before, the same way - // the summary's is: with `<= to` this day chart would carry a second bucket - // holding the click at 2026-09-25 00:30, and two adjacent charts drawn a day - // apart would each show it. + // The chart's own half-open end is the fill loop's, not the clause's. With + // `at <= w.toMs` the loop draws a bucket for the instant the window ends at — + // an empty one, because the clause returned no row for it — and two charts + // drawn a day apart each show a bar for the same day. + // + // The clause's upper bound is a separate guard and this block does not cover + // it: `< to` written as `<= to` leaves every chart test green, because the + // extra group comes back and the fill loop never asks for it, and only the + // summary catches it. The clause's lower bound the chart does cover — `>= + // from` as `> from` fails this test and the per-link one. So an endpoint + // added later that reuses `windowClause` inherits its upper bound from the + // summary's tests and from nothing here. it('stops before the bucket its end names', async () => { const r = await app.inject({ method: 'GET', @@ -372,6 +384,8 @@ describe('GET /api/reports/timeseries', () => { link: LINK_B, bucket: 'hour', buckets: [{ at: '2026-09-24T10:00:00.000Z', clicks: 1, visitors: 1 }], + // Not narrowed by the link filter, any more than the summary's is. + newestHour: '2026-09-25T00:00:00.000Z', }) }) @@ -424,10 +438,15 @@ describe('GET /api/reports/timeseries', () => { expect(buckets).toHaveLength(100) expect(buckets[0]?.at).toBe('2026-06-16T00:00:00.000Z') expect(buckets[99]?.at).toBe('2026-09-23T00:00:00.000Z') - // Every fixture click is on the 24th, which is the instant this window - // ends at, so every one of the hundred buckets is a real zero rather than - // a row that was not returned. + // The window ends where the fixtures start: the earliest click is at 10:15 + // on the 24th and this stops at midnight that morning. So all hundred + // buckets are zeroes the fill put there rather than rows the query left + // out — which is exactly what a hundred quiet days look like too. expect(buckets.reduce((n, b) => n + b.clicks, 0)).toBe(0) + // Which is what the freshness is for: the newest hour the install holds is + // past the end of this window, so these hundred days are quiet and not + // days nothing has been shipped for yet. The whole table, not the window. + expect(r.json().newestHour).toBe('2026-09-25T00:00:00.000Z') }) it.each([ diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index e8d9f7c..4fc8400 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -40,7 +40,11 @@ import { fail } from './http.js' /** The server-side bound on one report query. Shorter than the client's own. */ export const CH_MAX_EXECUTION_SECONDS = 25 -/** Exported because the chart aligns to its own bucket and the log aligns to nothing. */ +/** + * The grain the rollups answer at, and so the alignment the summary counts in. + * The chart aligns to its own bucket size instead. Exported for the suite, + * which calls `parseWindow` directly and has to pass what a route would pass. + */ export const HOUR_MS = 3_600_000 /** @@ -326,11 +330,18 @@ export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): v /** * The chart: one row per bucket, with the empty ones filled in. * - * Filled here rather than by ClickHouse's `WITH FILL`, because the loop that - * fills is then the same loop whose count decided whether this window was - * allowed at all — a response cannot come back longer than the ceiling that - * let it through. The other way round, the ceiling is in this file and the - * filling is in a query, and nothing holds them to each other. + * Filled in this process rather than by ClickHouse's `WITH FILL`. It is not + * one loop doing both jobs: the count is `bucketCount` in the shared + * vocabulary and the fill is the loop below, two expressions that can be + * changed apart. What holds them to each other is that both walk the same + * half-open `[fromMs, toMs)` a `step` at a time from the same aligned bounds, + * and what notices when they stop agreeing is the test at exactly the + * ceiling: it asks for a window that counts 2,000 buckets and asserts 2,000 + * came back. `WITH FILL` would put the filling in a query and leave the + * ceiling here, where no test can hold the two together at all. + * + * Two queries under one slot, and a slot is held for their sum: the buckets, + * and the freshness of the whole install, which is not about this window. * * The bucket count is checked here and not by `parseWindow`, because the two * bounds do not compose: four hundred days is inside the window bound and @@ -375,10 +386,24 @@ export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): v GROUP BY at ORDER BY at`, params: windowParams(w), }) - // Keyed by the same text ClickHouse returned, so nothing has to agree - // about how to format an instant twice. `toString` of a DateTime gives - // nineteen characters and `chTime` gives milliseconds too, so the key - // is the sliced form of ours. + // The same freshness the summary answers with, and a chart needs it + // more than a number does. Every bucket the rollup has no row for is + // drawn as a zero, so a window running past the newest hour the install + // holds is a run of real-looking zeroes with nothing in the response to + // tell "nothing shipped yet" from "nobody clicked" — and a picture that + // cannot say how fresh it is gets read as current. The whole table, not + // this window, which is the question it answers. + const [newest] = await chRows<{ newest: string }>({ + ch, + req, + query: 'SELECT toString(max(hour)) AS newest FROM clicks_hourly', + params: {}, + }) + // Two formatters produce this key and the slice is what makes them + // agree: ClickHouse's `toString` of a DateTime is nineteen characters, + // `chTime` adds milliseconds. Checked against a running store rather + // than assumed, because a key that never matched would draw every + // bucket as a zero and every assertion about the length would pass. const found = new Map(rows.map((row) => [row.at, row])) const out: { at: string; clicks: number; visitors: number }[] = [] // `at < w.toMs`, matching the half-open clause the query ran with: the @@ -397,6 +422,7 @@ export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): v link: w.linkId, bucket: q.bucket, buckets: out, + newestHour: newestHourOrNull(newest?.newest), } }, ) From e91b6bae29ab02f090a553ccfbd473f3117ec2db Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 15:26:15 +0000 Subject: [PATCH 28/65] admin: the top of a dimension over a window --- packages/admin/src/reports.test.ts | 284 ++++++++++++++++++++++++++++- packages/admin/src/reports.ts | 120 +++++++++++- 2 files changed, 400 insertions(+), 4 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 6a8ab93..79773de 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -64,12 +64,30 @@ beforeAll(async () => { await resetDatabases(pool, ch) await insert([ click(), - click({ click_id: '01920000-0000-7000-8000-00000000000b', visitor_id: 'v2', country: 'US' }), + // Another visitor, another country, and the second value of every open + // dimension a breakdown reads: a fixture in which `device`, `os`, `browser` + // and `referrer` each held one value could not tell a rollup that wrote the + // real column from one that wrote a constant, and could not tell a + // breakdown that read the dimension it was asked for from one that read + // some other. + click({ + click_id: '01920000-0000-7000-8000-00000000000b', + visitor_id: 'v2', + country: 'US', + device: 'tablet', + os: 'android', + browser: 'safari', + referrer: 'https://news.example.com/other', + }), // The same visitor as the first, on another link: one visitor, two clicks. click({ click_id: '01920000-0000-7000-8000-00000000000c', link_id: LINK_B, path: '/b' }), // A bot, blocked, in the same hour, by the same visitor as the first. It // reached no target, so its target id is empty — which is also the only // empty dimension value in these fixtures. + // + // Its device is a third value nobody else has, which is what gives the + // device breakdown two values on one click each: the order of those two is + // the tie-break and nothing else. click({ click_id: '01920000-0000-7000-8000-00000000000d', traffic_class: 'bot', @@ -80,6 +98,10 @@ beforeAll(async () => { signals: ['ua_bot'], destination: '', target_id: '', + device: 'mobile', + os: 'android', + browser: 'safari', + referrer: 'https://news.example.com/other', }), // An hour later, and outside the narrow window one test asks for. The same // visitor as the first, which is what makes a day's visitor count something @@ -474,6 +496,10 @@ describe('GET /api/reports/timeseries', () => { // for a name nobody has, the alignment becomes NaN, and the caller is told // their `to` is not after their `from` — about a field they got right. So what // pins the enum is the field the refusal names. + // + // The breakdown's own enum needs no such test: a dimension nobody has is a + // 400 from the schema and, as a plain string, a query that reads no rows and + // answers 200. it('names the bucket when the bucket is one nobody has', async () => { const r = await app.inject({ method: 'GET', @@ -514,6 +540,259 @@ describe('GET /api/reports/timeseries', () => { }) }) +describe('GET /api/reports/breakdown', () => { + const url = (extra: string) => `/api/reports/breakdown?${WINDOW}&${extra}` + + const breakdown = (extra: string, on: FastifyInstance = app) => + on.inject({ method: 'GET', url: url(extra), headers: read(cookie) }) + + /** The window every whole-body assertion below echoes back. */ + const COUNTED = { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z' } + + // Whole, and this is one of the four bodies asserted whole: `link`, + // `dimension` and `truncated` are each echoed or derived, and each one is + // asserted somewhere off the value it carries here — the link below, the + // keyed dimension below that, and the cut list further down. A field only + // ever seen at its least interesting value can be replaced by a constant. + // + // The click half an hour past the end of this window is what pins the + // clause's upper bound here: counted, DE would be five clicks by two + // visitors rather than four by one. + it('breaks a window down by country, most clicks first', async () => { + const r = await breakdown('dimension=country') + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: COUNTED, + link: null, + dimension: 'country', + truncated: false, + rows: [ + // Four clicks by one visitor, which is the number no arithmetic over + // the rows gives: the four are three rollup rows — two hours and two + // links — and adding their visitor counts gives three. + { value: 'DE', clicks: 4, visitors: 1 }, + { value: 'US', clicks: 1, visitors: 1 }, + ], + }) + }) + + // The other table, and whole for the `dimension` echo: a constant `'country'` + // there answers the test above correctly and this one wrongly. + it('breaks it down by a dimension that keys the hourly rollup instead', async () => { + const r = await breakdown('dimension=class') + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: COUNTED, + link: null, + dimension: 'class', + truncated: false, + rows: [ + // Two visitors over four clicks that are three rollup rows of one, two + // and one visitor: summed they are four, merged they are two. + { value: 'human', clicks: 4, visitors: 2 }, + { value: 'bot', clicks: 1, visitors: 1 }, + ], + }) + }) + + it.each<[string, { value: string; clicks: number; visitors: number }[]]>([ + [ + 'action', + [ + { value: '', clicks: 4, visitors: 2 }, + { value: 'block', clicks: 1, visitors: 1 }, + ], + ], + [ + 'outcome', + [ + { value: 'target', clicks: 4, visitors: 2 }, + { value: 'blocked', clicks: 1, visitors: 1 }, + ], + ], + [ + 'os', + [ + { value: 'windows', clicks: 3, visitors: 1 }, + { value: 'android', clicks: 2, visitors: 2 }, + ], + ], + [ + 'browser', + [ + { value: 'chrome', clicks: 3, visitors: 1 }, + { value: 'safari', clicks: 2, visitors: 2 }, + ], + ], + [ + 'referrer', + [ + { value: 'blog.example.com', clicks: 3, visitors: 1 }, + { value: 'news.example.com', clicks: 2, visitors: 2 }, + ], + ], + ])('answers %s from whichever table holds it', async (dimension, rows) => { + const r = await breakdown(`dimension=${dimension}`) + expect(r.statusCode).toBe(200) + expect(r.json().rows).toEqual(rows) + }) + + // The tie-break, which is the only thing that decides the last two rows here: + // one click each, so ordered by clicks alone they come back in whatever order + // the parts were read in, and a cut list would be arbitrary. + it('breaks a tie between two values on the value, ascending', async () => { + const r = await breakdown('dimension=device') + expect(r.statusCode).toBe(200) + expect(r.json().rows).toEqual([ + { value: 'desktop', clicks: 3, visitors: 1 }, + { value: 'mobile', clicks: 1, visitors: 1 }, + { value: 'tablet', clicks: 1, visitors: 1 }, + ]) + }) + + // An empty value is a real answer — a click that reached no target — and is + // shown as one rather than dropped, so the rows add up to the total beside + // them. + it('keeps an empty value rather than dropping it', async () => { + const r = await breakdown('dimension=target') + expect(r.json().rows).toEqual([ + { value: '00000000-0000-4000-8000-0000000000b1', clicks: 4, visitors: 2 }, + { value: '', clicks: 1, visitors: 1 }, + ]) + }) + + // Both ends of the shared clause in one window, and the numbers rather than + // the length: the hour this starts at holds a click, so `>= from` written as + // `> from` empties the list, and the hour it ends at holds another by its own + // visitor, so `< to` written as `<= to` doubles both numbers. The chart + // cannot see the upper bound at all, so an endpoint that reuses the clause + // pins it or nothing does. + it('starts at the hour it names and stops before the hour it ends at', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/breakdown?from=2026-09-24T11:00:00.000Z&to=2026-09-25T00:00:00.000Z&dimension=country', + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.json().rows).toEqual([{ value: 'DE', clicks: 1, visitors: 1 }]) + }) + + it('counts whole hours and says which hours it counted', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/reports/breakdown?from=2026-09-24T10:30:00.000Z&to=2026-09-24T11:10:00.000Z&dimension=country', + headers: read(cookie), + }) + // Asked for 10:30 to 11:10; counted 10:00 to 12:00, which is both hours. + expect(r.json().window).toEqual({ + from: '2026-09-24T10:00:00.000Z', + to: '2026-09-24T12:00:00.000Z', + }) + // Unaligned, the four clicks at 10:15 fall outside their own window and + // only the one at 11:30 is counted, so these numbers are the alignment. + expect(r.json().rows).toEqual([ + { value: 'DE', clicks: 4, visitors: 1 }, + { value: 'US', clicks: 1, visitors: 1 }, + ]) + }) + + // Whole, for the `link` echo: the bodies above carry `link: null`, which a + // constant would answer too. + it('counts one link when asked for one', async () => { + const r = await breakdown(`dimension=country&link=${LINK_B}`) + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: COUNTED, + link: LINK_B, + dimension: 'country', + truncated: false, + rows: [{ value: 'DE', clicks: 1, visitors: 1 }], + }) + }) + + // Whole, for `truncated`: it is false in every other body asserted whole. + it('cuts the list at the limit asked for and says it was cut', async () => { + const r = await breakdown('dimension=country&limit=1') + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: COUNTED, + link: null, + dimension: 'country', + truncated: true, + rows: [{ value: 'DE', clicks: 4, visitors: 1 }], + }) + }) + + // Away from one, where a cut and an off-by-one in either direction look + // different: three values, two asked for. The row past the limit is read — + // that is how `truncated` is known without a second query — and not shown. + it('cuts a longer list at the limit too, and shows neither more nor fewer', async () => { + const r = await breakdown('dimension=device&limit=2') + expect(r.json().truncated).toBe(true) + expect(r.json().rows).toEqual([ + { value: 'desktop', clicks: 3, visitors: 1 }, + { value: 'mobile', clicks: 1, visitors: 1 }, + ]) + }) + + it('does not say it was cut when it was not', async () => { + const r = await breakdown('dimension=country&limit=2') + expect(r.json().truncated).toBe(false) + expect(r.json().rows).toHaveLength(2) + }) + + // The ceiling itself, from both sides. Written out rather than computed from + // MAX_BREAKDOWN_ROWS: a bound derived from the constant it is testing moves + // when the constant does and goes on passing. Without the 500 the ceiling + // could be one row tighter than the number it promises and nothing would say + // so; without the 501 it could be absent. + it('takes the most rows it will return, and refuses one more', async () => { + const at = await breakdown('dimension=country&limit=500') + expect(at.statusCode).toBe(200) + expect(at.json().truncated).toBe(false) + const over = await breakdown('dimension=country&limit=501') + expect(over.statusCode).toBe(400) + expect(over.json().error).toBe('invalid_query') + }) + + it.each([ + ['a dimension nobody has', 'dimension=asn'], + ['no dimension at all', 'limit=10'], + ['a limit of zero', 'dimension=country&limit=0'], + ['a limit past the ceiling', 'dimension=country&limit=501'], + ['a limit that is not a number', 'dimension=country&limit=all'], + ['a field nobody knows', 'dimension=country&order=value'], + ])('refuses %s', async (_label, extra) => { + const r = await breakdown(extra) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_query') + }) + + // Its own, because the credential is asked for route by route with no hook + // over all of them: a route that forgot to ask would read every click the + // install has with the other two routes' 401 tests still green. + it('needs a credential', async () => { + const r = await app.inject({ + method: 'GET', + url: url('dimension=country'), + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + }) + + it('refuses when every report slot is taken', async () => { + const other = testApp(pool, clock, { ch, reportGate: new ConcurrencyGate(0) }) + try { + const r = await breakdown('dimension=country', other) + expect(r.statusCode).toBe(429) + expect(r.json().error).toBe('too_many_reports') + } finally { + await other.close() + } + }) +}) + // The route cannot reach this: `from` is a datetime in the schema, so an // unreadable date is a 400 before the parser sees it. The parser is exported // and the chart and the log will call it too, so the bound lives in the @@ -671,11 +950,12 @@ describe('when ClickHouse is not there', () => { // and the 401 tests pin that each route asks — not where it asks. Moved below // the query, an unauthenticated request takes a report slot and scans the // window it chose before being told it was never allowed to ask, which is the - // scan the credential is there to stop. Both routes, because each one asks + // scan the credential is there to stop. Every route, because each one asks // for its own and a rule one route knows is not inherited by its neighbour. it.each([ ['the summary', `/api/reports/summary?${WINDOW}`], ['the chart', `/api/reports/timeseries?${WINDOW}&bucket=hour`], + ['a breakdown', `/api/reports/breakdown?${WINDOW}&dimension=country`], ])('refuses %s without a credential before reading anything', async (_label, url) => { await onDeadWithLog(async (on, lines) => { const r = await on.inject({ method: 'GET', url, headers: { host: ADMIN_HOST } }) diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 4fc8400..9b59e81 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -23,12 +23,15 @@ import { BUCKET_MS, type ConcurrencyGate, + type KeyedDimension, MAX_REPORT_BUCKETS, MAX_REPORT_WINDOW_DAYS, MAX_REPORT_WINDOW_MS, REPORT_BUCKETS, + REPORT_DIMENSIONS, type ReportBucket, bucketCount, + isKeyedDimension, } from '@clickmonk/core' import type { ClickHouseClient } from '@clickmonk/db' import type { FastifyInstance, FastifyRequest } from 'fastify' @@ -126,8 +129,16 @@ export function parseWindow( } } -/** A parsed query, or a 400 that names the field. Query strings, not bodies. */ -export function readQuery(schema: z.ZodType, query: unknown): T { +/** + * A parsed query, or a 400 that names the field. Query strings, not bodies. + * + * `T` is the schema's *output*, which is why the third parameter is `unknown` + * rather than left to default to `T` as well: a schema with a `.default()` has + * an input in which that field is optional, and inferring `T` from both ends at + * once makes every defaulted field possibly-undefined in the handler that just + * asked for it to be filled in. + */ +export function readQuery(schema: z.ZodType, query: unknown): T { const r = schema.safeParse(query ?? {}) if (r.success) return r.data const issues = r.error.issues @@ -244,6 +255,30 @@ const BUCKET_EXPR: Record = { const TimeseriesQuery = z.object({ ...WINDOW_FIELDS, bucket: z.enum(REPORT_BUCKETS) }).strict() +/** Values in one breakdown, and the ceiling a caller may ask for. */ +export const DEFAULT_BREAKDOWN_ROWS = 100 +export const MAX_BREAKDOWN_ROWS = 500 + +/** + * The column each keyed dimension is stored under. A `Record` over the union + * rather than a lookup with a fallback: adding a dimension to the shared + * vocabulary without deciding where it comes from is then a type error here, + * not a breakdown that quietly returns nothing. + */ +const KEYED_COLUMN: Record = { + class: 'traffic_class', + action: 'action', + outcome: 'outcome', +} + +const BreakdownQuery = z + .object({ + ...WINDOW_FIELDS, + dimension: z.enum(REPORT_DIMENSIONS), + limit: z.coerce.number().int().min(1).max(MAX_BREAKDOWN_ROWS).default(DEFAULT_BREAKDOWN_ROWS), + }) + .strict() + export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): void { /** * The numbers at the top of a dashboard: how many clicks, how many people, @@ -427,4 +462,85 @@ export function registerReportRoutes(app: FastifyInstance, ctx: AdminContext): v }, ) }) + + /** + * The top values of one dimension over the window. + * + * Two tables answer it. The six open dimensions are rows of the per-dimension + * rollup, keyed by the dimension name; the three closed ones are keys of the + * hourly rollup, so they are read from there and grouped by their own column. + * Nothing in the query text comes from the request: the dimension chooses + * between two statements written here, and the only interpolated value is a + * row count the schema has already bounded to 1–500. + * + * Ordered by clicks descending and then by value ascending, because a tie + * broken by nothing at all is broken by whatever order the parts were read + * in — which makes a cut list arbitrary and a test pass or fail depending on + * the machine. The cut itself is one row past the limit: asking for one more + * than will be shown is how `truncated` can be true without a second query. + * + * No freshness here, unlike the summary and the chart. A breakdown is read + * beside one of those, which is where "nothing has shipped yet" is already + * said; a third copy of that query would be a third reader of the same text + * for a question this response does not raise. An empty list here is the + * window being empty, and what tells the operator why is the number above it. + * + * The per-dimension rollup also carries the domain a click was served on, and + * nothing here reads it. That is deliberate: a per-domain breakdown is the + * next thing an operator with several domains asks for, and adding a key + * column to an aggregating rollup afterwards is a migration and a backfill of + * every row already in it. + */ + app.get('/api/reports/breakdown', async (req) => { + requireCredential(req) + const q = readQuery(BreakdownQuery, req.query) + const w = parseWindow(q, { alignMs: HOUR_MS }) + const ch = requireCh(ctx, req) + return withSlot( + ctx.reportGate, + { code: 'too_many_reports', message: 'too many reports at once; try again' }, + async () => { + // Hoisted to a local const before the guard runs. `isKeyedDimension` + // narrows a local; it does not narrow a property of the parsed query, + // so `KEYED_COLUMN[q.dimension]` behind the same condition is an + // implicit-any index and does not compile. + const dimension = q.dimension + const keyed = isKeyedDimension(dimension) + const column = keyed ? KEYED_COLUMN[dimension] : 'value' + const table = keyed ? 'clicks_hourly' : 'clicks_hourly_dim' + const where = keyed + ? windowClause(w) + : `${windowClause(w)} AND dimension = {dimension:String}` + // The alias is `value` on both statements, and on the per-dimension + // rollup it is the name of the column it aliases. That is the one case + // where an alias shadowing a column is safe: it means what the column + // means. Nothing in the `WHERE` reads it — the clause names `hour`, + // `link_id` and `dimension` — so the trap that turns a shadowed column + // into an illegal-type error is not reachable here. + const rows = await chRows<{ value: string; clicks: string; visitors: string }>({ + ch, + req, + query: `SELECT ${column} AS value, + uniqExactMerge(clicks_state) AS clicks, + uniqExactMerge(visitors_state) AS visitors + FROM ${table} WHERE ${where} + GROUP BY value + ORDER BY clicks DESC, value ASC + LIMIT ${q.limit + 1}`, + params: { ...windowParams(w), ...(keyed ? {} : { dimension }) }, + }) + return { + window: { from: new Date(w.fromMs).toISOString(), to: new Date(w.toMs).toISOString() }, + link: w.linkId, + dimension, + truncated: rows.length > q.limit, + rows: rows.slice(0, q.limit).map((row) => ({ + value: row.value, + clicks: count(row.clicks), + visitors: count(row.visitors), + })), + } + }, + ) + }) } From 1a1804dc6f2b6cf36c1a75b1fcc239350db796fe Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 15:38:49 +0000 Subject: [PATCH 29/65] admin: a default nobody could read off the fixture --- packages/admin/src/reports.test.ts | 76 ++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 79773de..852cf36 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -793,6 +793,82 @@ describe('GET /api/reports/breakdown', () => { }) }) +/** + * How many rows a breakdown returns, in a window of its own. + * + * Neither the default nor the ceiling can be read off the fixture above: no + * dimension there holds more than three values, so any default of three or more + * and any ceiling of three or more answers every assertion in it. Both numbers + * need more values than they are, and more values than they are would move every + * count in every other block — so this owns its own window, which is the rule a + * per-window total follows. + * + * Its hour is in another month, so another partition, and it is older than the + * newest hour the install holds, so the freshness the summary and the chart + * assert does not move. No other window in this file reaches it: the hundred-day + * chart starts on 2026-06-16, the two-thousand-bucket one on 2026-07-02, and the + * window that would contain it is refused for being too long. + */ +describe('how many rows a breakdown returns', () => { + const WINDOW_OF_ITS_OWN = 'from=2026-05-10T03:00:00.000Z&to=2026-05-10T04:00:00.000Z' + + /** One host per click, in ascending order, so the tie-break decides the list. */ + const host = (i: number): string => `h${String(i).padStart(3, '0')}.example.com` + + const rows = (query: string) => + app.inject({ + method: 'GET', + url: `/api/reports/breakdown?${WINDOW_OF_ITS_OWN}&dimension=referrer&${query}`, + headers: read(cookie), + }) + + beforeAll(async () => { + // Five hundred and one values, one click each. A hundred and one would pin + // the default and leave the ceiling where it is — read off a fixture too + // small to reach it — so there are enough here to see five hundred rows come + // back and a five hundred and first left behind. + await insert( + Array.from({ length: 501 }, (_, i) => { + const n = String(i + 1).padStart(3, '0') + return click({ + click_id: `01920000-0000-7000-8000-100000000${n}`, + time: '2026-05-10 03:20:00.000', + visitor_id: `m${n}`, + referrer: `https://${host(i + 1)}/post`, + }) + }), + ) + }) + + // A hundred written out, not taken from DEFAULT_BREAKDOWN_ROWS: a bound + // derived from the constant it is testing moves when the constant does and + // goes on passing. Ninety-nine and a hundred and one both fail this. + it('returns a hundred rows when no limit is asked for, and says it cut the rest', async () => { + const r = await rows('') + expect(r.statusCode).toBe(200) + expect(r.json().truncated).toBe(true) + const got = r.json().rows as { value: string; clicks: number; visitors: number }[] + expect(got).toHaveLength(100) + // The top of the ordered list rather than a hundred of the five hundred and + // one: every value has one click, so which hundred come back is the + // tie-break and nothing else. + expect(got[0]).toEqual({ value: 'h001.example.com', clicks: 1, visitors: 1 }) + expect(got[99]).toEqual({ value: 'h100.example.com', clicks: 1, visitors: 1 }) + }) + + // The other half of the ceiling. That 500 is accepted and 501 refused is + // pinned above; this is that five hundred rows actually come back when five + // hundred values exist, which no window with three values in it can say. + it('returns five hundred rows when five hundred are asked for', async () => { + const r = await rows('limit=500') + expect(r.statusCode).toBe(200) + expect(r.json().truncated).toBe(true) + const got = r.json().rows as { value: string; clicks: number; visitors: number }[] + expect(got).toHaveLength(500) + expect(got[499]).toEqual({ value: 'h500.example.com', clicks: 1, visitors: 1 }) + }) +}) + // The route cannot reach this: `from` is a datetime in the schema, so an // unreadable date is a 400 before the parser sees it. The parser is exported // and the chart and the log will call it too, so the bound lives in the From b0be38ad07dc708d27b41695d1ecd54a7001f6e4 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 16:26:35 +0000 Subject: [PATCH 30/65] admin: an integer limit, and a credential asked before the queue --- packages/admin/src/reports.test.ts | 84 +++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 852cf36..462923c 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -4,6 +4,7 @@ import { type ClickHouseClient, createChClient } from '@clickmonk/db' import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import type { FastifyInstance } from 'fastify' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import type { AdminDeps } from './app.js' import { HttpError } from './http.js' import { HOUR_MS, newestHourOrNull, parseWindow } from './reports.js' import { ADMIN_HOST, clockFrom, read, signedIn, testApp } from './testing.js' @@ -64,12 +65,13 @@ beforeAll(async () => { await resetDatabases(pool, ch) await insert([ click(), - // Another visitor, another country, and the second value of every open - // dimension a breakdown reads: a fixture in which `device`, `os`, `browser` - // and `referrer` each held one value could not tell a rollup that wrote the - // real column from one that wrote a constant, and could not tell a - // breakdown that read the dimension it was asked for from one that read - // some other. + // Another visitor, another country, and a second value for five of the six + // open dimensions a breakdown reads — every one but the target, which gets + // its second value from the blocked click below, the one that reached none. + // A fixture in which `device`, `os`, `browser` and `referrer` each held one + // value could not tell a rollup that wrote the real column from one that + // wrote a constant, and could not tell a breakdown that read the dimension + // it was asked for from one that read some other. click({ click_id: '01920000-0000-7000-8000-00000000000b', visitor_id: 'v2', @@ -551,9 +553,10 @@ describe('GET /api/reports/breakdown', () => { // Whole, and this is one of the four bodies asserted whole: `link`, // `dimension` and `truncated` are each echoed or derived, and each one is - // asserted somewhere off the value it carries here — the link below, the - // keyed dimension below that, and the cut list further down. A field only - // ever seen at its least interesting value can be replaced by a constant. + // asserted somewhere off the value it carries here — the keyed dimension + // immediately below, the link further down, and the cut list after that. A + // field only ever seen at its least interesting value can be replaced by a + // constant. // // The click half an hour past the end of this window is what pins the // clause's upper bound here: counted, DE would be five clicks by two @@ -664,9 +667,10 @@ describe('GET /api/reports/breakdown', () => { // Both ends of the shared clause in one window, and the numbers rather than // the length: the hour this starts at holds a click, so `>= from` written as // `> from` empties the list, and the hour it ends at holds another by its own - // visitor, so `< to` written as `<= to` doubles both numbers. The chart - // cannot see the upper bound at all, so an endpoint that reuses the clause - // pins it or nothing does. + // visitor, so `< to` written as `<= to` doubles both numbers. The summary + // catches an inclusive upper bound too — the chart is the one that cannot see + // it — so what this adds is that the bound is pinned in this endpoint's own + // window rather than inherited from another route's fixture. it('starts at the hour it names and stops before the hour it ends at', async () => { const r = await app.inject({ method: 'GET', @@ -741,11 +745,13 @@ describe('GET /api/reports/breakdown', () => { expect(r.json().rows).toHaveLength(2) }) - // The ceiling itself, from both sides. Written out rather than computed from - // MAX_BREAKDOWN_ROWS: a bound derived from the constant it is testing moves - // when the constant does and goes on passing. Without the 500 the ceiling - // could be one row tighter than the number it promises and nothing would say - // so; without the 501 it could be absent. + // The ceiling itself, from both sides: the 500 is what says it is not one row + // tighter than the number it promises, and the 501 that it is there at all. + // Both written out rather than computed from MAX_BREAKDOWN_ROWS, since a bound + // derived from the constant it is testing moves when the constant does and + // goes on passing. What this window cannot say is that five hundred rows come + // back when five hundred values exist — three values is all it holds — which + // is what the block at the end of the file is for. it('takes the most rows it will return, and refuses one more', async () => { const at = await breakdown('dimension=country&limit=500') expect(at.statusCode).toBe(200) @@ -761,6 +767,13 @@ describe('GET /api/reports/breakdown', () => { ['a limit of zero', 'dimension=country&limit=0'], ['a limit past the ceiling', 'dimension=country&limit=501'], ['a limit that is not a number', 'dimension=country&limit=all'], + // Its own case, and the one the status code matters for: `all` does not + // coerce to a number at all, but 2.5 does, and without `.int()` it reaches + // ClickHouse as `LIMIT 3.5` and comes back as a 503 saying reporting is + // unavailable on this install. That is the failure the link's shape was + // moved into the parser to stop — a caller's own bad field reported to them + // as an outage — so what this pins is the 400 and not the refusal. + ['a limit that is not a whole number', 'dimension=country&limit=2.5'], ['a field nobody knows', 'dimension=country&order=value'], ])('refuses %s', async (_label, extra) => { const r = await breakdown(extra) @@ -973,13 +986,21 @@ describe('when ClickHouse is not there', () => { * query failure would have written is asserted absent as well, because a * guard moved after the query would otherwise leave only the status to tell * the two apart. + * + * `reportGate` is for the guards that have to come before the queue as well as + * before the query. A full gate and an unreachable store together give a + * misplaced guard two different wrong answers — 429 from inside the gate, 503 + * from after the query — and one right one, so the caller's own status pins the + * position rather than merely the presence. */ const onDeadWithLog = async ( fn: (on: FastifyInstance, lines: string[]) => Promise, + extra: Partial = {}, ): Promise => { const lines: string[] = [] const on = testApp(pool, clock, { ch: dead, + ...extra, log: { level: 'error', stream: { @@ -1032,14 +1053,27 @@ describe('when ClickHouse is not there', () => { ['the summary', `/api/reports/summary?${WINDOW}`], ['the chart', `/api/reports/timeseries?${WINDOW}&bucket=hour`], ['a breakdown', `/api/reports/breakdown?${WINDOW}&dimension=country`], - ])('refuses %s without a credential before reading anything', async (_label, url) => { - await onDeadWithLog(async (on, lines) => { - const r = await on.inject({ method: 'GET', url, headers: { host: ADMIN_HOST } }) - expect(r.statusCode).toBe(401) - expect(r.json().error).toBe('unauthenticated') - expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) - }) - }) + ])( + 'refuses %s without a credential before taking a slot or reading anything', + async (_label, url) => { + // The gate is full as well as the store unreachable, and that is the half + // the 401 tests could not see. Asked for after the query, the credential + // lets an unauthenticated request scan the window it chose — a 503 here. + // Asked for inside the gate but before the query, it lets that request + // take a report slot, so a flood of them answers the operator 429 without + // a row being read — and every test in this file stayed green on that one + // until this gate was put here. Only the position answers 401. + await onDeadWithLog( + async (on, lines) => { + const r = await on.inject({ method: 'GET', url, headers: { host: ADMIN_HOST } }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + }, + { reportGate: new ConcurrencyGate(0) }, + ) + }, + ) // The same claim for the chart's own ceiling, and it needs making separately: // a hundred days is well inside the four hundred the window bound allows, so From 69d0975b36e5b780114c131446bdf977ae918727 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 16:37:28 +0000 Subject: [PATCH 31/65] admin: the click log, with the network and not the address --- packages/admin/src/app.ts | 2 + packages/admin/src/clicks.test.ts | 548 ++++++++++++++++++++++++++ packages/admin/src/clicks.ts | 270 +++++++++++++ packages/admin/src/ip-hygiene.test.ts | 165 ++++++++ packages/core/src/click-record.ts | 14 +- 5 files changed, 997 insertions(+), 2 deletions(-) create mode 100644 packages/admin/src/clicks.test.ts create mode 100644 packages/admin/src/clicks.ts create mode 100644 packages/admin/src/ip-hygiene.test.ts diff --git a/packages/admin/src/app.ts b/packages/admin/src/app.ts index b247285..5a4e186 100644 --- a/packages/admin/src/app.ts +++ b/packages/admin/src/app.ts @@ -8,6 +8,7 @@ import Fastify, { type FastifyServerOptions, } from 'fastify' import { type Credential, authenticate, checkCsrf, hasBearer } from './auth.js' +import { registerClickRoutes } from './clicks.js' import { registerDomainRoutes } from './domains.js' import { HttpError, MAX_BODY_BYTES, securityHeaders } from './http.js' import { registerKeyRoutes } from './keys.js' @@ -274,6 +275,7 @@ export function buildAdminApp( registerLinkRoutes(app, ctx) registerSettingsRoutes(app, ctx) registerReportRoutes(app, ctx) + registerClickRoutes(app, ctx) return app } diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts new file mode 100644 index 0000000..d879fa8 --- /dev/null +++ b/packages/admin/src/clicks.test.ts @@ -0,0 +1,548 @@ +import { ClickHouseLogLevel } from '@clickhouse/client' +import { ConcurrencyGate } from '@clickmonk/core' +import { type ClickHouseClient, createChClient } from '@clickmonk/db' +import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' +import type { FastifyInstance } from 'fastify' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { ADMIN_HOST, clockFrom, read, signedIn, testApp } from './testing.js' + +const pool = testPg() +const ch = testCh() +const clock = clockFrom(new Date('2026-09-24T12:00:00.000Z')) +let app: FastifyInstance +let cookie = '' + +const LINK_A = '00000000-0000-4000-8000-0000000000a1' +const LINK_B = '00000000-0000-4000-8000-0000000000a2' +const DOMAIN = '00000000-0000-4000-8000-00000000000d' +const TARGET = '00000000-0000-4000-8000-0000000000b1' + +const click = (over: Record = {}): Record => ({ + click_id: '01920000-0000-7000-8000-000000000001', + time: '2026-09-24 10:00:00.000', + host: 'go.example.test', + path: '/a', + domain_id: DOMAIN, + link_id: LINK_A, + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/?c=01920000-0000-7000-8000-000000000001', + target_id: TARGET, + visitor_id: 'v1', + returning: 0, + country: 'DE', + region: '', + city: '', + geo_source: 'dbip', + device: 'desktop', + user_agent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/130', + referrer: 'https://blog.example.com/post', + ip: '198.51.100.77', + cap_unchecked: 0, + traffic_class: 'human', + signals: [], + action: '', + os: 'windows', + browser: 'chrome', + asn: 64500, + ...over, +}) + +const insert = (values: Record[]) => + ch.insert({ table: 'clicks', values, format: 'JSONEachRow' }) + +const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' + +/** The day before, which nothing above reads: see the block at the end of the file. */ +const DAY_BEFORE = 'from=2026-09-23T00:00:00.000Z&to=2026-09-24T00:00:00.000Z' + +beforeAll(async () => { + await resetDatabases(pool, ch) + await insert([ + click(), + click({ + click_id: '01920000-0000-7000-8000-000000000002', + time: '2026-09-24 10:01:00.000', + link_id: LINK_B, + path: '/b', + country: 'US', + visitor_id: 'v2', + returning: 1, + ip: '2001:db8:1234:5678:9abc:def0:1234:5678', + referrer: '', + asn: 0, + }), + click({ + click_id: '01920000-0000-7000-8000-000000000003', + time: '2026-09-24 10:02:00.000', + traffic_class: 'bot', + action: 'block', + outcome: 'blocked', + step: 'classify', + status: 403, + destination: '', + target_id: '', + country: '', + geo_source: '', + signals: ['ua_bot', 'head'], + ip: '', + cap_unchecked: 1, + }), + // A day earlier, so outside every window above, and recorded through a + // proxy that spelled the address as a bracketed literal with a port. Its + // own window, because a fifth row inside the day would move every count and + // every page boundary the block above asserts. + click({ + click_id: '01920000-0000-7000-8000-000000000004', + time: '2026-09-23 10:00:00.000', + ip: '[2001:db8:1234:5678:9abc:def0:1234:5678]:443', + }), + ]) +}) + +beforeEach(async () => { + app = testApp(pool, clock, { ch }) + cookie = await signedIn(app, pool, clock.now()) +}) + +afterEach(async () => { + await app.close() + await pool.query('TRUNCATE admin_account, admin_recovery_codes, sessions, api_keys') +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +describe('GET /api/clicks', () => { + it('shows a click whole, newest first, with the address as a network', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + const body = r.json() + expect(body.window).toEqual({ + from: '2026-09-24T00:00:00.000Z', + to: '2026-09-25T00:00:00.000Z', + }) + expect(body.nextCursor).toBeNull() + expect(body.clicks).toHaveLength(3) + expect(body.clicks[2]).toEqual({ + clickId: '01920000-0000-7000-8000-000000000001', + at: '2026-09-24T10:00:00.000Z', + host: 'go.example.test', + path: '/a', + domainId: DOMAIN, + linkId: LINK_A, + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/?c=01920000-0000-7000-8000-000000000001', + targetId: TARGET, + visitorId: 'v1', + returning: false, + country: 'DE', + region: null, + city: null, + geoSource: 'dbip', + device: 'desktop', + os: 'windows', + browser: 'chrome', + asn: 64500, + class: 'human', + signals: [], + action: null, + referrer: 'https://blog.example.com/post', + userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/130', + network: '198.51.100.0/24', + capUnchecked: false, + }) + }) + + /** + * The worker deletes a spool segment only after ClickHouse accepts it, so a + * crash in between ships the same segment again: two rows identical to the + * byte, until a merge collapses them. + * + * The duplicate is written **here and in its own INSERT**, not with the + * fixture. One INSERT is one part and the engine collapses equal sort keys as + * it writes a part, so a duplicate inside the fixture's array is gone before + * any test runs — and `FINAL` then has nothing to do, which is how this test + * passed with `FINAL` dropped. A second INSERT is a second part, and the raw + * count asserted below is what says the duplicate was still there when the log + * was read: a background merge may collapse it at any moment, and that count + * failing is the right answer to a fixture this test can no longer see. + */ + it('shows a click that was shipped twice once', async () => { + await insert([click()]) + const rs = await ch.query({ query: 'SELECT count() AS n FROM clicks', format: 'JSONEachRow' }) + // Four fixture rows — three in this window and the day before's — plus the + // copy just written. + expect(await rs.json()).toEqual([{ n: '5' }]) + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + const ids = r.json().clicks.map((c: { clickId: string }) => c.clickId) + expect(ids).toEqual([ + '01920000-0000-7000-8000-000000000003', + '01920000-0000-7000-8000-000000000002', + '01920000-0000-7000-8000-000000000001', + ]) + }) + + it('shows an IPv6 client as its /64 and an empty one as nothing', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + const clicks = r.json().clicks as { clickId: string; network: string | null }[] + const byId = new Map(clicks.map((c) => [c.clickId, c])) + expect(byId.get('01920000-0000-7000-8000-000000000002')?.network).toBe( + '2001:db8:1234:5678::/64', + ) + // Blanked by the retention pass, or never recorded: null, not an empty + // string a reader might print as an address. + expect(byId.get('01920000-0000-7000-8000-000000000003')?.network).toBeNull() + }) + + it('turns every empty column into nothing rather than an empty string', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + // The whole object, not a subset: a partial matcher passes against a + // response that also carries a field nobody meant to send — an `ip`, for + // instance — which is the one thing every assertion about this response has + // to be able to see. + expect(r.json().clicks[0]).toEqual({ + clickId: '01920000-0000-7000-8000-000000000003', + at: '2026-09-24T10:02:00.000Z', + host: 'go.example.test', + path: '/a', + domainId: DOMAIN, + linkId: LINK_A, + outcome: 'blocked', + step: 'classify', + status: 403, + destination: null, + targetId: null, + visitorId: 'v1', + returning: false, + country: null, + region: null, + city: null, + geoSource: null, + device: 'desktop', + os: 'windows', + browser: 'chrome', + asn: 64500, + class: 'bot', + signals: ['ua_bot', 'head'], + action: 'block', + referrer: 'https://blog.example.com/post', + userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/130', + network: null, + capUnchecked: true, + }) + // Three fields of the middle click, read one at a time rather than as a + // subset of it: each is a column whose empty or zero value means "not + // known" and must not come back as `0` or `''`. + const second = r.json().clicks[1] + expect(second.asn).toBeNull() + expect(second.referrer).toBeNull() + expect(second.returning).toBe(true) + }) + + /** + * The whole response, with the link filter carrying a link id rather than the + * null every other test here leaves it as. + * + * `link` is echoed from the parsed window, and an echoed field asserted only + * where it holds its least interesting value can be replaced by a constant + * with the rest of the suite green. So this one is asserted whole, at a value + * nothing else in the file sends. + */ + it('echoes the link it was asked for, and nothing else, whole', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&link=${LINK_B}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.json()).toEqual({ + window: { from: '2026-09-24T00:00:00.000Z', to: '2026-09-25T00:00:00.000Z' }, + link: LINK_B, + clicks: [ + { + clickId: '01920000-0000-7000-8000-000000000002', + at: '2026-09-24T10:01:00.000Z', + host: 'go.example.test', + path: '/b', + domainId: DOMAIN, + linkId: LINK_B, + outcome: 'target', + step: 'destination', + status: 302, + // The destination the base row carries: every fixture click here is + // copied from the first one, and only the fields a case is about are + // overridden. + destination: 'https://example.com/?c=01920000-0000-7000-8000-000000000001', + targetId: TARGET, + visitorId: 'v2', + returning: true, + country: 'US', + region: null, + city: null, + geoSource: 'dbip', + device: 'desktop', + os: 'windows', + browser: 'chrome', + asn: null, + class: 'human', + signals: [], + action: null, + referrer: null, + userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/130', + network: '2001:db8:1234:5678::/64', + capUnchecked: false, + }, + ], + nextCursor: null, + }) + }) + + it.each([ + ['link', `link=${LINK_B}`, ['01920000-0000-7000-8000-000000000002']], + ['class', 'class=bot', ['01920000-0000-7000-8000-000000000003']], + ['outcome', 'outcome=blocked', ['01920000-0000-7000-8000-000000000003']], + ['country', 'country=US', ['01920000-0000-7000-8000-000000000002']], + ])('filters by %s', async (_label, filter, expected) => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&${filter}`, + headers: read(cookie), + }) + expect(r.json().clicks.map((c: { clickId: string }) => c.clickId)).toEqual(expected) + }) + + it('takes every filter at once', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&link=${LINK_A}&class=bot&outcome=blocked`, + headers: read(cookie), + }) + expect(r.json().clicks).toHaveLength(1) + }) + + it('uses the window exactly as it was given, to the millisecond', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/clicks?from=2026-09-24T10:01:00.000Z&to=2026-09-24T10:02:00.000Z', + headers: read(cookie), + }) + // Half-open: the click at 10:01 is in, the one at 10:02 is not. + expect(r.json().clicks.map((c: { clickId: string }) => c.clickId)).toEqual([ + '01920000-0000-7000-8000-000000000002', + ]) + expect(r.json().window).toEqual({ + from: '2026-09-24T10:01:00.000Z', + to: '2026-09-24T10:02:00.000Z', + }) + }) + + it('pages with a cursor, and stops without one', async () => { + const first = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&limit=2`, + headers: read(cookie), + }) + expect(first.json().clicks).toHaveLength(2) + const cursor = first.json().nextCursor as string + expect(cursor).toMatch(/^\d+\.[0-9a-f-]{36}$/) + const second = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&limit=2&cursor=${encodeURIComponent(cursor)}`, + headers: read(cookie), + }) + expect(second.json().clicks.map((c: { clickId: string }) => c.clickId)).toEqual([ + '01920000-0000-7000-8000-000000000001', + ]) + expect(second.json().nextCursor).toBeNull() + }) + + it.each([ + ['a cursor that is not one', 'cursor=nonsense'], + ['a cursor with no id', 'cursor=1758708000000.'], + ['a cursor whose id is not an id', 'cursor=1758708000000.not-a-uuid-at-all-not-at-all-no-x'], + ['a class nobody has', 'class=spider'], + ['an outcome nobody has', 'outcome=maybe'], + ['a lower-case country', 'country=de'], + ['a country that is not a country code', 'country=DEU'], + ['a page past the ceiling', 'limit=201'], + ['a page of nothing', 'limit=0'], + // Its own case, and the one whose status code is the point: `2.5` coerces + // to a number where `all` does not, so without `.int()` it is interpolated + // as `LIMIT 3.5`, the store answers an error, and the caller is told + // reporting is unavailable on this install rather than that their own field + // was wrong. + ['a page that is not a whole number', 'limit=2.5'], + ['a field nobody knows', 'ip=198.51.100.77'], + ])('refuses %s', async (_label, extra) => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&${extra}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_query') + }) + + it('refuses a window longer than four hundred days', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/clicks?from=2025-01-01T00:00:00.000Z&to=2026-09-25T00:00:00.000Z', + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('window_too_long') + }) + + it('needs a credential', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + }) + + it('refuses when every report slot is taken', async () => { + const other = testApp(pool, clock, { ch, reportGate: new ConcurrencyGate(0) }) + try { + const r = await other.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(429) + expect(r.json().error).toBe('too_many_reports') + } finally { + await other.close() + } + }) +}) + +/** + * An address the way a proxy may have spelled it, in a window of its own. + * + * `truncateIp` answers null for anything that is not bare address text, a port + * and a bracketed literal included, and the stored column is a plain String the + * record schema bounds only in length. So the address is run through + * `addressOnly` first; without that, a click recorded this way reads as "no + * network" — the same answer as a click whose address the retention pass + * blanked, which is the one thing the two must not share. + */ +describe('GET /api/clicks, an address with brackets and a port', () => { + it('shows it as the network it came from', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${DAY_BEFORE}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect( + r + .json() + .clicks.map((c: { clickId: string; network: string | null }) => [c.clickId, c.network]), + ).toEqual([['01920000-0000-7000-8000-000000000004', '2001:db8:1234:5678::/64']]) + }) +}) + +describe('the log when ClickHouse is not there', () => { + let dead: ClickHouseClient + + beforeEach(() => { + // A port nothing listens on, in the range no service in this repository + // binds. The client does not connect until it is asked to, and its own + // logger is off: the refused connection is what these tests arrange, so + // printing it on every green run teaches a reader to skim the stderr a real + // failure appears in. + dead = createChClient({ + url: 'http://127.0.0.1:1', + username: 'clickmonk', + password: 'clickmonk', + database: 'clickmonk_test', + requestTimeoutMs: 2000, + logLevel: ClickHouseLogLevel.OFF, + }) + }) + + afterEach(async () => { + await dead.close() + }) + + it('answers 503 and names the store, not an internal error', async () => { + const on = testApp(pool, clock, { ch: dead }) + try { + const r = await on.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(503) + expect(r.json().error).toBe('reporting_unavailable') + expect(r.json().message).not.toContain('127.0.0.1') + } finally { + await on.close() + } + }) + + /** + * The credential is asked for before the queue, and this is what pins it + * there rather than merely pinning that it is asked for at all. + * + * Three positions, three answers, and the 401 test above can only see the + * first of them. With the gate full and the store unreachable: asked for + * after the query, an unauthenticated request scans the window it chose and + * gets a 503; asked for inside the gate but before the query, it takes a + * report slot and gets a 429, so a flood of them answers the operator 429 + * with no row read. Only asked for before both is the answer 401 — and the + * log line a failed query writes is asserted absent as well, because the + * status alone cannot tell a refusal from a query that happened to fail. + */ + it('refuses without a credential before taking a slot or reading anything', async () => { + const lines: string[] = [] + const on = testApp(pool, clock, { + ch: dead, + reportGate: new ConcurrencyGate(0), + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + const r = await on.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + } finally { + await on.close() + } + }) +}) diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts new file mode 100644 index 0000000..61db5c1 --- /dev/null +++ b/packages/admin/src/clicks.ts @@ -0,0 +1,270 @@ +/** + * The raw click log. + * + * Everything else in these reports reads an hourly rollup; this reads the + * clicks themselves, which is why it is the only place that has to think about + * two things the rollups solved once. + * + * **FINAL.** A segment the worker shipped twice is two identical rows here + * until a merge collapses them, so a plain SELECT would show one click as two. + * FINAL is the engine's own dedup, keyed exactly as the table is keyed. A + * hand-rolled `LIMIT 1 BY click_id` would be the same rule stated a second + * time, in a second place, where it could come to disagree with the one the + * rollups use. + * + * **The address is a network and the field says so.** A field called `ip` + * holding `198.51.100.0/24` invites its next reader to treat it as an address; + * `network` cannot be misread, and it means no response this service sends has + * a field called `ip` at all — which a test can check across the whole surface + * rather than one route at a time. The truncation itself is `truncateIp`, the + * same function the export calls, and it is never done in SQL: two copies of a + * rule about what an operator may see is one too many. + */ +import { OUTCOMES, TRAFFIC_CLASSES } from '@clickmonk/core' +import { addressOnly, truncateIp } from '@clickmonk/ipdata' +import type { FastifyInstance } from 'fastify' +import { z } from 'zod' +import type { AdminContext } from './app.js' +import { requireCredential } from './auth.js' +import { fail } from './http.js' +import { + type ReportWindow, + WINDOW_FIELDS, + chRows, + parseWindow, + readQuery, + requireCh, + withSlot, +} from './reports.js' + +/** Clicks in one page, and the ceiling a caller may ask for. */ +export const DEFAULT_CLICK_PAGE = 50 +export const MAX_CLICK_PAGE = 200 + +/** + * The filters the product promises: link, class, outcome, country and time. + * Shared with the export, which filters the same rows it would have listed. + */ +export const CLICK_FILTER_FIELDS = { + ...WINDOW_FIELDS, + class: z.enum(TRAFFIC_CLASSES).optional(), + outcome: z.enum(OUTCOMES).optional(), + country: z + .string() + .regex(/^[A-Z]{2}$/, 'must be a two-letter upper-case country code') + .optional(), +} + +const ListQuery = z + .object({ + ...CLICK_FILTER_FIELDS, + // `.int()`, and a refusal test for a fractional one: this number is + // interpolated into `LIMIT`, so without it `limit=2.5` reaches the store as + // `LIMIT 3.5`, ClickHouse answers an error, and the caller's own bad field + // is reported back to them as an install outage. + limit: z.coerce.number().int().min(1).max(MAX_CLICK_PAGE).default(DEFAULT_CLICK_PAGE), + cursor: z.string().max(80).optional(), + }) + .strict() + +/** + * A page boundary: the instant and the id of the last row of the page before. + * Opaque to the caller, and parsed strictly — it reaches a comparison against + * a `DateTime64` and a `UUID`, so anything that is not exactly those two things + * is refused here rather than by ClickHouse. + */ +const CURSOR_RE = /^(\d{1,15})\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/ + +export function parseCursor(value: string): { atMs: number; clickId: string } { + const m = CURSOR_RE.exec(value) + // `return fail(…)`: a bare call does not narrow `m`, because `fail` is a + // const arrow rather than a declaration. With the return, the lines below see + // a match; the casts are for the index signature, which does not know the + // pattern has two groups. + if (!m) return fail(400, 'invalid_query', 'cursor: not a cursor from a previous page') + return { atMs: Number(m[1] as string), clickId: m[2] as string } +} + +/** + * One row as `JSONEachRow` really hands it over. The distinction matters and is + * easy to get wrong: ClickHouse quotes only 64-bit integers, so `at_ms` from + * `toUnixTimestamp64Milli` arrives as a **string** while `status`, `returning`, + * `asn` and `cap_unchecked` arrive as **numbers**, and `signals` as an array. + */ +export interface ClickRow { + click_id: string + at_ms: string + host: string + path: string + domain_id: string + link_id: string + outcome: string + step: string + status: number + destination: string + target_id: string + visitor_id: string + returning: number + country: string + region: string + city: string + geo_source: string + device: string + user_agent: string + referrer: string + ip: string + cap_unchecked: number + traffic_class: string + signals: string[] + action: string + os: string + browser: string + asn: number +} + +/** + * The columns both readers select, in the order the CSV writes them. + * + * `toUnixTimestamp64Milli` rather than the column itself: an instant that + * travels as a number needs no agreement about how either end formats a + * timestamp, and it is also exactly what a cursor carries. + */ +export const CLICK_COLUMNS = `click_id, toUnixTimestamp64Milli(time) AS at_ms, host, path, + domain_id, link_id, outcome, step, status, destination, target_id, visitor_id, returning, + country, region, city, geo_source, device, os, browser, asn, traffic_class, signals, action, + referrer, user_agent, ip, cap_unchecked` + +const none = (s: string): string | null => (s === '' ? null : s) + +/** + * The network a click came from, or nothing. + * + * `addressOnly` first, always. `truncateIp` answers null for anything that is + * not bare address text — a port or a bracketed literal included — and the + * stored column is a plain `String` that the record schema bounds only in + * length, so a click recorded through a proxy that spelled the address + * `[2001:db8::5]:443` would otherwise read as "no network" rather than as the + * /64 it came from. Returning null is the right answer for a value the parser + * genuinely does not recognise and the wrong one for a value it would + * recognise a character later. + */ +const networkOf = (ip: string): string | null => (ip === '' ? null : truncateIp(addressOnly(ip))) + +/** One click over the API. The address is a network; nothing here is an address. */ +export function asClick(r: ClickRow): Record { + return { + clickId: r.click_id, + at: new Date(Number(r.at_ms)).toISOString(), + host: r.host, + path: r.path, + domainId: r.domain_id, + linkId: r.link_id, + outcome: r.outcome, + step: r.step, + status: r.status, + destination: none(r.destination), + targetId: none(r.target_id), + visitorId: r.visitor_id, + returning: r.returning === 1, + country: none(r.country), + // Recorded from day one and empty until city-level lands, which is a + // column that exists and a value that does not. + region: none(r.region), + city: none(r.city), + geoSource: none(r.geo_source), + device: r.device, + os: none(r.os), + browser: none(r.browser), + // Zero is how a click with no known network is stored, and zero is not an + // autonomous system number. + asn: r.asn === 0 ? null : r.asn, + class: none(r.traffic_class), + signals: r.signals, + action: none(r.action), + referrer: none(r.referrer), + userAgent: none(r.user_agent), + // Null for a click whose address was blanked by the retention pass, and + // null for anything the parser does not recognise: never the stored value. + network: networkOf(r.ip), + capUnchecked: r.cap_unchecked === 1, + } +} + +/** + * The WHERE clause and its parameters. Every value is a parameter; what the + * text carries is whether a clause is there at all. + */ +export function clickFilter( + q: { class?: string; outcome?: string; country?: string }, + w: ReportWindow, + cursor: { atMs: number; clickId: string } | null, +): { where: string; params: Record } { + const parts = [`time >= {from:DateTime64(3,'UTC')}`, `time < {to:DateTime64(3,'UTC')}`] + const params: Record = { from: w.from, to: w.to } + if (w.linkId !== null) { + parts.push('link_id = {link:UUID}') + params.link = w.linkId + } + if (q.class !== undefined) { + parts.push('traffic_class = {class:String}') + params.class = q.class + } + if (q.outcome !== undefined) { + parts.push('outcome = {outcome:String}') + params.outcome = q.outcome + } + if (q.country !== undefined) { + parts.push('country = {country:String}') + params.country = q.country + } + if (cursor !== null) { + // One tuple comparison rather than three clauses with an OR: the pair is + // what the order is by, so the pair is what the boundary is on. + parts.push(`(time, click_id) < ({cursorAt:DateTime64(3,'UTC')}, {cursorId:UUID})`) + params.cursorAt = new Date(cursor.atMs).toISOString().replace('T', ' ').replace('Z', '') + params.cursorId = cursor.clickId + } + return { where: parts.join(' AND '), params } +} + +export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): void { + /** + * One page of the log, newest first. + * + * The window is used exactly as it was sent — to the millisecond, half-open — + * unlike a report, which can only answer whole hours. Both say what they + * counted, because the two answers over the same ragged window are allowed to + * differ and an operator comparing them deserves to know why. + */ + app.get('/api/clicks', async (req) => { + requireCredential(req) + const q = readQuery(ListQuery, req.query) + const w = parseWindow(q, { alignMs: null }) + const cursor = q.cursor === undefined ? null : parseCursor(q.cursor) + const ch = requireCh(ctx, req) + return withSlot( + ctx.reportGate, + { code: 'too_many_reports', message: 'too many reports at once; try again' }, + async () => { + const { where, params } = clickFilter(q, w, cursor) + const rows = await chRows({ + ch, + req, + query: `SELECT ${CLICK_COLUMNS} FROM clicks FINAL WHERE ${where} + ORDER BY time DESC, click_id DESC LIMIT ${q.limit + 1}`, + params, + }) + const page = rows.slice(0, q.limit) + const last = page[page.length - 1] + return { + window: { from: new Date(w.fromMs).toISOString(), to: new Date(w.toMs).toISOString() }, + link: w.linkId, + clicks: page.map(asClick), + // Present only when a further page exists, so a caller stops rather + // than asking for ever. + nextCursor: rows.length > q.limit && last ? `${last.at_ms}.${last.click_id}` : null, + } + }, + ) + }) +} diff --git a/packages/admin/src/ip-hygiene.test.ts b/packages/admin/src/ip-hygiene.test.ts new file mode 100644 index 0000000..acde432 --- /dev/null +++ b/packages/admin/src/ip-hygiene.test.ts @@ -0,0 +1,165 @@ +import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' +import type { FastifyInstance } from 'fastify' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { clockFrom, read, signedIn, testApp } from './testing.js' + +/** + * No response this service sends carries a whole address, and no response has + * a field called `ip`. + * + * It is one file rather than an assertion inside each route's suite because the + * rule is about the surface: a route added later that selects `ip` and forgets + * to truncate it is caught here, by a test nobody has to remember to extend, + * as long as its URL is in the list below. + */ +const pool = testPg() +const ch = testCh() +const clock = clockFrom(new Date('2026-09-24T12:00:00.000Z')) +let app: FastifyInstance +let cookie = '' + +/** The whole address every fixture below was recorded with. */ +const HOST_ADDRESS = '198.51.100.77' +const V6_ADDRESS = '2001:db8:1234:5678:9abc:def0:1234:5678' +const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' + +// Every GET this service answers, so that the check is about the surface and +// not about the routes somebody thought of. The CSV export is added to this +// list by the task that adds the export. +const READS = [ + `/api/clicks?${WINDOW}`, + `/api/reports/summary?${WINDOW}`, + `/api/reports/timeseries?${WINDOW}&bucket=hour`, + `/api/reports/breakdown?${WINDOW}&dimension=country`, + '/api/links', + '/api/domains', + '/api/alerts', + '/api/keys', + '/api/settings', + '/api/sessions', + '/api/me', +] + +beforeAll(async () => { + await resetDatabases(pool, ch) + // Two clicks, one per family, each with its whole address stored. + await ch.insert({ + table: 'clicks', + values: [ + { + click_id: '01920000-0000-7000-8000-000000000001', + time: '2026-09-24 10:00:00.000', + host: 'go.example.test', + path: '/a', + domain_id: '00000000-0000-4000-8000-00000000000d', + link_id: '00000000-0000-4000-8000-0000000000a1', + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/', + target_id: '', + visitor_id: 'v1', + returning: 0, + country: 'DE', + region: '', + city: '', + geo_source: 'dbip', + device: 'desktop', + user_agent: 'ua', + referrer: '', + ip: HOST_ADDRESS, + cap_unchecked: 0, + traffic_class: 'human', + signals: [], + action: '', + os: 'windows', + browser: 'chrome', + asn: 64500, + }, + { + click_id: '01920000-0000-7000-8000-000000000002', + time: '2026-09-24 10:01:00.000', + host: 'go.example.test', + path: '/a', + domain_id: '00000000-0000-4000-8000-00000000000d', + link_id: '00000000-0000-4000-8000-0000000000a1', + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/', + target_id: '', + visitor_id: 'v2', + returning: 0, + country: 'DE', + region: '', + city: '', + geo_source: 'dbip', + device: 'desktop', + user_agent: 'ua', + referrer: '', + ip: V6_ADDRESS, + cap_unchecked: 0, + traffic_class: 'human', + signals: [], + action: '', + os: 'windows', + browser: 'chrome', + asn: 64500, + }, + ], + format: 'JSONEachRow', + }) +}) + +beforeEach(async () => { + app = testApp(pool, clock, { ch }) + cookie = await signedIn(app, pool, clock.now()) +}) + +afterEach(async () => { + await app.close() + await pool.query('TRUNCATE admin_account, admin_recovery_codes, sessions, api_keys') +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +describe('no response carries a whole address', () => { + it.each(READS)('%s', async (url) => { + const r = await app.inject({ method: 'GET', url, headers: read(cookie) }) + expect(r.statusCode).toBe(200) + // A `not.toContain` against an empty body passes without reading anything, + // so the body is asserted to be there first: a route that answered `{}` + // would otherwise satisfy both lines below. + expect(r.body.length).toBeGreaterThan(2) + expect(r.body).not.toContain(HOST_ADDRESS) + expect(r.body).not.toContain(V6_ADDRESS) + }) + + it('shows the network instead, in both families', async () => { + const log = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + expect(log.body).toContain('198.51.100.0/24') + expect(log.body).toContain('2001:db8:1234:5678::/64') + }) + + it('has no field called ip anywhere in the log', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}`, + headers: read(cookie), + }) + // Both fixture clicks, so the loop below has something to read: a loop over + // an empty list asserts nothing at all. + expect(r.json().clicks).toHaveLength(2) + for (const click of r.json().clicks as Record[]) { + expect(Object.keys(click)).not.toContain('ip') + expect(Object.keys(click)).toContain('network') + } + }) +}) diff --git a/packages/core/src/click-record.ts b/packages/core/src/click-record.ts index 7d25fa3..2e04f36 100644 --- a/packages/core/src/click-record.ts +++ b/packages/core/src/click-record.ts @@ -46,6 +46,16 @@ const V1_OUTCOMES = [ ] as const const V1_STEPS = ['resolve', 'limits', 'country', 'destination'] as const +/** + * Every outcome and step a click record may name, which is version 3's lists. + * Exported because the click log lets a caller filter by one, and a filter + * validated against a list written out a second time is a filter that silently + * stops matching when a version adds a value. The type equalities at the + * bottom of this file are what hold these to the evaluator's own unions. + */ +export const OUTCOMES = [...V1_OUTCOMES, 'blocked', 'safe', 'password'] as const +export const STEPS = [...V1_STEPS, 'classify', 'password'] as const + /** * One line of the spool as the redirect wrote it before traffic * classification. The worker still reads it: a spool written before an @@ -100,8 +110,8 @@ export const ClickRecordV2Schema = z */ export const ClickRecordV3Schema = ClickRecordV2Schema.extend({ v: z.literal(3), - outcome: z.enum([...V1_OUTCOMES, 'blocked', 'safe', 'password'] as const), - step: z.enum([...V1_STEPS, 'classify', 'password'] as const), + outcome: z.enum(OUTCOMES), + step: z.enum(STEPS), }) /** Every record version the worker ships. */ From a1801a0acb1d2371e53aad6269b5a99c33cf145a Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 16:45:49 +0000 Subject: [PATCH 32/65] admin: say why a merge broke this test --- packages/admin/src/clicks.test.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index d879fa8..fffda68 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -90,9 +90,13 @@ beforeAll(async () => { cap_unchecked: 1, }), // A day earlier, so outside every window above, and recorded through a - // proxy that spelled the address as a bracketed literal with a port. Its - // own window, because a fifth row inside the day would move every count and - // every page boundary the block above asserts. + // proxy that spelled the address as a bracketed literal with a port. The + // redirect writes a canonical bare address, so this is a claim about what + // the column may hold — a plain String bounded only in length, written by + // whatever shipped the record — rather than about what the redirect writes + // today, and that is exactly why it earns a test. Its own window, because a + // fifth row inside the day would move every count and every page boundary + // the block above asserts. click({ click_id: '01920000-0000-7000-8000-000000000004', time: '2026-09-23 10:00:00.000', @@ -180,9 +184,15 @@ describe('GET /api/clicks', () => { it('shows a click that was shipped twice once', async () => { await insert([click()]) const rs = await ch.query({ query: 'SELECT count() AS n FROM clicks', format: 'JSONEachRow' }) - // Four fixture rows — three in this window and the day before's — plus the - // copy just written. - expect(await rs.json()).toEqual([{ n: '5' }]) + // Five rows: four from the fixture — three in this window and the day + // before's — plus the copy just written. The message is the whole point of + // asserting this before reading the log: four rows here means a background + // merge collapsed the pair between the INSERT and this count, which is a + // race in the fixture and not a regression in the query below. + expect( + await rs.json(), + 'a background merge collapsed the re-shipped copy before the log was read, so this test could not see whether FINAL deduplicated anything. That is a race in the fixture, not a failure of the query. If it recurs, give this suite a ClickHouse database of its own; never SYSTEM STOP MERGES, which these shared test databases would hand to every other suite.', + ).toEqual([{ n: '5' }]) const r = await app.inject({ method: 'GET', url: `/api/clicks?${WINDOW}`, From 3fbafd8707ad3c721c3c8bcd05f0609ec8b1b7de Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 17:24:56 +0000 Subject: [PATCH 33/65] admin: a cursor the store can read, and ties a page cannot skip --- packages/admin/src/clicks.test.ts | 235 +++++++++++++++++++++++++----- packages/admin/src/clicks.ts | 39 ++++- 2 files changed, 237 insertions(+), 37 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index fffda68..88fc473 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -15,6 +15,7 @@ let cookie = '' const LINK_A = '00000000-0000-4000-8000-0000000000a1' const LINK_B = '00000000-0000-4000-8000-0000000000a2' const DOMAIN = '00000000-0000-4000-8000-00000000000d' +const DOMAIN_B = '00000000-0000-4000-8000-00000000000e' const TARGET = '00000000-0000-4000-8000-0000000000b1' const click = (over: Record = {}): Record => ({ @@ -49,21 +50,79 @@ const click = (over: Record = {}): Record => ( ...over, }) +/** + * `optimize_on_insert: 0`, so that a click written twice stays two rows. + * + * On by default, the engine collapses equal sort keys as it writes each part, + * which happens *before* any merge: a re-shipped click inside one INSERT would + * be one row by the time the first test ran, and `FINAL` would have nothing to + * deduplicate. With it off the pair is held in one part, with no merge to race + * and nothing set on the server that another suite on these shared databases + * could inherit. + */ const insert = (values: Record[]) => - ch.insert({ table: 'clicks', values, format: 'JSONEachRow' }) + ch.insert({ + table: 'clicks', + values, + format: 'JSONEachRow', + clickhouse_settings: { optimize_on_insert: 0 }, + }) + +/** + * Six clicks on one day, in three instants: three of them share a millisecond, + * two share another, one is alone. Ids ascend with the order they are written + * in, so that the order a page must come back in is the reverse of this list. + * + * A fixture in which no two clicks share an instant cannot see the difference + * between a boundary on the pair the order is by and a boundary on the time + * alone — and the second of those silently drops every row of a tie after the + * first, which is the one thing a log must never do. + */ +const TIED_IDS = [ + '01920000-0000-7000-8000-000000000011', + '01920000-0000-7000-8000-000000000012', + '01920000-0000-7000-8000-000000000013', + '01920000-0000-7000-8000-000000000014', + '01920000-0000-7000-8000-000000000015', + '01920000-0000-7000-8000-000000000016', +] +const TIED_TIMES = [ + '2026-09-22 10:00:00.000', + '2026-09-22 10:00:00.000', + '2026-09-22 10:00:00.000', + '2026-09-22 10:01:00.000', + '2026-09-22 10:01:00.000', + '2026-09-22 10:02:00.000', +] +const tied = TIED_IDS.map((id, i) => click({ click_id: id, time: TIED_TIMES[i] })) + +/** A well-formed id for a cursor whose instant is the thing under test. */ +const CURSOR_ID = '01920000-0000-7000-8000-0000000000ff' const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' -/** The day before, which nothing above reads: see the block at the end of the file. */ +/** Two days nothing above reads: see the two blocks at the end of the file. */ const DAY_BEFORE = 'from=2026-09-23T00:00:00.000Z&to=2026-09-24T00:00:00.000Z' +const TIED_DAY = 'from=2026-09-22T00:00:00.000Z&to=2026-09-23T00:00:00.000Z' beforeAll(async () => { await resetDatabases(pool, ch) await insert([ click(), + // The same click again: one segment, shipped twice, and two rows because of + // the insert setting above. + click(), + // Every field this row overrides is a field the first row also has, and a + // second value is the only thing that can tell a response reading the column + // from a response carrying a constant. Six of them — the host, the domain, + // the device, the OS, the browser and the user agent — had one value across + // the whole fixture, and the user agent is the most identifying thing in a + // click after the visitor id. click({ click_id: '01920000-0000-7000-8000-000000000002', time: '2026-09-24 10:01:00.000', + host: 'links.example.test', + domain_id: DOMAIN_B, link_id: LINK_B, path: '/b', country: 'US', @@ -72,6 +131,10 @@ beforeAll(async () => { ip: '2001:db8:1234:5678:9abc:def0:1234:5678', referrer: '', asn: 0, + device: 'ios', + os: 'ios', + browser: 'safari', + user_agent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_0) Safari/605', }), click({ click_id: '01920000-0000-7000-8000-000000000003', @@ -89,19 +152,21 @@ beforeAll(async () => { ip: '', cap_unchecked: 1, }), - // A day earlier, so outside every window above, and recorded through a - // proxy that spelled the address as a bracketed literal with a port. The - // redirect writes a canonical bare address, so this is a claim about what - // the column may hold — a plain String bounded only in length, written by - // whatever shipped the record — rather than about what the redirect writes - // today, and that is exactly why it earns a test. Its own window, because a - // fifth row inside the day would move every count and every page boundary - // the block above asserts. + // A day earlier, so outside every window above, and recorded through a proxy + // that spelled the address as a bracketed literal with a port. The redirect + // writes a canonical bare address, so this is a claim about what the column + // may hold — a plain String bounded only in length, written by whatever + // shipped the record — rather than about what the redirect writes today. + // Another row inside the day would move every count and every page boundary + // the first block asserts. click({ click_id: '01920000-0000-7000-8000-000000000004', time: '2026-09-23 10:00:00.000', ip: '[2001:db8:1234:5678:9abc:def0:1234:5678]:443', }), + // Six clicks two days earlier, three of them in one millisecond and two in + // another: see the paging block at the end of the file. + ...tied, ]) }) @@ -172,27 +237,20 @@ describe('GET /api/clicks', () => { * crash in between ships the same segment again: two rows identical to the * byte, until a merge collapses them. * - * The duplicate is written **here and in its own INSERT**, not with the - * fixture. One INSERT is one part and the engine collapses equal sort keys as - * it writes a part, so a duplicate inside the fixture's array is gone before - * any test runs — and `FINAL` then has nothing to do, which is how this test - * passed with `FINAL` dropped. A second INSERT is a second part, and the raw - * count asserted below is what says the duplicate was still there when the log - * was read: a background merge may collapse it at any moment, and that count - * failing is the right answer to a fixture this test can no longer see. + * The raw count is read first, and it is what says the pair was still two rows + * when the log was read — so that this test is about `FINAL` and not about a + * fixture that had already been deduplicated behind it. What holds it there is + * `optimize_on_insert: 0` on the fixture's INSERT, not luck. */ it('shows a click that was shipped twice once', async () => { - await insert([click()]) - const rs = await ch.query({ query: 'SELECT count() AS n FROM clicks', format: 'JSONEachRow' }) - // Five rows: four from the fixture — three in this window and the day - // before's — plus the copy just written. The message is the whole point of - // asserting this before reading the log: four rows here means a background - // merge collapsed the pair between the INSERT and this count, which is a - // race in the fixture and not a regression in the query below. + const rs = await ch.query({ + query: `SELECT count() AS n FROM clicks WHERE click_id = '01920000-0000-7000-8000-000000000001'`, + format: 'JSONEachRow', + }) expect( await rs.json(), - 'a background merge collapsed the re-shipped copy before the log was read, so this test could not see whether FINAL deduplicated anything. That is a race in the fixture, not a failure of the query. If it recurs, give this suite a ClickHouse database of its own; never SYSTEM STOP MERGES, which these shared test databases would hand to every other suite.', - ).toEqual([{ n: '5' }]) + 'the re-shipped copy is no longer a second row, so this test cannot see whether FINAL deduplicated anything and would pass either way. The fixture holds the pair with optimize_on_insert: 0 on its INSERT; if that has gone, or a merge rewrote the part, restore it there. Not SYSTEM STOP MERGES, which these shared test databases would hand to every other suite.', + ).toEqual([{ n: '2' }]) const r = await app.inject({ method: 'GET', url: `/api/clicks?${WINDOW}`, @@ -279,6 +337,12 @@ describe('GET /api/clicks', () => { * where it holds its least interesting value can be replaced by a constant * with the rest of the suite green. So this one is asserted whole, at a value * nothing else in the file sends. + * + * It is also where six columns of the click itself are read at their second + * value — the host, the domain, the device, the OS, the browser and the user + * agent — which is what makes them columns rather than constants this mapper + * could have written in. The other whole click, in the test above, carries the + * first value of each. */ it('echoes the link it was asked for, and nothing else, whole', async () => { const r = await app.inject({ @@ -294,9 +358,9 @@ describe('GET /api/clicks', () => { { clickId: '01920000-0000-7000-8000-000000000002', at: '2026-09-24T10:01:00.000Z', - host: 'go.example.test', + host: 'links.example.test', path: '/b', - domainId: DOMAIN, + domainId: DOMAIN_B, linkId: LINK_B, outcome: 'target', step: 'destination', @@ -312,15 +376,15 @@ describe('GET /api/clicks', () => { region: null, city: null, geoSource: 'dbip', - device: 'desktop', - os: 'windows', - browser: 'chrome', + device: 'ios', + os: 'ios', + browser: 'safari', asn: null, class: 'human', signals: [], action: null, referrer: null, - userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/130', + userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_0) Safari/605', network: '2001:db8:1234:5678::/64', capUnchecked: false, }, @@ -392,6 +456,17 @@ describe('GET /api/clicks', () => { ['a cursor that is not one', 'cursor=nonsense'], ['a cursor with no id', 'cursor=1758708000000.'], ['a cursor whose id is not an id', 'cursor=1758708000000.not-a-uuid-at-all-not-at-all-no-x'], + // Three instants no click can have, and all three used to reach the store. + // The first is one millisecond past what `DateTime64(3,'UTC')` represents, + // which ClickHouse does not refuse but silently clamps to 2299 — so the + // caller would be paged from an instant they never named. The second is the + // start of year 10000, where `toISOString` switches to its extended-year + // form and the text this module builds is not a timestamp at all: a 503 and + // an error-level query-failure line, after a report slot was taken, for a + // bad request. The third is the largest the pattern's digits allow. + ['a cursor one millisecond past the last instant', `cursor=10413792000000.${CURSOR_ID}`], + ['a cursor in year ten thousand', `cursor=253402300800000.${CURSOR_ID}`], + ['a cursor of nothing but nines', `cursor=999999999999999.${CURSOR_ID}`], ['a class nobody has', 'class=spider'], ['an outcome nobody has', 'outcome=maybe'], ['a lower-case country', 'country=de'], @@ -415,6 +490,48 @@ describe('GET /api/clicks', () => { expect(r.json().error).toBe('invalid_query') }) + /** + * The newest instant a cursor may name, one millisecond below the refusal + * above. Written out rather than read from `MAX_CURSOR_MS`, because a bound + * derived from the constant it is testing moves when the constant moves and + * goes on passing — and this end is the one that says the check is not a + * millisecond tighter than the type it protects. + * + * It reads every click in the window, since every click is older than it: + * the 200 is what says the store accepted the instant rather than refusing or + * clamping it. + */ + it('takes the last instant a click can have', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&cursor=10413791999999.${CURSOR_ID}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.json().clicks).toHaveLength(3) + }) + + // The ceiling from both sides. The 200 is what says it is not one row tighter + // than the number it promises — tightening it to 150 passed every other test + // in this file — and the 201 that it is there at all. Both literals, for the + // reason the cursor's last instant is a literal. + it('takes the largest page it will return, and refuses one more', async () => { + const at = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&limit=200`, + headers: read(cookie), + }) + expect(at.statusCode).toBe(200) + expect(at.json().nextCursor).toBeNull() + const over = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&limit=201`, + headers: read(cookie), + }) + expect(over.statusCode).toBe(400) + expect(over.json().error).toBe('invalid_query') + }) + it('refuses a window longer than four hundred days', async () => { const r = await app.inject({ method: 'GET', @@ -476,6 +593,58 @@ describe('GET /api/clicks, an address with brackets and a port', () => { }) }) +/** + * Paging across a tie, one click at a time. + * + * The boundary is on `(time, click_id)` because the order is on `(time, + * click_id)`, and the pair is the whole point: with a boundary on the time + * alone, the next page starts after the *instant* the last row had, so every + * other row sharing that instant is skipped. Six clicks in three instants, three + * of them tied and two of them tied, walked at one row a page: the wrong + * boundary returns three of the six and every assertion about a page's length + * still passes, which is why this walks the whole window and compares ids. + * + * Its own window, because six more clicks in the window above would move every + * count and every page boundary asserted there. + */ +describe('GET /api/clicks, paging across a tie', () => { + const idsOf = (r: { json: () => { clicks: { clickId: string }[] } }): string[] => + r.json().clicks.map((c) => c.clickId) + + it('walks every click exactly once, newest first', async () => { + const seen: string[] = [] + let url = `/api/clicks?${TIED_DAY}&limit=1` + // One request per click plus the one that ends it, and a hard stop well + // inside that: a loop whose bound is the thing it is testing would spin for + // ever on a cursor that does not advance. + for (let page = 0; page < 10; page++) { + const r = await app.inject({ method: 'GET', url, headers: read(cookie) }) + expect(r.statusCode).toBe(200) + seen.push(...idsOf(r)) + const next = r.json().nextCursor as string | null + if (next === null) break + url = `/api/clicks?${TIED_DAY}&limit=1&cursor=${encodeURIComponent(next)}` + } + // Newest first, and within one instant by descending id: the reverse of the + // order the fixture writes them in. + expect(seen).toEqual([...TIED_IDS].reverse()) + }) + + // The same six in one page, which says two things at once: nothing is lost + // without a cursor, and the page size a caller who names none gets is at least + // six. The number itself — fifty — is not readable off a fixture of six, the + // way the ceiling is readable off a limit of 201. + it('returns them all when no page size is asked for', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${TIED_DAY}`, + headers: read(cookie), + }) + expect(idsOf(r)).toEqual([...TIED_IDS].reverse()) + expect(r.json().nextCursor).toBeNull() + }) +}) + describe('the log when ClickHouse is not there', () => { let dead: ClickHouseClient diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 61db5c1..060e2f8 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -67,11 +67,35 @@ const ListQuery = z }) .strict() +/** + * The newest instant a cursor may name: the last millisecond + * `DateTime64(3, 'UTC')` represents, which is the type every click time and both + * window bounds are stored and compared as. + * + * A caller's value is checked against the **range** of the type it is bound + * into, not only against its shape, and this is the third field on this surface + * to need saying so — after a fractional row limit and a link that was not a + * link. Both of the others reached the store and came back as + * `reporting_unavailable`, which tells an operator their install is broken when + * what is broken is one field of one request. + * + * Past this instant the store does neither of the two things a reader expects. + * It does not refuse the value: `2300-01-01` and `9999-12-31` are both silently + * clamped to 2299, so a caller would page from an instant they never named. And + * the text this module builds from a larger number is not a timestamp at all — + * `toISOString()` switches to its extended-year form at year 10000, so + * `253402300800000` becomes `+010000-01-01 00:00:00.000`, which ClickHouse + * refuses as a query parameter: a 503 and an error-level query-failure line, + * after a report slot was taken, for what is a bad request. + */ +export const MAX_CURSOR_MS = Date.parse('2299-12-31T23:59:59.999Z') + /** * A page boundary: the instant and the id of the last row of the page before. - * Opaque to the caller, and parsed strictly — it reaches a comparison against - * a `DateTime64` and a `UUID`, so anything that is not exactly those two things - * is refused here rather than by ClickHouse. + * Opaque to the caller, and parsed strictly — it reaches a comparison against a + * `DateTime64` and a `UUID`, so the pattern refuses anything that is not those + * two shapes and the bound above refuses an instant outside what the first of + * them holds. */ const CURSOR_RE = /^(\d{1,15})\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/ @@ -82,7 +106,14 @@ export function parseCursor(value: string): { atMs: number; clickId: string } { // a match; the casts are for the index signature, which does not know the // pattern has two groups. if (!m) return fail(400, 'invalid_query', 'cursor: not a cursor from a previous page') - return { atMs: Number(m[1] as string), clickId: m[2] as string } + const atMs = Number(m[1] as string) + // The pattern's digits are unsigned, so the oldest instant a cursor can name + // is the epoch, which is inside the type's range: only the newest end is + // reachable and only it is checked. + if (atMs > MAX_CURSOR_MS) { + return fail(400, 'invalid_query', 'cursor: names an instant no click can have') + } + return { atMs, clickId: m[2] as string } } /** From 3fbbfe7025b795d0f66c07ca803bcefe8af94e77 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 17:27:17 +0000 Subject: [PATCH 34/65] admin: say what the address rule actually covers --- packages/admin/src/clicks.test.ts | 50 ++++++++++++++++--------- packages/admin/src/clicks.ts | 54 ++++++++++++++++++--------- packages/admin/src/ip-hygiene.test.ts | 26 +++++++++---- 3 files changed, 88 insertions(+), 42 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 88fc473..0604f21 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -152,18 +152,25 @@ beforeAll(async () => { ip: '', cap_unchecked: 1, }), - // A day earlier, so outside every window above, and recorded through a proxy - // that spelled the address as a bracketed literal with a port. The redirect - // writes a canonical bare address, so this is a claim about what the column - // may hold — a plain String bounded only in length, written by whatever - // shipped the record — rather than about what the redirect writes today. - // Another row inside the day would move every count and every page boundary - // the first block asserts. + // Two addresses the parser reads differently, a day earlier so that they are + // outside every window above: another row inside the day would move every + // count and every page boundary the first block asserts. + // + // The redirect writes a canonical bare address and refuses nothing, so what + // this column may hold is whatever shipped the record — a plain String + // bounded only in length. The bracketed form with a port is a network once + // the port is taken off; the zone-suffixed form is not an address at all and + // can be stored today, since nothing on the write path would reject it. click({ click_id: '01920000-0000-7000-8000-000000000004', time: '2026-09-23 10:00:00.000', ip: '[2001:db8:1234:5678:9abc:def0:1234:5678]:443', }), + click({ + click_id: '01920000-0000-7000-8000-000000000005', + time: '2026-09-23 10:01:00.000', + ip: '2001:db8:1234:5678:9abc:def0:1234:5678%eth0', + }), // Six clicks two days earlier, three of them in one millisecond and two in // another: see the paging block at the end of the file. ...tied, @@ -568,17 +575,23 @@ describe('GET /api/clicks', () => { }) /** - * An address the way a proxy may have spelled it, in a window of its own. + * Two addresses the parser reads differently, in a window of their own. * - * `truncateIp` answers null for anything that is not bare address text, a port - * and a bracketed literal included, and the stored column is a plain String the - * record schema bounds only in length. So the address is run through - * `addressOnly` first; without that, a click recorded this way reads as "no - * network" — the same answer as a click whose address the retention pass - * blanked, which is the one thing the two must not share. + * `truncateIp` answers null for anything that is not bare address text, so the + * stored value goes through `addressOnly` first: without that, a bracketed + * address with a port reads as "no network" — the answer a click whose address + * the retention pass blanked gets, which is the one thing the two must not + * share. + * + * The zone-suffixed form is the other half, and it stays null on purpose. There + * is no parse this module could do that would turn `%eth0` into a network, so + * what the operator is told is the truth: this click has no network. What a null + * therefore means is one of two things — the column was blanked, or whatever + * wrote the row was not this redirect — and nothing in the response distinguishes + * them, because nothing in the row does either. */ -describe('GET /api/clicks, an address with brackets and a port', () => { - it('shows it as the network it came from', async () => { +describe('GET /api/clicks, addresses a proxy may have spelled oddly', () => { + it('shows the bracketed one as a network and the zone-suffixed one as nothing', async () => { const r = await app.inject({ method: 'GET', url: `/api/clicks?${DAY_BEFORE}`, @@ -589,7 +602,10 @@ describe('GET /api/clicks, an address with brackets and a port', () => { r .json() .clicks.map((c: { clickId: string; network: string | null }) => [c.clickId, c.network]), - ).toEqual([['01920000-0000-7000-8000-000000000004', '2001:db8:1234:5678::/64']]) + ).toEqual([ + ['01920000-0000-7000-8000-000000000005', null], + ['01920000-0000-7000-8000-000000000004', '2001:db8:1234:5678::/64'], + ]) }) }) diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 060e2f8..bc4c4b7 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -14,11 +14,21 @@ * * **The address is a network and the field says so.** A field called `ip` * holding `198.51.100.0/24` invites its next reader to treat it as an address; - * `network` cannot be misread, and it means no response this service sends has - * a field called `ip` at all — which a test can check across the whole surface - * rather than one route at a time. The truncation itself is `truncateIp`, the - * same function the export calls, and it is never done in SQL: two copies of a - * rule about what an operator may see is one too many. + * `network` cannot be misread. The rule it makes checkable is about click data: + * no response built from a click carries a whole address, or a field called + * `ip`, on any route — which is a property of the surface rather than of this + * module, and what the check exists to catch is a route added later that selects + * a click's `ip` and forgets to truncate it. + * + * It is **not** a claim that no response names a whole address. `GET + * /api/sessions` hands the operator the addresses their own sessions were opened + * from, whole and deliberately, because that is how they recognise their own + * devices. The difference is whose address it is: a visitor never consented to + * being in an operator's log, and the operator is looking at themselves. + * + * The truncation itself is `truncateIp`, and it is never done in SQL: a rule + * about what an operator may see is stated once, in one place, so that a second + * reader of these rows cannot come to state it differently. */ import { OUTCOMES, TRAFFIC_CLASSES } from '@clickmonk/core' import { addressOnly, truncateIp } from '@clickmonk/ipdata' @@ -43,7 +53,11 @@ export const MAX_CLICK_PAGE = 200 /** * The filters the product promises: link, class, outcome, country and time. - * Shared with the export, which filters the same rows it would have listed. + * + * Exported, with the column list and the row mapper below, so that a second + * reader of these rows — a download of the same log is the one this product + * wants next — filters and maps them with this module's rules rather than with + * its own. This route is the only caller today. */ export const CLICK_FILTER_FIELDS = { ...WINDOW_FIELDS, @@ -154,7 +168,7 @@ export interface ClickRow { } /** - * The columns both readers select, in the order the CSV writes them. + * The columns the log selects, exported for the reason the filter fields are. * * `toUnixTimestamp64Milli` rather than the column itself: an instant that * travels as a number needs no agreement about how either end formats a @@ -171,15 +185,21 @@ const none = (s: string): string | null => (s === '' ? null : s) * The network a click came from, or nothing. * * `addressOnly` first, always. `truncateIp` answers null for anything that is - * not bare address text — a port or a bracketed literal included — and the - * stored column is a plain `String` that the record schema bounds only in - * length, so a click recorded through a proxy that spelled the address - * `[2001:db8::5]:443` would otherwise read as "no network" rather than as the - * /64 it came from. Returning null is the right answer for a value the parser - * genuinely does not recognise and the wrong one for a value it would - * recognise a character later. + * not bare address text, and the stored column is a plain `String` that the + * record schema bounds only in length, so a click recorded through a proxy that + * spelled the address `[2001:db8::5]:443` would otherwise read as "no network" + * rather than as the /64 it came from. + * + * What is left after that reads as nothing, and there is no better answer to + * give. A zone-suffixed address or a dotted quad with leading zeros is not + * something this parser will take, and the only other value to hand over is the + * stored one, which is the whole address. So a null here means one of two things + * and the response does not tell them apart, because the row does not either: + * the column was blanked by the retention pass, or whatever wrote the row was + * not this redirect. The empty string a blanked column holds is among the values + * `truncateIp` already refuses, so it needs no branch of its own. */ -const networkOf = (ip: string): string | null => (ip === '' ? null : truncateIp(addressOnly(ip))) +const networkOf = (ip: string): string | null => truncateIp(addressOnly(ip)) /** One click over the API. The address is a network; nothing here is an address. */ export function asClick(r: ClickRow): Record { @@ -214,8 +234,8 @@ export function asClick(r: ClickRow): Record { action: none(r.action), referrer: none(r.referrer), userAgent: none(r.user_agent), - // Null for a click whose address was blanked by the retention pass, and - // null for anything the parser does not recognise: never the stored value. + // Null for a click whose address was blanked by the retention pass, and null + // for anything the parser does not recognise. Never the stored value. network: networkOf(r.ip), capUnchecked: r.cap_unchecked === 1, } diff --git a/packages/admin/src/ip-hygiene.test.ts b/packages/admin/src/ip-hygiene.test.ts index acde432..d192ea6 100644 --- a/packages/admin/src/ip-hygiene.test.ts +++ b/packages/admin/src/ip-hygiene.test.ts @@ -4,13 +4,22 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from import { clockFrom, read, signedIn, testApp } from './testing.js' /** - * No response this service sends carries a whole address, and no response has - * a field called `ip`. + * No response carries a **visitor's** whole address, and no click in a response + * has a field called `ip`. * * It is one file rather than an assertion inside each route's suite because the - * rule is about the surface: a route added later that selects `ip` and forgets - * to truncate it is caught here, by a test nobody has to remember to extend, - * as long as its URL is in the list below. + * rule is about the surface: a route added later that selects a click's `ip` and + * forgets to truncate it is caught here, by a test nobody has to remember to + * extend, as long as its URL is in the list below. + * + * What it does not say is that no response anywhere names an address. `GET + * /api/sessions` is in the list and hands the operator the addresses their own + * sessions were opened from, whole and on purpose; it passes because the + * addresses in these fixtures are visitors' addresses, in ClickHouse, which the + * session list has no way to reach. So the two addresses below are the subject of + * the check, not merely two strings that happen not to appear: a route that reads + * a click and prints its address whole fails, and the operator's own session + * addresses go on being shown. */ const pool = testPg() const ch = testCh() @@ -23,9 +32,10 @@ const HOST_ADDRESS = '198.51.100.77' const V6_ADDRESS = '2001:db8:1234:5678:9abc:def0:1234:5678' const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' -// Every GET this service answers, so that the check is about the surface and -// not about the routes somebody thought of. The CSV export is added to this -// list by the task that adds the export. +// Every GET behind the credential, so that the check is about the surface and +// not about the routes somebody thought of. `/health` is the one GET left out: +// it is in front of the credential and answers a fixed status with no data of +// its own. A read added later belongs here. const READS = [ `/api/clicks?${WINDOW}`, `/api/reports/summary?${WINDOW}`, From b969feac13eee363a234e655a1aff2ef900da9b8 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 17:35:10 +0000 Subject: [PATCH 35/65] admin: a window the store reads as the window it was given --- packages/admin/src/clicks.test.ts | 75 ++++++++++++++++++++++++++++-- packages/admin/src/clicks.ts | 34 ++++---------- packages/admin/src/reports.test.ts | 68 +++++++++++++++++++++++++++ packages/admin/src/reports.ts | 27 +++++++++++ 4 files changed, 175 insertions(+), 29 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 0604f21..578eefb 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -96,6 +96,25 @@ const TIED_TIMES = [ ] const tied = TIED_IDS.map((id, i) => click({ click_id: id, time: TIED_TIMES[i] })) +/** + * Fifty-one clicks in one instant, in a month of its own and older than every + * other click in this file. + * + * The page size a caller gets when they name none cannot be read off a window of + * three or six clicks: any default of three or more answers every assertion in + * them. So this window holds one more click than the default, in a partition of + * its own so that it moves nothing else, and all fifty-one share an instant — a + * page of them is then cut by the id alone, which is the tie-break, so the same + * window says what the default is and which fifty it returns. + */ +const PAGE_INSTANT = '2026-08-15 10:00:00.000' +const PAGE_INSTANT_MS = Date.parse('2026-08-15T10:00:00.000Z') +const PAGE_IDS = Array.from( + { length: 51 }, + (_, i) => `01920000-0000-7000-8000-0000000002${(i + 1).toString(16).padStart(2, '0')}`, +) +const manyInOneInstant = PAGE_IDS.map((id) => click({ click_id: id, time: PAGE_INSTANT })) + /** A well-formed id for a cursor whose instant is the thing under test. */ const CURSOR_ID = '01920000-0000-7000-8000-0000000000ff' @@ -104,6 +123,7 @@ const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' /** Two days nothing above reads: see the two blocks at the end of the file. */ const DAY_BEFORE = 'from=2026-09-23T00:00:00.000Z&to=2026-09-24T00:00:00.000Z' const TIED_DAY = 'from=2026-09-22T00:00:00.000Z&to=2026-09-23T00:00:00.000Z' +const PAGE_DAY = 'from=2026-08-15T00:00:00.000Z&to=2026-08-16T00:00:00.000Z' beforeAll(async () => { await resetDatabases(pool, ch) @@ -174,6 +194,8 @@ beforeAll(async () => { // Six clicks two days earlier, three of them in one millisecond and two in // another: see the paging block at the end of the file. ...tied, + // Fifty-one in another month: see the default-page block at the end. + ...manyInOneInstant, ]) }) @@ -539,6 +561,21 @@ describe('GET /api/clicks', () => { expect(over.json().error).toBe('invalid_query') }) + // The window the store cannot hold, refused by this route because the parser + // every report route shares refuses it. Four hundred days in 9999 is inside + // the length bound, and the store would clamp it to 2299 and answer for a year + // nobody asked about — the refusal is what keeps the window this response + // echoes the window that was read. + it('refuses a window the store could not hold', async () => { + const r = await app.inject({ + method: 'GET', + url: '/api/clicks?from=9999-01-01T00:00:00.000Z&to=9999-12-31T00:00:00.000Z', + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_query') + }) + it('refuses a window longer than four hundred days', async () => { const r = await app.inject({ method: 'GET', @@ -646,10 +683,9 @@ describe('GET /api/clicks, paging across a tie', () => { expect(seen).toEqual([...TIED_IDS].reverse()) }) - // The same six in one page, which says two things at once: nothing is lost - // without a cursor, and the page size a caller who names none gets is at least - // six. The number itself — fifty — is not readable off a fixture of six, the - // way the ceiling is readable off a limit of 201. + // The same six in one page: nothing is lost when no cursor is given. What the + // page size actually is belongs to the block below, which owns a window big + // enough to read it off. it('returns them all when no page size is asked for', async () => { const r = await app.inject({ method: 'GET', @@ -661,6 +697,37 @@ describe('GET /api/clicks, paging across a tie', () => { }) }) +/** + * The page a caller gets when they ask for no page size. + * + * Fifty of fifty-one, and the fifty it returns are the fifty highest ids: with + * every click at one instant the order is the id alone, so a cut at the wrong end + * — an ascending tie-break — returns a different fifty and not a different + * number, which no assertion about the length could see. The one left out is + * named, and the cursor is asserted whole, so the page ends where the next one + * starts. + * + * Fifty is written out rather than read from `DEFAULT_CLICK_PAGE`, for the reason + * every bound in this file is written out. + */ +describe('GET /api/clicks, the page size nobody asked for', () => { + it('answers fifty of them, newest id first, and says there are more', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${PAGE_DAY}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + const ids = r.json().clicks.map((c: { clickId: string }) => c.clickId) + expect(ids).toHaveLength(50) + // The fifty highest ids, descending: everything but the lowest. + expect(ids).toEqual([...PAGE_IDS].slice(1).reverse()) + expect(ids).not.toContain(PAGE_IDS[0]) + // And the boundary is the last row of this page, not the end of the window. + expect(r.json().nextCursor).toBe(`${PAGE_INSTANT_MS}.${PAGE_IDS[1]}`) + }) +}) + describe('the log when ClickHouse is not there', () => { let dead: ClickHouseClient diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index bc4c4b7..8b79424 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -38,6 +38,7 @@ import type { AdminContext } from './app.js' import { requireCredential } from './auth.js' import { fail } from './http.js' import { + MAX_STORE_MS, type ReportWindow, WINDOW_FIELDS, chRows, @@ -81,35 +82,18 @@ const ListQuery = z }) .strict() -/** - * The newest instant a cursor may name: the last millisecond - * `DateTime64(3, 'UTC')` represents, which is the type every click time and both - * window bounds are stored and compared as. - * - * A caller's value is checked against the **range** of the type it is bound - * into, not only against its shape, and this is the third field on this surface - * to need saying so — after a fractional row limit and a link that was not a - * link. Both of the others reached the store and came back as - * `reporting_unavailable`, which tells an operator their install is broken when - * what is broken is one field of one request. - * - * Past this instant the store does neither of the two things a reader expects. - * It does not refuse the value: `2300-01-01` and `9999-12-31` are both silently - * clamped to 2299, so a caller would page from an instant they never named. And - * the text this module builds from a larger number is not a timestamp at all — - * `toISOString()` switches to its extended-year form at year 10000, so - * `253402300800000` becomes `+010000-01-01 00:00:00.000`, which ClickHouse - * refuses as a query parameter: a 503 and an error-level query-failure line, - * after a report slot was taken, for what is a bad request. - */ -export const MAX_CURSOR_MS = Date.parse('2299-12-31T23:59:59.999Z') - /** * A page boundary: the instant and the id of the last row of the page before. * Opaque to the caller, and parsed strictly — it reaches a comparison against a * `DateTime64` and a `UUID`, so the pattern refuses anything that is not those - * two shapes and the bound above refuses an instant outside what the first of + * two shapes and `MAX_STORE_MS` refuses an instant outside what the first of * them holds. + * + * That range is the window's, imported rather than written out again here: one + * fact about what the store can hold, in one place, because a second spelling of + * it is how the two come to disagree. What the store does either side of it — a + * clamp below year 10000 and a refused parameter above — is why the check + * exists, and the constant's own comment says it. */ const CURSOR_RE = /^(\d{1,15})\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/ @@ -124,7 +108,7 @@ export function parseCursor(value: string): { atMs: number; clickId: string } { // The pattern's digits are unsigned, so the oldest instant a cursor can name // is the epoch, which is inside the type's range: only the newest end is // reachable and only it is checked. - if (atMs > MAX_CURSOR_MS) { + if (atMs > MAX_STORE_MS) { return fail(400, 'invalid_query', 'cursor: names an instant no click can have') } return { atMs, clickId: m[2] as string } diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 462923c..9c509af 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -904,6 +904,74 @@ describe('parseWindow', () => { expect((thrown as HttpError).code).toBe('invalid_query') }) + /** + * The ends of the window against the ends of the type it is compared as. + * + * The length bound above does not imply a position bound: four hundred days in + * the year 9999 is inside it, inside the bucket ceiling, and outside what + * `DateTime64(3,'UTC')` can hold. What the store does with it is the reason + * this is a refusal rather than a curiosity — below year 10000 it clamps + * silently, so a report asked about 9999 is answered about 2299 with nothing + * saying so, and above it the parameter is refused and the caller is told the + * install is unavailable. + * + * Every instant is written out rather than taken from `MIN_STORE_MS` and + * `MAX_STORE_MS`: a bound derived from the constant it is testing moves when + * the constant moves and goes on passing. + */ + it.each([ + ['the first instant the store holds', '1900-01-01T00:00:00.000Z', '1900-01-02T00:00:00.000Z'], + ['the last', '2299-12-30T00:00:00.000Z', '2299-12-31T23:59:59.999Z'], + ])('takes a window at %s', (_label, from, to) => { + const w = parseWindow({ from, to }, { alignMs: null }) + expect(w.fromMs).toBe(Date.parse(from)) + expect(w.toMs).toBe(Date.parse(to)) + }) + + it.each([ + // One millisecond before the first instant, and a day either side of the + // last: the length of each is well inside four hundred days, which is the + // half the length bound cannot see. + [ + 'a day before the first instant it holds', + '1899-12-31T00:00:00.000Z', + '1900-01-01T00:00:00.000Z', + ], + ['a day after its last', '2299-12-31T00:00:00.000Z', '2300-01-01T00:00:00.000Z'], + ['a window in the year nine thousand', '9999-01-01T00:00:00.000Z', '9999-12-31T00:00:00.000Z'], + ])('refuses a window at %s', (_label, from, to) => { + let thrown: unknown + try { + parseWindow({ from, to }, { alignMs: null }) + } catch (err) { + thrown = err + } + expect(thrown).toBeInstanceOf(HttpError) + expect((thrown as HttpError).status).toBe(400) + expect((thrown as HttpError).code).toBe('invalid_query') + }) + + /** + * Raising `to` to the next hour is itself a way past the end, which is why the + * check runs after the alignment and not on what the caller sent. The last + * millisecond of 9999 aligns into year 10000, where `toISOString` stops + * producing a timestamp at all: unaligned this window would be refused for + * being past 2299 anyway, so what this pins is that an *aligned* end is what + * gets checked. + */ + it('refuses a window the alignment pushes past the end', () => { + let thrown: unknown + try { + parseWindow( + { from: '9999-12-31T22:00:00.000Z', to: '9999-12-31T23:59:59.999Z' }, + { alignMs: HOUR_MS }, + ) + } catch (err) { + thrown = err + } + expect((thrown as HttpError).code).toBe('invalid_query') + }) + // The link's shape belongs here for the same reason the length does. Bound // into a query as a UUID, a link that is not one comes back from the store as // a failure, so a caller reaching this function without the route's schema diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 9b59e81..8b15a62 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -66,6 +66,26 @@ export const WINDOW_FIELDS = { link: LinkId.optional(), } +/** + * The instants the store can hold: the range `DateTime64(3, 'UTC')` represents, + * which is the type every click time, every rollup hour and both ends of every + * window are compared as. + * + * **A bound on a window's length is not a bound on where it sits.** Four hundred + * days in the year 9999 is inside every length and bucket ceiling here, and what + * happens to it is the reason this exists: ClickHouse does not refuse a + * parameter past 2299, it silently clamps it, so a report asked for 9999 would + * be answered for 2299 with nothing in the response saying so. Past year 9999 it + * is worse in the other direction — `toISOString()` switches to its extended-year + * form, the text built below is not a timestamp at all, and the store's refusal + * reaches the caller as `reporting_unavailable`: an install outage for what is + * one bad field of one request. The same rule as the row limit, the link and the + * log's cursor: **a caller's value is checked against the range of the type it + * is bound into, not only against its shape.** + */ +export const MIN_STORE_MS = Date.parse('1900-01-01T00:00:00.000Z') +export const MAX_STORE_MS = Date.parse('2299-12-31T23:59:59.999Z') + export interface ReportWindow { fromMs: number toMs: number @@ -112,6 +132,13 @@ export function parseWindow( if (toMs - fromMs > MAX_REPORT_WINDOW_MS) { return fail(400, 'window_too_long', `a window may be at most ${MAX_REPORT_WINDOW_DAYS} days`) } + // After the alignment, because the alignment is what a query is run with — and + // because raising `to` to the next hour is itself a way past the end: a window + // ending at the last millisecond of 9999 is aligned into year 10000, which is + // where the store's refusal turns into a 503. + if (fromMs < MIN_STORE_MS || toMs > MAX_STORE_MS) { + return fail(400, 'invalid_query', 'from, to: must name instants the store can hold') + } // The shape of the link, here and not only in a route's schema, for the same // reason the length is here: this value is bound into a query as a UUID, so a // caller who reaches this function without that schema turns a bad request From 3220b433379142931050a748f6c1a1e3792a1d48 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 17:37:20 +0000 Subject: [PATCH 36/65] admin: an alignment the range check has to see --- packages/admin/src/reports.test.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/admin/src/reports.test.ts b/packages/admin/src/reports.test.ts index 9c509af..1bd6f04 100644 --- a/packages/admin/src/reports.test.ts +++ b/packages/admin/src/reports.test.ts @@ -953,22 +953,26 @@ describe('parseWindow', () => { /** * Raising `to` to the next hour is itself a way past the end, which is why the - * check runs after the alignment and not on what the caller sent. The last - * millisecond of 9999 aligns into year 10000, where `toISOString` stops - * producing a timestamp at all: unaligned this window would be refused for - * being past 2299 anyway, so what this pins is that an *aligned* end is what - * gets checked. + * check runs on the aligned window and not on what the caller sent. + * + * This window is legal unaligned — both ends are inside the range, and it is + * asserted here that they are — and illegal aligned, because the ceil to the + * next hour lands in 2300 where the store clamps. One input, two answers, and + * the only thing that decides is which side of the alignment the check sits on. + * A window in the year 9999 could not pin that: it is outside the range before + * the alignment as well, so a check in the wrong place refuses it too. */ - it('refuses a window the alignment pushes past the end', () => { + it('refuses a window the alignment pushes past the end, and takes it unaligned', () => { + const q = { from: '2299-12-30T00:00:00.000Z', to: '2299-12-31T23:30:00.000Z' } + expect(parseWindow(q, { alignMs: null }).toMs).toBe(Date.parse('2299-12-31T23:30:00.000Z')) let thrown: unknown try { - parseWindow( - { from: '9999-12-31T22:00:00.000Z', to: '9999-12-31T23:59:59.999Z' }, - { alignMs: HOUR_MS }, - ) + parseWindow(q, { alignMs: HOUR_MS }) } catch (err) { thrown = err } + expect(thrown).toBeInstanceOf(HttpError) + expect((thrown as HttpError).status).toBe(400) expect((thrown as HttpError).code).toBe('invalid_query') }) From 7f173dc13d9d2ca7e88f82b31d54e46935447e2a Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 17:52:07 +0000 Subject: [PATCH 37/65] admin: the click log as a file, a row at a time --- packages/admin/src/app.ts | 18 +- packages/admin/src/clicks.test.ts | 315 +++++++++++++++++++++++++- packages/admin/src/clicks.ts | 182 ++++++++++++++- packages/admin/src/csv.test.ts | 57 +++++ packages/admin/src/csv.ts | 37 +++ packages/admin/src/ip-hygiene.test.ts | 12 +- 6 files changed, 610 insertions(+), 11 deletions(-) create mode 100644 packages/admin/src/csv.test.ts create mode 100644 packages/admin/src/csv.ts diff --git a/packages/admin/src/app.ts b/packages/admin/src/app.ts index 5a4e186..511e38b 100644 --- a/packages/admin/src/app.ts +++ b/packages/admin/src/app.ts @@ -8,7 +8,7 @@ import Fastify, { type FastifyServerOptions, } from 'fastify' import { type Credential, authenticate, checkCsrf, hasBearer } from './auth.js' -import { registerClickRoutes } from './clicks.js' +import { EXPORT_ROW_CAP, registerClickRoutes } from './clicks.js' import { registerDomainRoutes } from './domains.js' import { HttpError, MAX_BODY_BYTES, securityHeaders } from './http.js' import { registerKeyRoutes } from './keys.js' @@ -36,10 +36,10 @@ export const DNS_CHECKS_IN_FLIGHT = 2 * Past these the answer is a refusal with `retry-after`, never a queued query, * for the same reason the on-demand DNS check refuses rather than queues. * - * The export gate is declared here and nothing takes it yet: the endpoint it - * bounds does not exist. It is separate from the report gate, and smaller, - * because an export holds a ClickHouse result open for as long as its client - * takes to read it, which is not a bound this process can set. + * The export gate is separate from the report gate, and smaller, because an + * export holds a ClickHouse result open for as long as its client takes to read + * it, which is not a bound this process can set. Separate so that a download an + * operator started does not lock their own dashboard out. */ export const REPORT_QUERIES_IN_FLIGHT = 2 export const EXPORTS_IN_FLIGHT = 1 @@ -89,6 +89,12 @@ export interface AdminDeps { ch?: ClickHouseClient reportGate?: ConcurrencyGate exportGate?: ConcurrencyGate + /** + * Rows one export writes at most. Injectable only so that a test can reach + * the cap with three rows instead of a million; nothing in the product sets + * it. + */ + exportRowCap?: number /** * Where a link with no slug of its own gets one. `newSlug` unless a caller * says otherwise, and nothing in the product says otherwise: it is here so @@ -113,6 +119,7 @@ export interface AdminContext extends AdminDeps { checkGate: ConcurrencyGate reportGate: ConcurrencyGate exportGate: ConcurrencyGate + exportRowCap: number slugSource: () => string } @@ -160,6 +167,7 @@ export function buildAdminApp( checkGate: deps.checkGate ?? new ConcurrencyGate(DNS_CHECKS_IN_FLIGHT), reportGate: deps.reportGate ?? new ConcurrencyGate(REPORT_QUERIES_IN_FLIGHT), exportGate: deps.exportGate ?? new ConcurrencyGate(EXPORTS_IN_FLIGHT), + exportRowCap: deps.exportRowCap ?? EXPORT_ROW_CAP, slugSource: deps.slugSource ?? newSlug, } diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 578eefb..fd05eaa 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -611,6 +611,243 @@ describe('GET /api/clicks', () => { }) }) +/** + * The same log, as a file. + * + * Every data row is asserted whole where the fixture gives it an interesting + * value, rather than by a substring: the header row and the values come from one + * list, so a column dropped from it changes both — and what a substring could + * not see is a value written into the wrong column, or a cell that stopped being + * quoted. The two rows read whole are the bot click, which carries a list, four + * empty columns and no network, and the oldest click, which carries a value in + * every column but two. + */ +describe('GET /api/clicks.csv', () => { + const lines = (body: string): string[] => body.split('\r\n').filter((l) => l.length > 0) + + /** + * The first cell of every data row. Not a `toContain` on the whole body for an + * id: the base fixture click's `destination` carries its own click id, so a + * body holding the second click also holds the first click's id and an + * absent-id assertion would pass against an export that had not dropped it. + */ + const idsOf = (body: string): string[] => + lines(body) + .slice(1) + .map((l) => l.slice(1, l.indexOf('","'))) + + const HEADER = + '"clickId","at","host","path","domainId","linkId","outcome","step","status","destination","targetId","visitorId","returning","country","region","city","geoSource","device","os","browser","asn","class","signals","action","referrer","userAgent","network","capUnchecked"' + + it('exports the rows the log would have listed, newest first, as a file', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.headers['content-type']).toBe('text/csv; charset=utf-8') + expect(r.headers['content-disposition']).toBe( + 'attachment; filename="clicks-20260924T000000Z-20260925T000000Z.csv"', + ) + expect(r.headers['x-clickmonk-truncated']).toBe('false') + expect(r.headers['x-clickmonk-row-cap']).toBe('1000000') + const out = lines(r.body) + expect(out[0]).toBe(HEADER) + expect(out).toHaveLength(4) + expect(idsOf(r.body)).toEqual([ + '01920000-0000-7000-8000-000000000003', + '01920000-0000-7000-8000-000000000002', + '01920000-0000-7000-8000-000000000001', + ]) + // The bot click, whole: a list of signals joined into one cell, four columns + // that are empty because the click had nothing in them, and no network + // because its address column was blanked. + expect(out[1]).toBe( + `"01920000-0000-7000-8000-000000000003","2026-09-24T10:02:00.000Z","go.example.test","/a","${DOMAIN}","${LINK_A}","blocked","classify","403","","","v1","false","","","","","desktop","windows","chrome","64500","bot","ua_bot head","block","https://blog.example.com/post","Mozilla/5.0 (Windows NT 10.0) Chrome/130","","true"`, + ) + // The oldest click, whole: the network the address was truncated to, the + // instant as the JSON spells it, and every other column at a real value. + expect(out[3]).toBe( + `"01920000-0000-7000-8000-000000000001","2026-09-24T10:00:00.000Z","go.example.test","/a","${DOMAIN}","${LINK_A}","target","destination","302","https://example.com/?c=01920000-0000-7000-8000-000000000001","${TARGET}","v1","false","DE","","","dbip","desktop","windows","chrome","64500","human","","","https://blog.example.com/post","Mozilla/5.0 (Windows NT 10.0) Chrome/130","198.51.100.0/24","false"`, + ) + }) + + /** + * The re-shipped pair is still two rows in the table, asserted first for the + * reason the log's own dedup test asserts it: without that, a fixture a merge + * had already collapsed would pass this whether the export says FINAL or not. + */ + it('writes a click that was shipped twice once', async () => { + const rs = await ch.query({ + query: `SELECT count() AS n FROM clicks WHERE click_id = '01920000-0000-7000-8000-000000000001'`, + format: 'JSONEachRow', + }) + expect( + await rs.json(), + 'the re-shipped copy is no longer a second row, so this test would pass whether or not the export deduplicates. The fixture holds the pair with optimize_on_insert: 0 on its INSERT.', + ).toEqual([{ n: '2' }]) + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect( + idsOf(r.body).filter((id) => id === '01920000-0000-7000-8000-000000000001'), + ).toHaveLength(1) + }) + + it('holds nothing: no content-length, and a chunked body', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.headers['content-length']).toBeUndefined() + expect(r.headers['transfer-encoding']).toBe('chunked') + }) + + it('says in a header when the window held more than the cap, and stops at the cap', async () => { + const capped = testApp(pool, clock, { ch, exportRowCap: 2 }) + try { + const r = await capped.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.headers['x-clickmonk-truncated']).toBe('true') + expect(r.headers['x-clickmonk-row-cap']).toBe('2') + // A header row and the two newest clicks; the third is not there. + expect(lines(r.body)).toHaveLength(3) + expect(idsOf(r.body)).toEqual([ + '01920000-0000-7000-8000-000000000003', + '01920000-0000-7000-8000-000000000002', + ]) + } finally { + await capped.close() + } + }) + + it('does not claim truncation when the window held exactly the cap', async () => { + const capped = testApp(pool, clock, { ch, exportRowCap: 3 }) + try { + const r = await capped.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.headers['x-clickmonk-truncated']).toBe('false') + expect(lines(r.body)).toHaveLength(4) + } finally { + await capped.close() + } + }) + + it('takes the same filters as the log', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}&class=bot`, + headers: read(cookie), + }) + expect(lines(r.body)).toHaveLength(2) + expect(idsOf(r.body)).toEqual(['01920000-0000-7000-8000-000000000003']) + }) + + it.each([ + ['a class nobody has', 'class=spider'], + ['a field nobody knows', 'limit=10'], + [ + 'a cursor, which an export does not take', + 'cursor=1758708000000.01920000-0000-7000-8000-000000000001', + ], + ])('refuses %s, before a single byte is written', async (_label, extra) => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}&${extra}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(400) + expect(r.json().error).toBe('invalid_query') + // No file began: a query parsed after the headers were set answers a 400 + // that a browser saves as a download. + expect(r.headers['content-type']).toMatch(/application\/json/) + expect(r.headers['content-disposition']).toBeUndefined() + }) + + it('needs a credential, and writes no bytes of a file', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + // A refusal that had already started the body would be a 401 with CSV in + // it, which is what a guard moved below the stream looks like. + expect(r.headers['content-type']).toMatch(/application\/json/) + expect(r.body).not.toContain('"clickId"') + }) + + it('gives the export slot back when the stream has ended', async () => { + const one = new ConcurrencyGate(1) + const shared = testApp(pool, clock, { ch, exportGate: one }) + try { + const first = await shared.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(first.statusCode).toBe(200) + // The first export has finished, so the slot is back and the second is not + // refused. A slot given back when the handler returned would make this + // pass while bounding nothing, which is why the refusal below is the test + // that pins the gate. + const second = await shared.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(second.statusCode).toBe(200) + expect(one.stats().inFlight).toBe(0) + } finally { + await shared.close() + } + }) + + it('refuses when the export slot is taken', async () => { + const full = testApp(pool, clock, { ch, exportGate: new ConcurrencyGate(0) }) + try { + const r = await full.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(429) + expect(r.json().error).toBe('too_many_exports') + expect(r.headers['retry-after']).toBe('1') + // And no file began: the refusal is in front of the query, not in front + // of the last row. + expect(r.headers['content-type']).toMatch(/application\/json/) + expect(r.body).not.toContain('"clickId"') + } finally { + await full.close() + } + }) + + it('uses the report gate for nothing, so an export does not lock out a dashboard', async () => { + const busy = testApp(pool, clock, { ch, reportGate: new ConcurrencyGate(0) }) + try { + const r = await busy.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + } finally { + await busy.close() + } + }) +}) + /** * Two addresses the parser reads differently, in a window of their own. * @@ -728,7 +965,7 @@ describe('GET /api/clicks, the page size nobody asked for', () => { }) }) -describe('the log when ClickHouse is not there', () => { +describe('the log and the export when ClickHouse is not there', () => { let dead: ClickHouseClient beforeEach(() => { @@ -807,4 +1044,80 @@ describe('the log when ClickHouse is not there', () => { await on.close() } }) + + /** + * The export answers the same 503, and gives its slot back, so a store that is + * down does not read as a queue that is full. + * + * Two requests in a row through the same app, whose export gate holds one + * slot: the probe fails inside the gate, and the slot is given back by the + * handler's own `finally` because the generator never took ownership of it. + * Without that `finally` the second request is a 429 about an export nobody is + * running. + */ + it('answers 503 for an export and does not keep the slot it took', async () => { + const on = testApp(pool, clock, { ch: dead }) + try { + const first = await on.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(first.statusCode).toBe(503) + expect(first.json().error).toBe('reporting_unavailable') + // No file began, and no header of one: the probe runs before any of them + // is set. + expect(first.headers['content-type']).toMatch(/application\/json/) + expect(first.headers['content-disposition']).toBeUndefined() + const second = await on.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(second.statusCode).toBe(503) + expect(second.json().error).toBe('reporting_unavailable') + } finally { + await on.close() + } + }) + + /** + * The export asks for the credential before it takes a slot and before it + * reads anything, and this is what pins it there. + * + * Three positions, three answers, and the 401 beside the other export tests + * can only see the first of them. With the export gate full and the store + * unreachable: asked for after the probe, an unauthenticated request scans the + * window it chose and gets a 503; asked for inside the gate but before the + * probe, it gets a 429; only asked for before both is the answer 401. The + * query-failure line is asserted absent as well, because a status alone cannot + * tell a refusal from a query that happened to fail. + */ + it('refuses an export without a credential before taking a slot or reading anything', async () => { + const lines: string[] = [] + const on = testApp(pool, clock, { + ch: dead, + exportGate: new ConcurrencyGate(0), + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + const r = await on.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST }, + }) + expect(r.statusCode).toBe(401) + expect(r.json().error).toBe('unauthenticated') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + } finally { + await on.close() + } + }) }) diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 8b79424..2f321b5 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -1,5 +1,10 @@ /** - * The raw click log. + * The raw click log, as a page of JSON and as a file. + * + * Two representations of one resource rather than two endpoints: the filters, + * the column list, the dedup and the truncation are written once here and both + * readers use them, so a rule about what an operator may see cannot come to be + * stated differently in the download than in the page. * * Everything else in these reports reads an hourly rollup; this reads the * clicks themselves, which is why it is the only place that has to think about @@ -30,14 +35,17 @@ * about what an operator may see is stated once, in one place, so that a second * reader of these rows cannot come to state it differently. */ +import { Readable } from 'node:stream' import { OUTCOMES, TRAFFIC_CLASSES } from '@clickmonk/core' import { addressOnly, truncateIp } from '@clickmonk/ipdata' import type { FastifyInstance } from 'fastify' import { z } from 'zod' import type { AdminContext } from './app.js' import { requireCredential } from './auth.js' +import { csvLine } from './csv.js' import { fail } from './http.js' import { + CH_MAX_EXECUTION_SECONDS, MAX_STORE_MS, type ReportWindow, WINDOW_FIELDS, @@ -56,9 +64,10 @@ export const MAX_CLICK_PAGE = 200 * The filters the product promises: link, class, outcome, country and time. * * Exported, with the column list and the row mapper below, so that a second - * reader of these rows — a download of the same log is the one this product - * wants next — filters and maps them with this module's rules rather than with - * its own. This route is the only caller today. + * reader of these rows filters and maps them with this module's rules rather + * than with its own. The export below is that second reader, and it takes these + * fields and no others: a page size and a cursor are how a page of JSON is + * walked and mean nothing to a file. */ export const CLICK_FILTER_FIELDS = { ...WINDOW_FIELDS, @@ -262,6 +271,57 @@ export function clickFilter( return { where: parts.join(' AND '), params } } +/** Rows one export writes at most. */ +export const EXPORT_ROW_CAP = 1_000_000 + +/** + * The CSV's columns, in order, by the same names the JSON uses: one vocabulary + * for one resource in two representations. + * + * Written out rather than taken from an example row, so that a field added to + * `asClick` and forgotten here is a failing test rather than a column that + * quietly stopped being exported. + */ +export const CSV_FIELDS = [ + 'clickId', + 'at', + 'host', + 'path', + 'domainId', + 'linkId', + 'outcome', + 'step', + 'status', + 'destination', + 'targetId', + 'visitorId', + 'returning', + 'country', + 'region', + 'city', + 'geoSource', + 'device', + 'os', + 'browser', + 'asn', + 'class', + 'signals', + 'action', + 'referrer', + 'userAgent', + 'network', + 'capUnchecked', +] as const + +const ExportQuery = z.object({ ...CLICK_FILTER_FIELDS }).strict() + +/** `20260924T000000Z`: a filename with no colons and nothing from the request in it. */ +const stamp = (ms: number): string => + new Date(ms) + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d{3}/, '') + export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): void { /** * One page of the log, newest first. @@ -302,4 +362,118 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo }, ) }) + + /** + * The log as a file. + * + * A year of clicks is a file this process must never hold, so the rows are + * read from the store a block at a time and written out a row at a time. The + * response therefore has no `content-length`: a body with one is a body + * somebody buffered. + * + * **Its own gate, taken by hand.** The handler returns as soon as the stream + * is handed to Fastify, while the body is still being written, so a slot + * released in a `finally` around this handler — which is what `withSlot` does + * — would be given back at the beginning of the export rather than at the end + * and would bound nothing. It is given back in the generator's own `finally`, + * which runs whether the stream ended, failed, or the client hung up. The gate + * is the export's own and not the report gate, so that a download an operator + * started does not lock their dashboard out. + * + * **What bounds the slot.** `max_execution_time` on both statements, and + * nothing else: Fastify's `requestTimeout` does not bound a handler — a + * three-second handler was measured answering after 3,029 ms under a + * one-second setting — and the handler has returned before most of this work + * happens anyway. The slot is held for the probe plus the export, so it is + * held for the sum of the two bounds; a client that reads slowly holds it + * until the store ends the query, which is why the ceiling on how many run at + * once is one. + * + * **Why a probe query.** An export cut at the cap says so in a header, and a + * header has to be decided before the first byte of the body. A trailer would + * be the obvious answer and is not one: `curl` does not show trailers and most + * clients drop them. So one query asks whether there is a row past the cap, + * which stops as soon as it has found cap + 1 rows and therefore costs no more + * than the export it precedes. + */ + app.get('/api/clicks.csv', async (req, reply) => { + requireCredential(req) + const q = readQuery(ExportQuery, req.query) + const w = parseWindow(q, { alignMs: null }) + const ch = requireCh(ctx, req) + const cap = ctx.exportRowCap + + // Every refusal above this line is answered before a slot is taken and + // before a byte of a file exists: a bad field, a window the store cannot + // hold and a missing credential are all JSON, with no `content-disposition` + // on them. + if (!ctx.exportGate.tryEnter()) { + // `return fail(…)`, never a bare call: `fail` is a const arrow, so + // TypeScript does not treat a bare call as ending the path. + return fail(429, 'too_many_exports', 'an export is already running; try again', { + 'retry-after': '1', + }) + } + let started = false + try { + const { where, params } = clickFilter(q, w, null) + const [probe] = await chRows<{ n: string }>({ + ch, + req, + query: `SELECT count() AS n FROM (SELECT 1 FROM clicks FINAL WHERE ${where} LIMIT ${cap + 1})`, + params, + }) + const truncated = Number(probe?.n ?? 0) > cap + + // Not through `chRows`, which waits for every row and hands back an + // array. This one is read a block at a time below, and carries the same + // server-side bound. + const rows = await ch.query({ + query: `SELECT ${CLICK_COLUMNS} FROM clicks FINAL WHERE ${where} + ORDER BY time DESC, click_id DESC LIMIT ${cap}`, + query_params: params, + format: 'JSONEachRow', + clickhouse_settings: { max_execution_time: CH_MAX_EXECUTION_SECONDS }, + }) + + async function* body(): AsyncGenerator { + try { + yield csvLine([...CSV_FIELDS]) + for await (const block of rows.stream()) { + for (const row of block) { + // The same mapper the page of JSON uses, so the file carries the + // network and never the address it was truncated from. + const click = asClick(row.json()) + yield csvLine(CSV_FIELDS.map((f) => click[f])) + } + } + } catch (err) { + // The status and the headers are long gone. Ending the connection + // without a well-formed last line is the only honest thing left: a + // 200 that ends in a complete-looking file with half the rows missing + // is exactly what this endpoint exists not to do. + req.log.error({ err }, 'clickhouse failed while an export was streaming') + throw err + } finally { + ctx.exportGate.leave() + } + } + + reply.header('content-type', 'text/csv; charset=utf-8') + reply.header( + 'content-disposition', + `attachment; filename="clicks-${stamp(w.fromMs)}-${stamp(w.toMs)}.csv"`, + ) + reply.header('x-clickmonk-row-cap', String(cap)) + // Before the body, which is the whole reason for the probe above. + reply.header('x-clickmonk-truncated', truncated ? 'true' : 'false') + started = true + return reply.send(Readable.from(body())) + } finally { + // Only when the generator never took ownership of the slot — the probe + // failed, say. Once the stream is handed over, its own `finally` is what + // gives it back. + if (!started) ctx.exportGate.leave() + } + }) } diff --git a/packages/admin/src/csv.test.ts b/packages/admin/src/csv.test.ts new file mode 100644 index 0000000..2c2eccc --- /dev/null +++ b/packages/admin/src/csv.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' +import { csvCell, csvLine } from './csv.js' + +describe('csvCell', () => { + it.each([ + ['a plain value', 'chrome', '"chrome"'], + ['a number', 302, '"302"'], + ['a boolean', true, '"true"'], + ['nothing at all', null, '""'], + ['nothing at all, the other way', undefined, '""'], + ['an empty string', '', '""'], + ])('quotes %s', (_label, value, expected) => { + expect(csvCell(value)).toBe(expected) + }) + + it('doubles a quote inside a value rather than ending the cell', () => { + expect(csvCell('say "hello"')).toBe('"say ""hello"""') + }) + + it('keeps a comma and a newline inside the quotes', () => { + expect(csvCell('a,b')).toBe('"a,b"') + expect(csvCell('a\nb')).toBe('"a\nb"') + }) + + it('joins a list of tokens with spaces', () => { + expect(csvCell(['ua_bot', 'head'])).toBe('"ua_bot head"') + expect(csvCell([])).toBe('""') + }) + + /** + * A click log is a file of strings a stranger chose: a user agent and a + * referrer come from the request. A spreadsheet treats a cell opening with + * one of these as a formula to run, and quoting does not stop it. The + * apostrophe is visible in a spreadsheet and is part of the value to a + * parser, which is the trade: a visible character beats a cell that runs. + */ + it.each([ + ['=cmd|/c calc', `"'=cmd|/c calc"`], + ['+1+1', `"'+1+1"`], + ['-1+1', `"'-1+1"`], + ['@SUM(A1)', `"'@SUM(A1)"`], + ['\tlead', `"'\tlead"`], + ['\rlead', `"'\rlead"`], + ])('defuses a cell that opens with %s', (value, expected) => { + expect(csvCell(value)).toBe(expected) + }) + + it('leaves a value that merely contains one of those alone', () => { + expect(csvCell('a=b')).toBe('"a=b"') + }) +}) + +describe('csvLine', () => { + it('joins cells with commas and ends the line the way the format says', () => { + expect(csvLine(['a', 1, null])).toBe('"a","1",""\r\n') + }) +}) diff --git a/packages/admin/src/csv.ts b/packages/admin/src/csv.ts new file mode 100644 index 0000000..43146a4 --- /dev/null +++ b/packages/admin/src/csv.ts @@ -0,0 +1,37 @@ +/** + * Writing CSV, by hand, because the alternative is a dependency for two + * functions. + * + * Every cell is quoted, whether it needs to be or not: a writer that decides + * per cell is a writer with a rule to get wrong, and a reader cannot tell the + * difference. RFC 4180 line endings, because the thing most likely to open + * this file is a spreadsheet on Windows. + * + * No byte-order mark. Excel reads one as a hint about encoding; every parser + * that does not reads it as part of the first column's name. + */ + +export const CSV_EOL = '\r\n' + +/** + * A spreadsheet runs a cell that opens with one of these as a formula, and + * quoting does not stop it — the quotes are the CSV's, and the spreadsheet + * strips them before it looks. A click log is made of strings a stranger + * chose: the user agent and the referrer both come from the request. So a cell + * that opens with one is prefixed with an apostrophe, which a spreadsheet shows + * and does not run, and which a parser reads as one more character of the + * value. That is the trade, and it is the right way round for a file an + * operator opens by double-clicking it. + */ +const FORMULA_LEAD = /^[=+\-@\t\r]/ + +export function csvCell(value: unknown): string { + if (value === null || value === undefined) return '""' + const text = Array.isArray(value) ? value.join(' ') : String(value) + const guarded = FORMULA_LEAD.test(text) ? `'${text}` : text + return `"${guarded.replaceAll('"', '""')}"` +} + +export function csvLine(values: unknown[]): string { + return `${values.map(csvCell).join(',')}${CSV_EOL}` +} diff --git a/packages/admin/src/ip-hygiene.test.ts b/packages/admin/src/ip-hygiene.test.ts index d192ea6..011d152 100644 --- a/packages/admin/src/ip-hygiene.test.ts +++ b/packages/admin/src/ip-hygiene.test.ts @@ -38,6 +38,7 @@ const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' // its own. A read added later belongs here. const READS = [ `/api/clicks?${WINDOW}`, + `/api/clicks.csv?${WINDOW}`, `/api/reports/summary?${WINDOW}`, `/api/reports/timeseries?${WINDOW}&bucket=hour`, `/api/reports/breakdown?${WINDOW}&dimension=country`, @@ -148,7 +149,7 @@ describe('no response carries a whole address', () => { expect(r.body).not.toContain(V6_ADDRESS) }) - it('shows the network instead, in both families', async () => { + it('shows the network instead, in both families, in the log and the export', async () => { const log = await app.inject({ method: 'GET', url: `/api/clicks?${WINDOW}`, @@ -156,6 +157,15 @@ describe('no response carries a whole address', () => { }) expect(log.body).toContain('198.51.100.0/24') expect(log.body).toContain('2001:db8:1234:5678::/64') + // The file too, and quoted as a cell: the export maps its rows through the + // same truncation rather than writing the column it selected. + const csv = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(csv.body).toContain('"198.51.100.0/24"') + expect(csv.body).toContain('"2001:db8:1234:5678::/64"') }) it('has no field called ip anywhere in the log', async () => { From bb11e8bd9ea854c351de459e1d522de1c1b2d522 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 18:05:34 +0000 Subject: [PATCH 38/65] admin: a cap the query cannot lie about, and a formula that cannot run --- packages/admin/src/app.ts | 14 ++-- packages/admin/src/clicks.test.ts | 108 ++++++++++++++++++++++++++++++ packages/admin/src/clicks.ts | 51 ++++++++++++-- 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/packages/admin/src/app.ts b/packages/admin/src/app.ts index 511e38b..0a39aa5 100644 --- a/packages/admin/src/app.ts +++ b/packages/admin/src/app.ts @@ -8,7 +8,7 @@ import Fastify, { type FastifyServerOptions, } from 'fastify' import { type Credential, authenticate, checkCsrf, hasBearer } from './auth.js' -import { EXPORT_ROW_CAP, registerClickRoutes } from './clicks.js' +import { EXPORT_ROW_CAP, checkExportRowCap, registerClickRoutes } from './clicks.js' import { registerDomainRoutes } from './domains.js' import { HttpError, MAX_BODY_BYTES, securityHeaders } from './http.js' import { registerKeyRoutes } from './keys.js' @@ -90,9 +90,10 @@ export interface AdminDeps { reportGate?: ConcurrencyGate exportGate?: ConcurrencyGate /** - * Rows one export writes at most. Injectable only so that a test can reach - * the cap with three rows instead of a million; nothing in the product sets - * it. + * Rows one export writes at most. Injectable so that a test can reach the cap + * with three rows instead of a million; nothing in the product sets it today, + * and whatever does will be configuration an operator wrote, which is why it + * is refused at boot rather than trusted — see `checkExportRowCap`. */ exportRowCap?: number /** @@ -167,7 +168,10 @@ export function buildAdminApp( checkGate: deps.checkGate ?? new ConcurrencyGate(DNS_CHECKS_IN_FLIGHT), reportGate: deps.reportGate ?? new ConcurrencyGate(REPORT_QUERIES_IN_FLIGHT), exportGate: deps.exportGate ?? new ConcurrencyGate(EXPORTS_IN_FLIGHT), - exportRowCap: deps.exportRowCap ?? EXPORT_ROW_CAP, + // Checked here rather than where it is interpolated: a bad one is a fault in + // how this service was built, so it belongs at boot with the name of the + // thing that is wrong, not in the 503 every export would answer with. + exportRowCap: checkExportRowCap(deps.exportRowCap ?? EXPORT_ROW_CAP), slugSource: deps.slugSource ?? newSlug, } diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index fd05eaa..d3f3a9b 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -115,6 +115,23 @@ const PAGE_IDS = Array.from( ) const manyInOneInstant = PAGE_IDS.map((id) => click({ click_id: id, time: PAGE_INSTANT })) +/** + * One click whose user agent opens the way a formula does, in a month of its own. + * + * A spreadsheet runs what the file hands it, not what the writer's helper + * returned, so the defusing is read back off a streamed body and not only off + * `csvCell`. A user agent is a string a stranger chose, which is what makes this + * the field to do it with. Its own month, older than every other click in this + * file, so no count and no page boundary above moves. + */ +const FORMULA_ID = '01920000-0000-7000-8000-000000000031' +const FORMULA_UA = '=HYPERLINK("https://example.com","click")' +const formulaClick = click({ + click_id: FORMULA_ID, + time: '2026-07-10 10:00:00.000', + user_agent: FORMULA_UA, +}) + /** A well-formed id for a cursor whose instant is the thing under test. */ const CURSOR_ID = '01920000-0000-7000-8000-0000000000ff' @@ -124,6 +141,7 @@ const WINDOW = 'from=2026-09-24T00:00:00.000Z&to=2026-09-25T00:00:00.000Z' const DAY_BEFORE = 'from=2026-09-23T00:00:00.000Z&to=2026-09-24T00:00:00.000Z' const TIED_DAY = 'from=2026-09-22T00:00:00.000Z&to=2026-09-23T00:00:00.000Z' const PAGE_DAY = 'from=2026-08-15T00:00:00.000Z&to=2026-08-16T00:00:00.000Z' +const FORMULA_DAY = 'from=2026-07-10T00:00:00.000Z&to=2026-07-11T00:00:00.000Z' beforeAll(async () => { await resetDatabases(pool, ch) @@ -196,6 +214,9 @@ beforeAll(async () => { ...tied, // Fifty-one in another month: see the default-page block at the end. ...manyInOneInstant, + // One in a month of its own, whose user agent a spreadsheet would run: see + // the export's formula block at the end. + formulaClick, ]) }) @@ -848,6 +869,93 @@ describe('GET /api/clicks.csv', () => { }) }) +/** + * A value a spreadsheet would run, read back off the file. + * + * The unit tests for `csvCell` say the helper defuses one; this says the row the + * export wrote did. They are different claims, and only the second one is about + * the file an operator double-clicks. Its own window, so the fixture click is the + * only row in it and the whole line can be asserted. + */ +describe('GET /api/clicks.csv, a value a spreadsheet would run', () => { + it('defuses the cell and leaves the value readable in it', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${FORMULA_DAY}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + const out = r.body.split('\r\n').filter((l) => l.length > 0) + expect(out).toHaveLength(2) + // The whole row, written out: the apostrophe in front of the user agent, the + // quotes inside it doubled, and every other cell where it belongs. + expect(out[1]).toBe( + `"${FORMULA_ID}","2026-07-10T10:00:00.000Z","go.example.test","/a","${DOMAIN}","${LINK_A}","target","destination","302","https://example.com/?c=01920000-0000-7000-8000-000000000001","${TARGET}","v1","false","DE","","","dbip","desktop","windows","chrome","64500","human","","","https://blog.example.com/post","'=HYPERLINK(""https://example.com"",""click"")","198.51.100.0/24","false"`, + ) + // And said directly: no cell in the file opens with the formula. Asserted + // after the line above, so an empty body could not satisfy it. + expect(r.body).not.toContain('"=HYPERLINK') + }) +}) + +/** + * The cap, where it is set. + * + * It reaches the query text as a `LIMIT`, so it is refused against the range it is + * bound into and not only against its type — and refused when the app is built, + * which is where a value from configuration arrives. Both ends, with literals, + * for the reason every bound in this file is a literal: one derived from the + * constant it is testing moves when the constant moves and goes on passing. + */ +describe('the export row cap, where it is set', () => { + it.each([ + ['nothing to export', 0], + ['a negative number of rows', -1], + ['a fraction of a row', 2.5], + ['more rows than a file holds', 10_000_001], + ['not a number at all', Number.NaN], + ])('refuses %s, when the app is built', (_label, cap) => { + expect(() => testApp(pool, clock, { ch, exportRowCap: cap })).toThrow(/exportRowCap/) + }) + + it('takes the smallest cap there is, and exports one row', async () => { + const small = testApp(pool, clock, { ch, exportRowCap: 1 }) + try { + const r = await small.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.headers['x-clickmonk-row-cap']).toBe('1') + expect(r.headers['x-clickmonk-truncated']).toBe('true') + expect(r.body.split('\r\n').filter((l) => l.length > 0)).toHaveLength(2) + } finally { + await small.close() + } + }) + + // The other end, and a request through it: the 200 is what says the store took + // `LIMIT 10000000` rather than answering an error the caller would be told was + // an install outage. + it('takes the largest cap there is, and the store takes the query it makes', async () => { + const large = testApp(pool, clock, { ch, exportRowCap: 10_000_000 }) + try { + const r = await large.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.headers['x-clickmonk-row-cap']).toBe('10000000') + expect(r.headers['x-clickmonk-truncated']).toBe('false') + expect(r.body.split('\r\n').filter((l) => l.length > 0)).toHaveLength(4) + } finally { + await large.close() + } + }) +}) + /** * Two addresses the parser reads differently, in a window of their own. * diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 2f321b5..bb943f9 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -271,8 +271,40 @@ export function clickFilter( return { where: parts.join(' AND '), params } } -/** Rows one export writes at most. */ +/** + * Rows one export writes at most, and the largest an install may set it to. + * + * The default sits just under the 1,048,576 rows a spreadsheet will open, which + * is the ceiling that matters to the person the file is for. The other one is a + * file's rather than the store's: ten million rows of click log is tens of + * gigabytes, and an install asking for more has stopped asking for a spreadsheet. + */ export const EXPORT_ROW_CAP = 1_000_000 +export const MAX_EXPORT_ROW_CAP = 10_000_000 + +/** + * The cap, checked against the range it is bound into and not only its type. + * + * It reaches the query text as a `LIMIT`, and the probe's as `LIMIT cap + 1`, so + * it is a caller value with a longer path: it comes from whoever built this + * service, which on an install is configuration, which is a file an operator + * edits. A fractional one is `LIMIT 3.5` and a negative one is a syntax error — + * each one a store error that every export from then on would answer with, and + * that the caller would be told is an install outage rather than a number + * somebody typed. So it is refused here, at boot, naming the thing that is + * wrong, which is the bargain `loadConfig` already makes for a bad port. + * + * The default goes through it too: a range nothing is checked against is a range + * that can quietly stop containing the value it was written for. + */ +export function checkExportRowCap(cap: number): number { + if (!Number.isInteger(cap) || cap < 1 || cap > MAX_EXPORT_ROW_CAP) { + throw new Error( + `exportRowCap: must be a whole number of rows from 1 to ${MAX_EXPORT_ROW_CAP}, not ${cap}`, + ) + } + return cap +} /** * The CSV's columns, in order, by the same names the JSON uses: one vocabulary @@ -448,10 +480,19 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo } } } catch (err) { - // The status and the headers are long gone. Ending the connection - // without a well-formed last line is the only honest thing left: a - // 200 that ends in a complete-looking file with half the rows missing - // is exactly what this endpoint exists not to do. + // **A download cannot be retracted.** The 200 and every header went + // out before the first row, so there is no status left to change and + // nothing honest to append: what the caller gets is a truncated file + // and a connection that ended mid-line, and what they see is a broken + // download. Appending a well-formed last line would be worse — a file + // that looks complete with half the rows missing is the one thing this + // endpoint exists not to produce. + // + // So this line is the only record that the file is short, which is why + // it is at error level: an operator who reads it counts the rows, and + // one who never sees it trusts a file that ends early. A failure + // *before* the first byte is a different case — the probe's is a 503, + // and the stream query's own is the error handler's 500. req.log.error({ err }, 'clickhouse failed while an export was streaming') throw err } finally { From ab9b3ba1b98924f27b63ea65cc5ecc9db4db7e0b Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 18:47:04 +0000 Subject: [PATCH 39/65] admin: a slot a cancelled download gives back --- packages/admin/src/clicks.test.ts | 84 +++++++++++++++++++++++++++++++ packages/admin/src/clicks.ts | 77 ++++++++++++++++++++++++---- 2 files changed, 150 insertions(+), 11 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index d3f3a9b..91d92ea 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -1,3 +1,4 @@ +import { request as httpRequest } from 'node:http' import { ClickHouseLogLevel } from '@clickhouse/client' import { ConcurrencyGate } from '@clickmonk/core' import { type ClickHouseClient, createChClient } from '@clickmonk/db' @@ -869,6 +870,89 @@ describe('GET /api/clicks.csv', () => { }) }) +/** + * A gate that also counts the slots it handed out, so that a test about giving one + * back can say the export took one in the first place. A case that asserts an + * empty gate without that is a case an unserved request satisfies. + */ +class CountingGate extends ConcurrencyGate { + entered = 0 + override tryEnter(): boolean { + const ok = super.tryEnter() + if (ok) this.entered++ + return ok + } +} + +/** + * A caller that hangs up, over a real socket. + * + * `inject` cannot ask this question: it has no socket to close, and the question + * is what the server does when the peer goes away. `node:http` rather than + * `fetch`, because the host guard reads the `Host` header and undici does not let + * a caller set it. + * + * The timings are the ones that mattered: a hangup in the first few milliseconds + * lands before the first row is pulled, which is the window where the generator's + * `finally` never runs because the generator never started — and where, with one + * export slot in the process, one request and an immediate close used to take the + * endpoint out until a restart. + */ +describe('GET /api/clicks.csv, a caller that hangs up', () => { + const settle = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) + + /** Waits until the export holds the slot, so no case here is vacuous. */ + const untilEntered = async (gate: CountingGate): Promise => { + for (let waited = 0; waited < 2000; waited += 5) { + if (gate.entered > 0) return + await settle(5) + } + throw new Error('the export never took a slot, so this case would prove nothing') + } + + it.each([0, 1, 2, 5, 10, 25])( + 'gives the slot back when the caller hangs up %ims after the export took it', + async (ms) => { + const gate = new CountingGate(1) + const served = testApp(pool, clock, { ch, exportGate: gate }) + try { + await served.listen({ host: '127.0.0.1', port: 0 }) + const at = served.server.address() + const port = typeof at === 'object' && at !== null ? at.port : 0 + const req = httpRequest({ + host: '127.0.0.1', + port, + path: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST, cookie }, + }) + // The hangup is the arrangement, so the socket errors it causes are not a + // failure of anything. + req.on('error', () => {}) + req.end() + await untilEntered(gate) + await settle(ms) + req.destroy() + // Long enough for the server to notice the closed socket and destroy the + // stream. A fixed wait rather than a poll: a loop that waited for the gate + // to empty would pass whatever the release took, including never. + await settle(250) + expect(gate.entered).toBe(1) + expect(gate.stats().inFlight).toBe(0) + // And the consequence an operator would see: with a single slot, one + // leaked slot answers every later export 429 until the process restarts. + const again = await served.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(again.statusCode).toBe(200) + } finally { + await served.close() + } + }, + ) +}) + /** * A value a spreadsheet would run, read back off the file. * diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index bb943f9..e88fb91 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -407,10 +407,10 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo * is handed to Fastify, while the body is still being written, so a slot * released in a `finally` around this handler — which is what `withSlot` does * — would be given back at the beginning of the export rather than at the end - * and would bound nothing. It is given back in the generator's own `finally`, - * which runs whether the stream ended, failed, or the client hung up. The gate - * is the export's own and not the report gate, so that a download an operator - * started does not lock their dashboard out. + * and would bound nothing. It is given back by `release` below, which every + * path calls and which runs once. The gate is the export's own and not the + * report gate, so that a download an operator started does not lock their + * dashboard out. * * **What bounds the slot.** `max_execution_time` on both statements, and * nothing else: Fastify's `requestTimeout` does not bound a handler — a @@ -447,6 +447,39 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo }) } let started = false + let released = false + /** + * The result set once there is one, as the one thing that has to be closed. + * Typed by the method called on it rather than by the client's own result + * type, which nothing else in this service names. + */ + let result: { close: () => void } | null = null + + /** + * Gives the slot back, once, from whichever path reaches it first. + * + * **Never conditioned on the body having been entered**, and this is the + * whole point of it. When a caller hangs up before the first row is pulled, + * Fastify destroys the stream and Node calls `return()` on a generator that + * has not run a line — which completes it *without* running its `finally`. + * A release that lived only there would therefore never happen, while + * `started` had already told the handler's own `finally` to skip; both paths + * skipped, and with one export slot in the process a single request and an + * immediate close took the endpoint out until a restart. So the stream's + * `close` and `error` release too, and this runs once because on an ordinary + * export all three of them fire. + * + * Closing the result set is insurance rather than a measured need: whether + * the store keeps working on a query nobody is reading was not established, + * and closing one whose rows are already exhausted costs nothing. + */ + const release = (): void => { + if (released) return + released = true + result?.close() + ctx.exportGate.leave() + } + try { const { where, params } = clickFilter(q, w, null) const [probe] = await chRows<{ n: string }>({ @@ -467,11 +500,26 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo format: 'JSONEachRow', clickhouse_settings: { max_execution_time: CH_MAX_EXECUTION_SECONDS }, }) + result = rows + /** + * The blocks, taken here rather than inside the generator, so that + * `release` always has a pipeline to close: closing a result set nobody has + * streamed destroys its response stream directly, and a destroyed stream + * with nothing listening for an error takes the process down instead of + * ending an export. Once this exists, the close is handled inside the + * client's own pipeline. + */ + const blocks = rows.stream() + // The error is handled where the rows are read, in the generator's `catch`. + // This listener is for the case where nobody is reading them — a result set + // closed after the caller hung up — which would otherwise be an uncaught + // exception rather than a download that stopped. + blocks.on('error', () => {}) async function* body(): AsyncGenerator { try { yield csvLine([...CSV_FIELDS]) - for await (const block of rows.stream()) { + for await (const block of blocks) { for (const row of block) { // The same mapper the page of JSON uses, so the file carries the // network and never the address it was truncated from. @@ -496,7 +544,7 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo req.log.error({ err }, 'clickhouse failed while an export was streaming') throw err } finally { - ctx.exportGate.leave() + release() } } @@ -508,13 +556,20 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo reply.header('x-clickmonk-row-cap', String(cap)) // Before the body, which is the whole reason for the probe above. reply.header('x-clickmonk-truncated', truncated ? 'true' : 'false') + const file = Readable.from(body()) + // Both events, and neither is spare: `close` is what a stream destroyed + // before it was ever read emits, and it is the only signal on that path; + // `error` is a stream that failed, which on a hangup can arrive first. + file.on('close', release) + file.on('error', release) started = true - return reply.send(Readable.from(body())) + return reply.send(file) } finally { - // Only when the generator never took ownership of the slot — the probe - // failed, say. Once the stream is handed over, its own `finally` is what - // gives it back. - if (!started) ctx.exportGate.leave() + // Only when nothing downstream can release it any more — the probe failed, + // say, and there is no stream. Once the stream exists, its own `close` + // gives the slot back, and releasing here would give it back at the + // beginning of the export. + if (!started) release() } }) } From 55ac4f5bee002d8e8853aefff3a6adeaf016f5af Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 18:53:12 +0000 Subject: [PATCH 40/65] admin: the guards nothing was watching, and five sentences that are not true --- packages/admin/src/clicks.test.ts | 194 +++++++++++++++++++++++++++--- packages/admin/src/clicks.ts | 30 +++-- packages/admin/src/csv.test.ts | 5 + packages/admin/src/csv.ts | 9 +- 4 files changed, 212 insertions(+), 26 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 91d92ea..0a294f6 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -1,10 +1,12 @@ import { request as httpRequest } from 'node:http' +import { Readable } from 'node:stream' import { ClickHouseLogLevel } from '@clickhouse/client' import { ConcurrencyGate } from '@clickmonk/core' import { type ClickHouseClient, createChClient } from '@clickmonk/db' import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import type { FastifyInstance } from 'fastify' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { CSV_FIELDS, type ClickRow, asClick } from './clicks.js' import { ADMIN_HOST, clockFrom, read, signedIn, testApp } from './testing.js' const pool = testPg() @@ -98,8 +100,9 @@ const TIED_TIMES = [ const tied = TIED_IDS.map((id, i) => click({ click_id: id, time: TIED_TIMES[i] })) /** - * Fifty-one clicks in one instant, in a month of its own and older than every - * other click in this file. + * Fifty-one clicks in one instant, in a month of its own. Older than every click + * in the windows above, and newer than the one in the formula block below, which + * is the oldest in the file. * * The page size a caller gets when they name none cannot be read off a window of * three or six clicks: any default of three or more answers every assertion in @@ -682,14 +685,17 @@ describe('GET /api/clicks.csv', () => { '01920000-0000-7000-8000-000000000002', '01920000-0000-7000-8000-000000000001', ]) - // The bot click, whole: a list of signals joined into one cell, four columns - // that are empty because the click had nothing in them, and no network - // because its address column was blanked. + // The bot click, whole: a list of signals joined into one cell, and seven + // empty ones — the four this click had nothing in (destination, target, + // country, geo source), the two no click carries yet (region, city), and the + // network, because its address column was blanked. expect(out[1]).toBe( `"01920000-0000-7000-8000-000000000003","2026-09-24T10:02:00.000Z","go.example.test","/a","${DOMAIN}","${LINK_A}","blocked","classify","403","","","v1","false","","","","","desktop","windows","chrome","64500","bot","ua_bot head","block","https://blog.example.com/post","Mozilla/5.0 (Windows NT 10.0) Chrome/130","","true"`, ) - // The oldest click, whole: the network the address was truncated to, the - // instant as the JSON spells it, and every other column at a real value. + // The oldest click in this window, whole: the network the address was + // truncated to, the instant as the JSON spells it, and a real value in every + // column but four — region and city, which no click carries yet, and the + // signals and action a click nothing was decided about has none of. expect(out[3]).toBe( `"01920000-0000-7000-8000-000000000001","2026-09-24T10:00:00.000Z","go.example.test","/a","${DOMAIN}","${LINK_A}","target","destination","302","https://example.com/?c=01920000-0000-7000-8000-000000000001","${TARGET}","v1","false","DE","","","dbip","desktop","windows","chrome","64500","human","","","https://blog.example.com/post","Mozilla/5.0 (Windows NT 10.0) Chrome/130","198.51.100.0/24","false"`, ) @@ -1073,6 +1079,37 @@ describe('GET /api/clicks, addresses a proxy may have spelled oddly', () => { ['01920000-0000-7000-8000-000000000004', '2001:db8:1234:5678::/64'], ]) }) + + /** + * The same two rows through the file, which is cover rather than a second + * check: the export maps with `asClick`, so it cannot answer differently + * without the page answering differently too. It is here because this is the + * surface where a wrong answer is a privacy incident, and because "it goes + * through the same mapper" is a claim about the code that a test should not + * have to take on trust. + */ + it('exports the same two, as a network and as nothing', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks.csv?${DAY_BEFORE}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + const rows = r.body + .split('\r\n') + .filter((l) => l.length > 0) + .slice(1) + expect(rows).toHaveLength(2) + // The id and the network of each, read as cells rather than as substrings of + // the whole body: a `toContain` on `2001:db8:1234:5678::/64` would pass on a + // file that had written it into the wrong row. + expect(rows.map((l) => [l.slice(1, l.indexOf('","')), l.split(',').at(-2)])).toEqual([ + ['01920000-0000-7000-8000-000000000005', '""'], + ['01920000-0000-7000-8000-000000000004', '"2001:db8:1234:5678::/64"'], + ]) + // And the whole address neither of them may carry. + expect(r.body).not.toContain('2001:db8:1234:5678:9abc:def0:1234:5678') + }) }) /** @@ -1274,16 +1311,19 @@ describe('the log and the export when ClickHouse is not there', () => { }) /** - * The export asks for the credential before it takes a slot and before it - * reads anything, and this is what pins it there. + * The export asks for the credential before it takes a slot, and this is what + * pins it there rather than merely pinning that it asks at all. + * + * **Two positions on this route, not three.** The gate comes before the query + * here, so an unauthenticated request whose credential check has been moved + * anywhere below `tryEnter` is answered 429 by the full gate — after the probe + * or before it, the answer is the same. Only asked for first is it 401. The + * three-answer shape belongs to the report routes, where the query is inside + * the gate; the numbers depend on the order a route actually has and are worth + * measuring rather than copying. * - * Three positions, three answers, and the 401 beside the other export tests - * can only see the first of them. With the export gate full and the store - * unreachable: asked for after the probe, an unauthenticated request scans the - * window it chose and gets a 503; asked for inside the gate but before the - * probe, it gets a 429; only asked for before both is the answer 401. The - * query-failure line is asserted absent as well, because a status alone cannot - * tell a refusal from a query that happened to fail. + * The query-failure line is asserted absent as well, because a status alone + * cannot tell a refusal from a query that happened to fail. */ it('refuses an export without a credential before taking a slot or reading anything', async () => { const lines: string[] = [] @@ -1312,4 +1352,126 @@ describe('the log and the export when ClickHouse is not there', () => { await on.close() } }) + + /** + * The gate comes before the store read, and this is the case that says so. + * + * The same app as above and a credential this time: a full gate and an + * unreachable store. Refused in front of the probe, the answer is 429 and + * nothing was read; refused behind it, the probe fails first and the answer is + * 503 with a query-failure line — which is a refused export that has already + * scanned a window the caller chose, and a flood of them is a scan each. The + * log line is what separates the two, because a 429 could also be a 429 that + * happened after a successful read. + */ + it('refuses a full gate before reading anything at all', async () => { + const lines: string[] = [] + const on = testApp(pool, clock, { + ch: dead, + exportGate: new ConcurrencyGate(0), + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + const r = await on.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(429) + expect(r.json().error).toBe('too_many_exports') + expect(lines.filter((l) => l.includes('clickhouse query failed'))).toEqual([]) + } finally { + await on.close() + } + }) +}) + +/** + * The file's columns against the mapper's fields. + * + * Two lists written in two places for one resource, so the failure to design + * against is a field added to `asClick` — the next geo column, say — that the + * export silently stops carrying. Nothing in the export's own tests can see that: + * the header row and the values both come from `CSV_FIELDS`, so a file missing a + * column is internally consistent and every assertion about it passes. + * + * The two sides come from different places, which is the point: the keys the + * mapper actually returns, against the list the file is written from. + */ +describe('the export columns and the fields a click has', () => { + it('names every field the log maps, and no others', () => { + // The fixture row as `JSONEachRow` hands it over: the stored column names, + // with `time` replaced by the millisecond form the column list selects. The + // leftover `time` key is ignored by the mapper and by this comparison. + const row = { ...click(), at_ms: '1758708000000' } as unknown as ClickRow + expect([...CSV_FIELDS].sort()).toEqual(Object.keys(asClick(row)).sort()) + }) +}) + +/** + * A store that stops answering partway through a file. + * + * A stub client, because there is no way to ask a healthy ClickHouse to fail in + * the middle of a result set — and the line under test is the only record an + * operator ever gets that a file they downloaded is short. The stub answers the + * probe, hands over one row, and then fails. + */ +describe('GET /api/clicks.csv, a store that fails mid-file', () => { + const failingCh = (row: Record): ClickHouseClient => + ({ + query: async ({ query }: { query: string }) => + query.includes('count()') + ? { json: async () => [{ n: '1' }] } + : { + close: () => {}, + stream: () => + Readable.from( + (async function* () { + yield [{ json: () => row }] + throw new Error('the store stopped answering') + })(), + { objectMode: true }, + ), + }, + }) as unknown as ClickHouseClient + + it('logs that the file is short, and gives the slot back', async () => { + const lines: string[] = [] + const gate = new CountingGate(1) + const on = testApp(pool, clock, { + ch: failingCh({ ...click(), at_ms: '1758708000000' }), + exportGate: gate, + log: { + level: 'error', + stream: { + write(line: string) { + lines.push(line) + }, + }, + }, + }) + try { + // The body is what a caller would be left holding, so a rejected inject is + // as much of an answer as a resolved one: what is asserted is the log line + // and the slot, which are what the server owes either way. + await on + .inject({ method: 'GET', url: `/api/clicks.csv?${WINDOW}`, headers: read(cookie) }) + .then(() => undefined) + .catch(() => undefined) + expect(gate.entered).toBe(1) + expect( + lines.filter((l) => l.includes('clickhouse failed while an export was streaming')), + ).toHaveLength(1) + expect(gate.stats().inFlight).toBe(0) + } finally { + await on.close() + } + }) }) diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index e88fb91..2f4f9e7 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -310,9 +310,10 @@ export function checkExportRowCap(cap: number): number { * The CSV's columns, in order, by the same names the JSON uses: one vocabulary * for one resource in two representations. * - * Written out rather than taken from an example row, so that a field added to - * `asClick` and forgotten here is a failing test rather than a column that - * quietly stopped being exported. + * Written out rather than taken from an example row, and held to `asClick` by a + * test that compares this list against the keys that mapper returns: a field + * added to one and forgotten in the other is then a failure rather than a column + * that quietly stopped being exported. */ export const CSV_FIELDS = [ 'clickId', @@ -347,7 +348,16 @@ export const CSV_FIELDS = [ const ExportQuery = z.object({ ...CLICK_FILTER_FIELDS }).strict() -/** `20260924T000000Z`: a filename with no colons and nothing from the request in it. */ +/** + * `20260924T000000Z`: the instant as digits, with the colons a filename cannot + * carry taken out. + * + * The filename is the window the caller asked for, so it is not free of the + * request — what it is free of is anything the caller chose the *shape* of. Both + * halves are numbers by the time they reach here: `parseWindow` has turned the + * caller's text into milliseconds, and this formats those, so the only characters + * that can appear are the ones `toISOString` produces. + */ const stamp = (ms: number): string => new Date(ms) .toISOString() @@ -530,11 +540,13 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo } catch (err) { // **A download cannot be retracted.** The 200 and every header went // out before the first row, so there is no status left to change and - // nothing honest to append: what the caller gets is a truncated file - // and a connection that ended mid-line, and what they see is a broken - // download. Appending a well-formed last line would be worse — a file - // that looks complete with half the rows missing is the one thing this - // endpoint exists not to produce. + // nothing honest to append. What the caller is left with is a file of + // whole, well-formed lines that stops early — the rows written so far, + // each one complete — inside a chunked body that never got its + // terminating chunk, which is what makes the transfer an error the + // client can see rather than a short file it cannot. Appending a last + // line would throw that away: a file that ends cleanly with half the + // rows missing is the one thing this endpoint exists not to produce. // // So this line is the only record that the file is short, which is why // it is at error level: an operator who reads it counts the rows, and diff --git a/packages/admin/src/csv.test.ts b/packages/admin/src/csv.test.ts index 2c2eccc..55c4eec 100644 --- a/packages/admin/src/csv.test.ts +++ b/packages/admin/src/csv.test.ts @@ -39,8 +39,13 @@ describe('csvCell', () => { ['+1+1', `"'+1+1"`], ['-1+1', `"'-1+1"`], ['@SUM(A1)', `"'@SUM(A1)"`], + // The three whitespace leads together: a spreadsheet skips them before it + // decides whether the cell is a formula, so each one hides an `=` from a + // check that reads only the first character. ['\tlead', `"'\tlead"`], ['\rlead', `"'\rlead"`], + ['\nlead', `"'\nlead"`], + ['\t=1+1', `"'\t=1+1"`], ])('defuses a cell that opens with %s', (value, expected) => { expect(csvCell(value)).toBe(expected) }) diff --git a/packages/admin/src/csv.ts b/packages/admin/src/csv.ts index 43146a4..addce2a 100644 --- a/packages/admin/src/csv.ts +++ b/packages/admin/src/csv.ts @@ -22,8 +22,15 @@ export const CSV_EOL = '\r\n' * and does not run, and which a parser reads as one more character of the * value. That is the trade, and it is the right way round for a file an * operator opens by double-clicking it. + * + * The three whitespace characters are in the list for one reason: a spreadsheet + * skips leading whitespace before it decides whether a cell is a formula, so a + * tab, a carriage return or a newline in front of an `=` hides it from a check + * that only looks at the first character. All three, not two of them — a list + * missing one of a set that is here for a single reason is an asymmetry somebody + * will later read as a decision. */ -const FORMULA_LEAD = /^[=+\-@\t\r]/ +const FORMULA_LEAD = /^[=+\-@\t\r\n]/ export function csvCell(value: unknown): string { if (value === null || value === undefined) return '""' From 05387858b82e203e327bfab87efdbb0cfcfa06a6 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 18:56:21 +0000 Subject: [PATCH 41/65] admin: a release counted, not assumed --- packages/admin/src/clicks.test.ts | 44 ++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 0a294f6..6e59166 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -877,17 +877,27 @@ describe('GET /api/clicks.csv', () => { }) /** - * A gate that also counts the slots it handed out, so that a test about giving one - * back can say the export took one in the first place. A case that asserts an - * empty gate without that is a case an unserved request satisfies. + * A gate that counts the slots it handed out and the times they were given back. + * + * The first count is so that a test about giving a slot back can say the export + * took one: a case asserting an empty gate without it is a case an unserved + * request satisfies. The second is because the release is called from three + * places and the gate itself cannot tell — `leave()` on an empty gate does + * nothing, so a release that ran twice is invisible in `inFlight` until two + * exports run at once and one of them gives the other's slot away. */ class CountingGate extends ConcurrencyGate { entered = 0 + left = 0 override tryEnter(): boolean { const ok = super.tryEnter() if (ok) this.entered++ return ok } + override leave(): void { + this.left++ + super.leave() + } } /** @@ -907,6 +917,34 @@ class CountingGate extends ConcurrencyGate { describe('GET /api/clicks.csv, a caller that hangs up', () => { const settle = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) + /** + * The ordinary export, for the count rather than the file. + * + * Three paths call the release — the generator's `finally` and the stream's + * `close` and `error` — and on an export that simply finishes, two of them do. + * The gate cannot notice the second: `leave()` on an empty gate does nothing. + * So what is asserted is the number of calls, because a release that runs twice + * is a slot handed back that was never taken, and with two exports allowed at + * once that is one of them freeing the other's. + */ + it('gives the slot back exactly once when the export finishes', async () => { + const gate = new CountingGate(1) + const one = testApp(pool, clock, { ch, exportGate: gate }) + try { + const r = await one.inject({ + method: 'GET', + url: `/api/clicks.csv?${WINDOW}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(gate.entered).toBe(1) + expect(gate.left).toBe(1) + expect(gate.stats().inFlight).toBe(0) + } finally { + await one.close() + } + }) + /** Waits until the export holds the slot, so no case here is vacuous. */ const untilEntered = async (gate: CountingGate): Promise => { for (let waited = 0; waited < 2000; waited += 5) { From e94b9130b11805f65b73c341e2fb7fde4866f433 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 18:57:59 +0000 Subject: [PATCH 42/65] admin: a count inject could not take --- packages/admin/src/clicks.test.ts | 51 +++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 6e59166..b30707c 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -918,25 +918,52 @@ describe('GET /api/clicks.csv, a caller that hangs up', () => { const settle = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) /** - * The ordinary export, for the count rather than the file. + * An export read to the end over a real socket, for the count rather than the + * file. * * Three paths call the release — the generator's `finally` and the stream's - * `close` and `error` — and on an export that simply finishes, two of them do. - * The gate cannot notice the second: `leave()` on an empty gate does nothing. - * So what is asserted is the number of calls, because a release that runs twice - * is a slot handed back that was never taken, and with two exports allowed at - * once that is one of them freeing the other's. + * `close` and `error` — and an export that simply finishes takes two of them. + * The gate cannot notice the second: `leave()` on an empty gate does nothing, + * so a release that ran twice is invisible in `inFlight` until two exports run + * at once and one of them hands back the other's slot. What is asserted is + * therefore the number of calls. + * + * Over a socket and not through `inject`, which was measured calling it once: + * `inject` has no connection to close, so the stream's `close` never reaches + * this. An idempotence this file pinned through `inject` would be an assertion + * about light-my-request. */ - it('gives the slot back exactly once when the export finishes', async () => { + it('gives the slot back exactly once when the export is read to the end', async () => { const gate = new CountingGate(1) const one = testApp(pool, clock, { ch, exportGate: gate }) try { - const r = await one.inject({ - method: 'GET', - url: `/api/clicks.csv?${WINDOW}`, - headers: read(cookie), + await one.listen({ host: '127.0.0.1', port: 0 }) + const at = one.server.address() + const port = typeof at === 'object' && at !== null ? at.port : 0 + const body = await new Promise((resolve, reject) => { + const req = httpRequest( + { + host: '127.0.0.1', + port, + path: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST, cookie }, + }, + (res) => { + let text = '' + res.setEncoding('utf8') + res.on('data', (chunk: string) => { + text += chunk + }) + res.on('end', () => resolve(text)) + }, + ) + req.on('error', reject) + req.end() }) - expect(r.statusCode).toBe(200) + // The whole file came down the socket, which nothing else here checks: + // every other export in this suite is read through `inject`. + expect(body.split('\r\n').filter((l) => l.length > 0)).toHaveLength(4) + await settle(100) expect(gate.entered).toBe(1) expect(gate.left).toBe(1) expect(gate.stats().inFlight).toBe(0) From 315543453a073b2d5da403ac54bb2b084cf0dd80 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:10:55 +0000 Subject: [PATCH 43/65] worker: drop clicks and addresses past their retention --- .env.example | 9 + README.md | 10 + docker-compose.yml | 3 + packages/worker/src/config.test.ts | 13 + packages/worker/src/config.ts | 6 + packages/worker/src/index.ts | 6 +- packages/worker/src/retention.test.ts | 378 ++++++++++++++++++++++++++ packages/worker/src/retention.ts | 281 +++++++++++++++++++ packages/worker/src/settings.ts | 36 ++- 9 files changed, 740 insertions(+), 2 deletions(-) create mode 100644 packages/worker/src/retention.test.ts create mode 100644 packages/worker/src/retention.ts diff --git a/.env.example b/.env.example index e5442c9..7901a6f 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,15 @@ CLICKMONK_DNS_CHECK=on # Which resolvers to ask, comma separated, each an address and optionally a # port ([2001:db8::1]:5353 for IPv6). Empty means the host's own. CLICKMONK_DNS_SERVERS= +# How often the worker looks for clicks and addresses past their retention +# periods. An hour is plenty; shorten it only if you are watching it work. The +# periods themselves are settings on the install, not here: +# clickmonk settings set --keep-clicks 30 --keep-addresses 7 +# Each period is a floor rather than a deadline. Clicks are deleted a whole +# month at a time, so 30 days keeps between 30 and 61 days of them, and the +# same goes for how long an address stays on a click. Running this more often +# does not change that. +CLICKMONK_RETENTION_INTERVAL_MS=3600000 # Where X-Forwarded-For is believed from: comma-separated addresses, ranges # such as 172.17.0.0/16, or loopback, linklocal and uniquelocal. The default # names the private and loopback ranges, which is where Caddy is: it is the diff --git a/README.md b/README.md index 553d121..1005e89 100644 --- a/README.md +++ b/README.md @@ -329,6 +329,16 @@ periods cannot be read — and nothing is deleted while it reads that way. A mis the likelier of the two: it is one `DELETE` away, and the defaults would answer it with a period nobody chose. +The worker enforces both periods once an hour, and **each one is a floor rather than a +deadline.** Clicks are stored a month at a time and deleted the same way, so a month of +them goes only once its newest possible click is past the period: 90 days keeps between 90 +and 121 days, and 30 days of addresses keeps them between 30 and 61. Lowering a period +takes effect on the next pass and never rewrites anything already deleted. What is deleted +is the raw clicks and the addresses on them; the hourly totals every report but the click +log reads are kept, which is why a chart still answers for a window older than the clicks +behind it. A blanked address leaves an empty network in an export, the same as an address +that could never be looked up. + For a script, mint an API key and send it as `Authorization: Bearer …`: ```sh diff --git a/docker-compose.yml b/docker-compose.yml index 6bf79b3..d8e93eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,6 +116,9 @@ services: CLICKMONK_IPDATA_UPDATE: ${CLICKMONK_IPDATA_UPDATE:-on} CLICKMONK_DNS_CHECK: ${CLICKMONK_DNS_CHECK:-on} CLICKMONK_DNS_SERVERS: ${CLICKMONK_DNS_SERVERS:-} + # An hour by default. The periods themselves are settings on the install, + # not here: `clickmonk settings set --keep-clicks 30`. + CLICKMONK_RETENTION_INTERVAL_MS: ${CLICKMONK_RETENTION_INTERVAL_MS:-3600000} # The worker itself never reads this. The CLI runs in this container, and # `domain add` refuses a link domain equal to the admin host name: a # domain under that name would take its requests to the admin API and diff --git a/packages/worker/src/config.test.ts b/packages/worker/src/config.test.ts index 7b2ee62..ea06180 100644 --- a/packages/worker/src/config.test.ts +++ b/packages/worker/src/config.test.ts @@ -57,6 +57,19 @@ describe('loadConfig', () => { ]) }) + it('looks for clicks past their retention every hour', () => { + expect(loadConfig(base).retentionIntervalMs).toBe(3_600_000) + }) + + it('refuses a retention interval under a second', () => { + expect(() => loadConfig({ ...base, CLICKMONK_RETENTION_INTERVAL_MS: '999' })).toThrow( + /CLICKMONK_RETENTION_INTERVAL_MS/, + ) + expect( + loadConfig({ ...base, CLICKMONK_RETENTION_INTERVAL_MS: '1000' }).retentionIntervalMs, + ).toBe(1000) + }) + it('turns the checks off, and refuses any value but on or off', () => { expect(loadConfig({ ...base, CLICKMONK_DNS_CHECK: 'off' }).dnsCheck).toBe(false) expect(() => loadConfig({ ...base, CLICKMONK_DNS_CHECK: 'no' })).toThrow(/CLICKMONK_DNS_CHECK/) diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index d552ba8..b07eb79 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -42,6 +42,10 @@ const Schema = z.object({ // validated here rather than inside node:dns, which throws at the first // query with a message that names nothing. CLICKMONK_DNS_SERVERS: DnsServers, + // How often the retention pass runs. There is no on/off here on purpose: + // "never" is a value the setting takes, and two ways to turn one thing off + // is how they come to disagree. + CLICKMONK_RETENTION_INTERVAL_MS: z.coerce.number().int().min(1000).default(3_600_000), }) export interface WorkerConfig { @@ -53,6 +57,7 @@ export interface WorkerConfig { dnsCheck: boolean dnsCheckIntervalMs: number dnsServers: string[] + retentionIntervalMs: number } export function loadConfig(env: NodeJS.ProcessEnv): WorkerConfig { @@ -73,5 +78,6 @@ export function loadConfig(env: NodeJS.ProcessEnv): WorkerConfig { dnsCheck: e.CLICKMONK_DNS_CHECK === 'on', dnsCheckIntervalMs: e.CLICKMONK_DNS_CHECK_INTERVAL_MS, dnsServers: e.CLICKMONK_DNS_SERVERS, + retentionIntervalMs: e.CLICKMONK_RETENTION_INTERVAL_MS, } } diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 6d1bf27..b595206 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -3,6 +3,7 @@ import { createChClient, createPgPool, migrateToLatest } from '@clickmonk/db' import { startUpdater } from '@clickmonk/ipdata' import { loadConfig } from './config.js' import { createResolver, startDomainChecker } from './domains.js' +import { startRetention } from './retention.js' import { startShipper } from './shipper.js' const config = loadConfig(process.env) @@ -14,11 +15,12 @@ let stopping = false let shipper: { stop(): Promise } | null = null let updater: { stop(): Promise } | null = null let domains: { stop(): Promise } | null = null +let retention: { stop(): Promise } | null = null async function shutdown(): Promise { if (stopping) return stopping = true - await Promise.all([shipper?.stop(), updater?.stop(), domains?.stop()]) + await Promise.all([shipper?.stop(), updater?.stop(), domains?.stop(), retention?.stop()]) await Promise.allSettled([pg.end(), ch.close()]) } process.on('SIGTERM', () => void shutdown()) @@ -59,4 +61,6 @@ if (!stopping) { } else { log('domain DNS checks are off: a domain stays as it was added') } + retention = startRetention({ pg, ch, intervalMs: config.retentionIntervalMs, log }) + log('deleting raw clicks and addresses past their retention periods') } diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts new file mode 100644 index 0000000..d211dbe --- /dev/null +++ b/packages/worker/src/retention.test.ts @@ -0,0 +1,378 @@ +import type { ClickHouseClient, Pool } from '@clickmonk/db' +import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { partitionEndMs, runRetention, startRetention } from './retention.js' + +const pool: Pool = testPg() +const ch: ClickHouseClient = testCh() + +/** + * The instant every pass below is run at. Fixed, and the fixtures are fixed + * around it, because the whole of this module is arithmetic on months and a + * fixture relative to the real now would cross a month boundary and change + * which partitions the answer names. + */ +const NOW = new Date('2026-09-24T12:00:00.000Z') + +const click = (id: string, time: string, ip: string): Record => ({ + click_id: id, + time, + host: 'go.example.test', + path: '/a', + domain_id: '00000000-0000-4000-8000-00000000000d', + link_id: '00000000-0000-4000-8000-0000000000a1', + outcome: 'target', + step: 'destination', + status: 302, + destination: 'https://example.com/', + target_id: '', + visitor_id: 'v1', + returning: 0, + country: 'DE', + region: '', + city: '', + geo_source: 'dbip', + device: 'desktop', + user_agent: 'ua', + referrer: '', + ip, + cap_unchecked: 0, + traffic_class: 'human', + signals: [], + action: '', + os: 'windows', + browser: 'chrome', + asn: 64500, +}) + +async function rows(query: string): Promise { + const rs = await ch.query({ query, format: 'JSONEachRow' }) + return rs.json() +} + +const partitions = async (): Promise => { + const r = await rows<{ partition_id: string }>( + "SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'clicks' AND active ORDER BY partition_id", + ) + return r.map((p) => p.partition_id) +} + +const addresses = async (): Promise<{ partition_id: string; ip: string }[]> => + rows('SELECT _partition_id AS partition_id, ip FROM clicks FINAL ORDER BY partition_id, click_id') + +const rollupClicks = async (): Promise => { + const r = await rows<{ n: string }>('SELECT uniqExactMerge(clicks_state) AS n FROM clicks_hourly') + return Number(r[0]?.n ?? 0) +} + +/** Waits for every mutation on `clicks` to finish. Bounded, and never a sleep of hope. */ +async function settleMutations(): Promise { + for (let i = 0; i < 100; i++) { + const r = await rows<{ n: string }>( + "SELECT count() AS n FROM system.mutations WHERE database = currentDatabase() AND table = 'clicks' AND NOT is_done", + ) + if (Number(r[0]?.n ?? 0) === 0) return + await new Promise((resolve) => setTimeout(resolve, 50)) + } + throw new Error('a mutation on clicks never finished') +} + +/** + * Five months of clicks, one per month, each with a whole address. With a + * 90-day raw period and a 30-day address period, run at 2026-09-24: + * + * raw cutoff 2026-06-26 — a partition ending at or before this is dropped + * address cutoff 2026-08-25 — one ending at or before this is blanked + * + * 202605 ends 2026-06-01 dropped + * 202606 ends 2026-07-01 kept, address blanked + * 202607 ends 2026-08-01 kept, address blanked + * 202608 ends 2026-09-01 kept, address kept + * 202609 ends 2026-10-01 kept, address kept + */ +async function seed(): Promise { + await ch.command({ query: 'TRUNCATE TABLE clicks' }) + await ch.command({ query: 'TRUNCATE TABLE clicks_hourly' }) + await ch.command({ query: 'TRUNCATE TABLE clicks_hourly_dim' }) + await ch.insert({ + table: 'clicks', + values: [ + click('01920000-0000-7000-8000-000000000005', '2026-05-15 10:00:00.000', '198.51.100.5'), + click('01920000-0000-7000-8000-000000000006', '2026-06-15 10:00:00.000', '198.51.100.6'), + click('01920000-0000-7000-8000-000000000007', '2026-07-15 10:00:00.000', '198.51.100.7'), + click('01920000-0000-7000-8000-000000000008', '2026-08-15 10:00:00.000', '198.51.100.8'), + click('01920000-0000-7000-8000-000000000009', '2026-09-15 10:00:00.000', '198.51.100.9'), + ], + format: 'JSONEachRow', + }) +} + +beforeAll(async () => { + await resetDatabases(pool, ch) +}) + +beforeEach(async () => { + await pool.query('TRUNCATE settings') + await pool.query('INSERT INTO settings DEFAULT VALUES') + await settleMutations() + await seed() +}) + +afterAll(async () => { + await pool.end() + await ch.close() +}) + +describe('partitionEndMs', () => { + it.each([ + ['202605', '2026-06-01T00:00:00.000Z'], + ['202612', '2027-01-01T00:00:00.000Z'], + ['202601', '2026-02-01T00:00:00.000Z'], + ])('is the first instant after the month %s names', (id, iso) => { + expect(partitionEndMs(id)).toBe(Date.parse(iso)) + }) + + it.each([ + ['a partition that is not a month', '202613'], + ['a month of zero', '202600'], + ['a day partition', '20260524'], + ['the whole table as one partition', 'all'], + ['something with a quote in it', "2026'05"], + ['an empty id', ''], + ])('refuses %s', (_label, id) => { + expect(partitionEndMs(id)).toBeNull() + }) +}) + +describe('runRetention', () => { + it('drops a partition whose last possible row is past the period, and keeps the next one', async () => { + const r = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(r.dropped).toEqual(['202605']) + expect(await partitions()).toEqual(['202606', '202607', '202608', '202609']) + }) + + it('keeps every rollup number when it drops a partition', async () => { + expect(await rollupClicks()).toBe(5) + await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(await rollupClicks()).toBe(5) + }) + + it('blanks the address in every partition past the address period and no others', async () => { + const r = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(r.blanked).toEqual(['202606', '202607']) + expect(await addresses()).toEqual([ + { partition_id: '202606', ip: '' }, + { partition_id: '202607', ip: '' }, + { partition_id: '202608', ip: '198.51.100.8' }, + { partition_id: '202609', ip: '198.51.100.9' }, + ]) + }) + + it('does not blank a partition twice', async () => { + await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + const second = await runRetention({ pg: pool, ch, now: NOW }) + expect(second.blanked).toEqual([]) + expect(second.dropped).toEqual([]) + }) + + it('blanks nothing while a mutation is still running on the table', async () => { + await ch.command({ query: 'SYSTEM STOP MERGES clicks' }) + try { + await ch.command({ + query: "ALTER TABLE clicks UPDATE ip = '' IN PARTITION ID '202609' WHERE ip != ''", + }) + const r = await runRetention({ pg: pool, ch, now: NOW }) + expect(r.mutationInFlight).toBe(true) + expect(r.blanked).toEqual([]) + // And the drop still happened: a partition dropped whole is not a + // mutation and is not waiting for one. + expect(r.dropped).toEqual(['202605']) + } finally { + await ch.command({ query: 'SYSTEM START MERGES clicks' }) + await settleMutations() + } + }) + + it('does nothing at all when the stored retention cannot be read', async () => { + await pool.query('ALTER TABLE settings DROP CONSTRAINT settings_raw_retention_valid') + await pool.query('UPDATE settings SET raw_retention_days = -5') + try { + const r = await runRetention({ pg: pool, ch, now: NOW }) + expect(r).toEqual({ + dropped: [], + blanked: [], + skipped: [], + mutationInFlight: false, + ranNothing: true, + }) + expect(await partitions()).toEqual(['202605', '202606', '202607', '202608', '202609']) + expect(await addresses()).toHaveLength(5) + } finally { + // The row goes back inside the bound before the constraint does: a CHECK + // added over a row that violates it is refused, and the failure would + // arrive as this test's, in place of whatever it was actually asserting. + await pool.query('UPDATE settings SET raw_retention_days = 90') + await pool.query( + 'ALTER TABLE settings ADD CONSTRAINT settings_raw_retention_valid CHECK (raw_retention_days IS NULL OR raw_retention_days BETWEEN 1 AND 3650)', + ) + } + }) + + // The other half of the same rule, and the likelier half: a row deleted by + // hand is one statement away, and the periods nobody chose are the ones a + // default would supply. Separate from the test above because they arrive by + // different paths — a row that fails the schema, and no row to read — and a + // build that answered one with the defaults would still pass the other. + it('does nothing at all when there is no settings row', async () => { + await pool.query('TRUNCATE settings') + const r = await runRetention({ pg: pool, ch, now: NOW }) + expect(r).toEqual({ + dropped: [], + blanked: [], + skipped: [], + mutationInFlight: false, + ranNothing: true, + }) + expect(await partitions()).toEqual(['202605', '202606', '202607', '202608', '202609']) + expect(await addresses()).toHaveLength(5) + }) + + it('deletes nothing when both periods are never', async () => { + await pool.query('UPDATE settings SET raw_retention_days = NULL, ip_retention_days = NULL') + const r = await runRetention({ pg: pool, ch, now: NOW }) + expect(r.dropped).toEqual([]) + expect(r.blanked).toEqual([]) + expect(await partitions()).toEqual(['202605', '202606', '202607', '202608', '202609']) + }) + + it('drops without blanking when only the raw period is set', async () => { + await pool.query('UPDATE settings SET ip_retention_days = NULL') + const r = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(r.dropped).toEqual(['202605']) + expect(r.blanked).toEqual([]) + expect((await addresses()).every((a) => a.ip !== '')).toBe(true) + }) + + it('blanks without dropping when only the address period is set', async () => { + await pool.query('UPDATE settings SET raw_retention_days = NULL') + const r = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(r.dropped).toEqual([]) + // 202605 is past the address period too, and is still here to be blanked. + expect(r.blanked).toEqual(['202605', '202606', '202607']) + expect(await partitions()).toEqual(['202605', '202606', '202607', '202608', '202609']) + }) + + it('lowers a period and takes effect on the next pass, without rewriting history', async () => { + await pool.query('UPDATE settings SET raw_retention_days = 30') + const r = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + // Cutoff 2026-08-25: every partition ending at or before it goes. + expect(r.dropped).toEqual(['202605', '202606', '202607']) + expect(await partitions()).toEqual(['202608', '202609']) + expect(await rollupClicks()).toBe(5) + }) + + // A period read a moment before the delete is a period that can have been + // changed by the time the delete runs, and nothing can put a Postgres read + // and a ClickHouse drop in one transaction. The pass therefore reads under + // the settings row's lock and holds it, which is what this pins: the pass is + // made to wait on a lock an operator's own transaction holds, the operator + // sets both periods to never inside it, and what the pass then acts on is + // the row as committed — not the ninety days that were there when it + // started. + it('cannot enforce a period an operator changed while the pass was waiting for the row', async () => { + const blocked = async (): Promise => { + const r = await pool.query<{ n: string }>( + `SELECT count(*) AS n FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid + WHERE NOT l.granted AND a.datname = current_database()`, + ) + return Number(r.rows[0]?.n ?? 0) > 0 + } + const operator = await pool.connect() + try { + await operator.query('BEGIN') + await operator.query('SELECT raw_retention_days FROM settings FOR UPDATE') + const running = runRetention({ pg: pool, ch, now: NOW }) + let waiting = false + for (let i = 0; i < 200 && !waiting; i++) { + waiting = await blocked() + if (!waiting) await new Promise((resolve) => setTimeout(resolve, 25)) + } + // Bounded, and asserted rather than assumed: a pass that never waits for + // this lock has read the period without it, and the rest of this test + // would be measuring a race instead of a guard. + expect(waiting, 'the pass did not wait for the settings row').toBe(true) + await operator.query( + 'UPDATE settings SET raw_retention_days = NULL, ip_retention_days = NULL', + ) + await operator.query('COMMIT') + const r = await running + expect(r.dropped).toEqual([]) + expect(r.blanked).toEqual([]) + expect(await partitions()).toEqual(['202605', '202606', '202607', '202608', '202609']) + expect((await addresses()).every((a) => a.ip !== '')).toBe(true) + } finally { + await operator.query('ROLLBACK').catch(() => {}) + operator.release() + } + }) + + it('considers at most as many partitions as it is given, oldest first', async () => { + const r = await runRetention({ pg: pool, ch, now: NOW, maxPartitions: 1 }) + await settleMutations() + expect(r.dropped).toEqual(['202605']) + expect(r.blanked).toEqual([]) + }) + + it('logs and skips a partition id that is not a month rather than putting it in a statement', async () => { + const said: string[] = [] + // A second table partitioned by something else cannot exist here, so the + // skip is reached through the parser instead: the function that turns an + // id into an instant is the gate, and it is tested above. What this pins is + // that the pass asks it and acts on the answer. + const r = await runRetention({ + pg: pool, + ch, + now: NOW, + log: (m) => said.push(m), + partitionSource: async () => ['202605', 'all', "2026'05"], + }) + await settleMutations() + expect(r.dropped).toEqual(['202605']) + expect(r.skipped).toEqual(['all', "2026'05"]) + expect(said.join(' ')).toContain('not a month') + // The table is still there: nothing was interpolated. + expect(await partitions()).toEqual(['202606', '202607', '202608', '202609']) + }) +}) + +describe('startRetention', () => { + it('runs a pass and stops when it is told to', async () => { + const passes: string[] = [] + const loop = startRetention({ + pg: pool, + ch, + intervalMs: 50, + now: () => NOW, + log: (m) => passes.push(m), + }) + try { + for (let i = 0; i < 100 && passes.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 20)) + } + expect(passes.length).toBeGreaterThan(0) + } finally { + await loop.stop() + } + await settleMutations() + const after = passes.length + await new Promise((resolve) => setTimeout(resolve, 150)) + expect(passes.length).toBe(after) + }) +}) diff --git a/packages/worker/src/retention.ts b/packages/worker/src/retention.ts new file mode 100644 index 0000000..4c6768c --- /dev/null +++ b/packages/worker/src/retention.ts @@ -0,0 +1,281 @@ +/** + * Deleting what this install said it would stop keeping. + * + * Two jobs, on their own periods. Raw clicks past the raw period are dropped a + * whole partition at a time; the address on a click past the address period is + * blanked in place. Rollups are never touched — they are small, and they are + * what answers a question older than the raw window. + * + * **A period is a floor, not a promise.** `clicks` is partitioned by month, so + * a partition can only go once its last possible row is past the period: with + * ninety days an install keeps between ninety and a hundred and twenty-one + * days, and with thirty days of addresses it keeps them between thirty and + * sixty-one. Partitioning by day to tighten that would multiply the part count + * by thirty on the one table that grows without bound, and a row-level delete + * is a mutation — the expensive way to do what dropping a partition does for + * nothing. + * + * **Blanking is a mutation, so it is issued once.** A mutation rewrites the + * parts it touches, and re-issuing one every hour would rewrite a month of + * data every hour for ever. What decides whether to issue it is the condition + * itself — is there still a row in this partition with an address — rather than + * a record of what has been blanked, because a record is state that can + * disagree with the data it describes. And nothing is issued while a mutation + * is still running on the table, or a month that takes twenty minutes to + * rewrite would collect a new mutation every hour. + * + * **A partition id is the one value in this codebase that reaches SQL as + * text.** DDL takes no bound parameters. So the id has to come from + * ClickHouse's own `system.parts`, never from anything a request can reach, and + * it is put through `partitionEndMs` first — which returns null for anything + * that is not a month this table's partition key could have produced. An id + * that fails is logged and skipped. + * + * **When the stored retention cannot be read, this does nothing.** A default + * is a safe answer when the thing it stands in for is an action; a retention + * default deletes clicks. It is the only default in this product that does. + * + * **And the read is held under the settings row's lock for the whole pass**, + * which is the same lock a settings write takes. Nothing can span both stores, + * so without it an operator who sets a period to `never` — or deletes the row — + * a moment after the read has it enforced anyway, once, by a pass that read it + * as ninety days. The cost is that a settings write waits for a pass in + * flight, which is the DDL for at most one pass's worth of partitions. + */ +import type { ClickHouseClient, Pool, PoolClient } from '@clickmonk/db' +import { readSettingsLocked } from './settings.js' + +/** Partitions one pass looks at, oldest first. Two years of months. */ +export const RETENTION_PARTITIONS_PER_PASS = 24 + +const DAY_MS = 86_400_000 +const PARTITION_ID = /^(\d{4})(\d{2})$/ + +/** + * The first instant after the month a partition id names, or null when the id + * is not one this table's `toYYYYMM(time)` key could have produced. + * + * `Date.UTC(year, month)` with a one-based month is the first instant of the + * month after it, December included, because the month argument is zero-based + * and rolls over on its own. + */ +export function partitionEndMs(id: string): number | null { + const m = PARTITION_ID.exec(id) + if (!m) return null + const year = Number(m[1]) + const month = Number(m[2]) + if (month < 1 || month > 12) return null + return Date.UTC(year, month, 1) +} + +export interface RetentionResult { + /** Partition ids dropped, oldest first. */ + dropped: string[] + /** Partition ids whose address column was blanked this pass. */ + blanked: string[] + /** Partition ids this pass refused to name in a statement. */ + skipped: string[] + /** True when a mutation was already running, so nothing was blanked. */ + mutationInFlight: boolean + /** True when the stored retention could not be read and the pass did nothing. */ + ranNothing: boolean +} + +const empty = (over: Partial = {}): RetentionResult => ({ + dropped: [], + blanked: [], + skipped: [], + mutationInFlight: false, + ranNothing: false, + ...over, +}) + +async function scalar( + ch: ClickHouseClient, + query: string, + params: Record = {}, +): Promise { + const rs = await ch.query({ query, query_params: params, format: 'JSONEachRow' }) + const rows = await rs.json<{ n: string }>() + return Number(rows[0]?.n ?? 0) +} + +/** Every active partition of `clicks`, oldest first. */ +async function partitionsOfClicks(ch: ClickHouseClient): Promise { + const rs = await ch.query({ + query: `SELECT DISTINCT partition_id FROM system.parts + WHERE database = currentDatabase() AND table = 'clicks' AND active + ORDER BY partition_id`, + format: 'JSONEachRow', + }) + return (await rs.json<{ partition_id: string }>()).map((r) => r.partition_id) +} + +export interface RetentionOptions { + pg: Pool + ch: ClickHouseClient + now: Date + maxPartitions?: number + /** A seam for the list of partitions only, so a test can reach the skip branch. */ + partitionSource?: () => Promise + log?: (msg: string, err?: unknown) => void +} + +/** + * One pass. Opens a transaction on Postgres, reads the retention under the + * settings row's lock, and holds it until the deleting is done. + */ +export async function runRetention(o: RetentionOptions): Promise { + const log = o.log ?? (() => {}) + const client = await o.pg.connect() + try { + await client.query('BEGIN') + const result = await pass(o, client, log) + await client.query('COMMIT') + return result + } catch (err) { + // Every exit path ends the transaction before the client goes back to the + // pool, and the lock goes with it: a client released mid-transaction is a + // row lock nothing will ever release and a settings write that hangs. + await client.query('ROLLBACK').catch(() => {}) + throw err + } finally { + client.release() + } +} + +async function pass( + o: RetentionOptions, + client: PoolClient, + log: (msg: string, err?: unknown) => void, +): Promise { + const read = await readSettingsLocked(client) + // On `retention`, never on `problem`: a missing row and an unreadable one are + // deliberately the same state here, so there is one branch and no second + // path to get wrong. `problem` only says which it was. + if (read.retention === null) { + log(`retention is not running: ${read.problem ?? 'the stored retention cannot be read'}`) + return empty({ ranNothing: true }) + } + const { rawRetentionDays, ipRetentionDays } = read.retention + // Both null is an answer — keep both for ever — and not the same as the + // branch above. The two are one `??` apart. + if (rawRetentionDays === null && ipRetentionDays === null) return empty() + + const nowMs = o.now.getTime() + const rawCutoff = rawRetentionDays === null ? null : nowMs - rawRetentionDays * DAY_MS + const ipCutoff = ipRetentionDays === null ? null : nowMs - ipRetentionDays * DAY_MS + + const all = await (o.partitionSource ?? (() => partitionsOfClicks(o.ch)))() + const considered = all.slice(0, o.maxPartitions ?? RETENTION_PARTITIONS_PER_PASS) + + const result = empty() + const toBlank: string[] = [] + for (const id of considered) { + const end = partitionEndMs(id) + if (end === null) { + result.skipped.push(id) + log(`skipped partition ${id}: not a month this table could have produced`) + continue + } + // The drop is checked first: a partition past both periods goes whole + // rather than being rewritten and then dropped. + if (rawCutoff !== null && end <= rawCutoff) { + await o.ch.command({ query: `ALTER TABLE clicks DROP PARTITION ID '${id}'` }) + result.dropped.push(id) + continue + } + if (ipCutoff !== null && end <= ipCutoff) toBlank.push(id) + } + + if (toBlank.length > 0) { + const running = await scalar( + o.ch, + `SELECT count() AS n FROM system.mutations + WHERE database = currentDatabase() AND table = 'clicks' AND NOT is_done`, + ) + if (running > 0) { + result.mutationInFlight = true + log(`${running} mutation(s) still running on clicks; no addresses blanked this pass`) + } else { + for (const id of toBlank) { + // The condition itself, and it stops at the first row it finds rather + // than counting a month of them. + // + // No `FINAL` here, unlike every other raw read in this service, and + // that is deliberate: this is an existence test bounded at one row, and + // a duplicate row still means there is an address in this partition. A + // dedup would cost a merge to answer a question that does not need one. + const left = await scalar( + o.ch, + `SELECT count() AS n FROM ( + SELECT 1 FROM clicks WHERE _partition_id = {p:String} AND ip != '' LIMIT 1)`, + { p: id }, + ) + if (left === 0) continue + await o.ch.command({ + query: `ALTER TABLE clicks UPDATE ip = '' IN PARTITION ID '${id}' WHERE ip != ''`, + }) + result.blanked.push(id) + } + } + } + + log( + `retention: dropped ${result.dropped.length}, blanked ${result.blanked.length}, skipped ${result.skipped.length}`, + ) + return result +} + +/** + * Runs it every interval, starting at once. The returned promise of `stop()` + * resolves after the pass in flight. Never rejects: a pass that throws is + * logged and the next one is tried, because the alternative is a worker that + * stops deleting and never says so. + */ +export function startRetention(o: { + pg: Pool + ch: ClickHouseClient + intervalMs?: number + now?: () => Date + log?: (msg: string, err?: unknown) => void +}): { stop(): Promise } { + const interval = o.intervalMs ?? 3_600_000 + const now = o.now ?? (() => new Date()) + const log = (msg: string, err?: unknown) => { + try { + o.log?.(msg, err) + } catch { + // A broken logger must not stop the pass. + } + } + let stopped = false + let wake: (() => void) | null = null + + const loop = (async () => { + while (!stopped) { + try { + await runRetention({ pg: o.pg, ch: o.ch, now: now(), log }) + } catch (err) { + log('retention failed; will try again', err) + } + if (stopped) break + await new Promise((resolve) => { + const t = setTimeout(resolve, interval) + wake = () => { + clearTimeout(t) + resolve() + } + }) + wake = null + } + })() + + return { + async stop() { + stopped = true + wake?.() + await loop + }, + } +} diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts index b9ed4c3..35fd25e 100644 --- a/packages/worker/src/settings.ts +++ b/packages/worker/src/settings.ts @@ -8,6 +8,10 @@ * in the worker package because that is where the other functions two * surfaces share already live. * + * The reader comes in two spellings — on the pool, and on a caller's client + * under the row's lock — sharing one fallback rule. The locked one exists for + * the retention pass, whose read has to hold still while it deletes. + * * The redirect is the deliberate exception. It reads these settings inside * the snapshot's single REPEATABLE READ READ ONLY transaction, together with * the domains and links, so that the configuration it serves is one @@ -121,7 +125,37 @@ function parseRow(row: SettingsRow): ParsedRow { */ export async function readSettings(pg: Pool): Promise { const r = await pg.query(`SELECT ${COLUMNS} FROM settings`) - const row = r.rows[0] + return fromRow(r.rows[0]) +} + +/** + * The same read, on a caller's client and under the row's lock. + * + * For a caller that only answers a request, the plain read above is right: the + * answer is a moment old and the next request reads again. For a caller that + * **deletes** on what it read, a moment is the whole problem — an operator can + * set a period to `never`, or delete the row outright, in the window between + * the read and the delete, and the pass would go on to enforce a period that + * is no longer stored. There is no transaction that can span both stores, so + * the lock is the join: this takes the same `FOR UPDATE` on the same row that + * `updateSettings` takes, so a write and a deleting pass serialise and the + * pass acts on the row as it stands for as long as it holds the lock. + * + * It takes no lock when the row is missing — there is nothing to lock — and + * that needs none: a missing row answers `retention: null`, and a caller that + * deletes does nothing at all on that answer. + * + * The caller owns the transaction, and owes it a `BEGIN` before this and a + * `COMMIT` or `ROLLBACK` after: the lock is held to the end of the + * transaction, and that duration is the point. + */ +export async function readSettingsLocked(client: PoolClient): Promise { + const r = await client.query(`SELECT ${COLUMNS} FROM settings FOR UPDATE`) + return fromRow(r.rows[0]) +} + +/** One fallback rule, whether the row was read under the lock or not. */ +function fromRow(row: SettingsRow | undefined): SettingsRead { if (!row) { return { traffic: DEFAULT_TRAFFIC_SETTINGS, From 9ae5f4e0affc3dea20b1b158ac1724d5feaf1dae Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:12:19 +0000 Subject: [PATCH 44/65] worker: the lock a failed retention pass must not keep --- packages/worker/src/retention.test.ts | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index d211dbe..85c8876 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -323,6 +323,40 @@ describe('runRetention', () => { } }) + // The other end of holding that lock: a pass that throws while it holds it + // must end its transaction before the client goes back to the pool. A client + // released inside an open transaction takes the row lock with it, and the + // next `settings set` an operator runs waits for a connection that is never + // coming back — one failed pass, and the install cannot change its settings + // until the worker restarts. + it('holds no lock after a pass that threw', async () => { + const idleInTransaction = async (): Promise => { + const r = await pool.query<{ n: string }>( + `SELECT count(*) AS n FROM pg_stat_activity + WHERE datname = current_database() AND state LIKE 'idle in transaction%'`, + ) + return Number(r.rows[0]?.n ?? 0) + } + await expect( + runRetention({ + pg: pool, + ch, + now: NOW, + partitionSource: async () => { + throw new Error('the partition list is unreadable') + }, + }), + ).rejects.toThrow('the partition list is unreadable') + // Bounded rather than immediate: a backend's reported state settles a + // moment after the statement that changed it. + let open = await idleInTransaction() + for (let i = 0; i < 40 && open > 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 25)) + open = await idleInTransaction() + } + expect(open, 'a connection went back to the pool inside a transaction').toBe(0) + }) + it('considers at most as many partitions as it is given, oldest first', async () => { const r = await runRetention({ pg: pool, ch, now: NOW, maxPartitions: 1 }) await settleMutations() From 836f62449544e837ecc75a2f781bd4b76a889537 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:14:00 +0000 Subject: [PATCH 45/65] worker: a stop during a pass is not an hour long --- packages/worker/src/retention.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index 85c8876..2e4c1c1 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -409,4 +409,19 @@ describe('startRetention', () => { await new Promise((resolve) => setTimeout(resolve, 150)) expect(passes.length).toBe(after) }) + + // `stop()` waits for the pass in flight, and the interval in production is an + // hour. So a loop told to stop *during* a pass must not go on to set the next + // hour's timer: nothing wakes a timer set after the stop, and `stop()` would + // then be a shutdown that hangs until the interval runs out — which SIGTERM + // does not wait for, so the container is killed instead of stopping. + it('stops without waiting out the interval when it is told to during a pass', async () => { + const loop = startRetention({ pg: pool, ch, intervalMs: 3_600_000, now: () => NOW }) + // The next statement after starting it, so the first pass — several round + // trips to two stores — is certainly still running. + const began = Date.now() + await loop.stop() + expect(Date.now() - began).toBeLessThan(10_000) + await settleMutations() + }) }) From b028edb568e0511abcfd406765448c776e6be209 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:17:28 +0000 Subject: [PATCH 46/65] worker: a connection cannot tell you it is the one still in a transaction --- packages/worker/src/retention.test.ts | 43 +++++++++++++++++---------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index 2e4c1c1..6b31d66 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -4,6 +4,13 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' import { partitionEndMs, runRetention, startRetention } from './retention.js' const pool: Pool = testPg() +/** + * A second pool, for the two tests that have to ask a question from outside + * the connection the pass is using — which is what an operator's own + * `settings set` is. Asking through `pool` risks asking on the pass's own + * client, and a connection cannot notice that it is itself the problem. + */ +const operators: Pool = testPg() const ch: ClickHouseClient = testCh() /** @@ -120,6 +127,7 @@ beforeEach(async () => { afterAll(async () => { await pool.end() + await operators.end() await ch.close() }) @@ -288,13 +296,13 @@ describe('runRetention', () => { // started. it('cannot enforce a period an operator changed while the pass was waiting for the row', async () => { const blocked = async (): Promise => { - const r = await pool.query<{ n: string }>( + const r = await operators.query<{ n: string }>( `SELECT count(*) AS n FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE NOT l.granted AND a.datname = current_database()`, ) return Number(r.rows[0]?.n ?? 0) > 0 } - const operator = await pool.connect() + const operator = await operators.connect() try { await operator.query('BEGIN') await operator.query('SELECT raw_retention_days FROM settings FOR UPDATE') @@ -330,13 +338,6 @@ describe('runRetention', () => { // coming back — one failed pass, and the install cannot change its settings // until the worker restarts. it('holds no lock after a pass that threw', async () => { - const idleInTransaction = async (): Promise => { - const r = await pool.query<{ n: string }>( - `SELECT count(*) AS n FROM pg_stat_activity - WHERE datname = current_database() AND state LIKE 'idle in transaction%'`, - ) - return Number(r.rows[0]?.n ?? 0) - } await expect( runRetention({ pg: pool, @@ -347,14 +348,24 @@ describe('runRetention', () => { }, }), ).rejects.toThrow('the partition list is unreadable') - // Bounded rather than immediate: a backend's reported state settles a - // moment after the statement that changed it. - let open = await idleInTransaction() - for (let i = 0; i < 40 && open > 0; i++) { - await new Promise((resolve) => setTimeout(resolve, 25)) - open = await idleInTransaction() + // From `operators`, never from `pool`: the pool hands the most recently + // released client back first, so a question asked through it would be asked + // *inside* the very transaction this is checking is over — and answered + // yes. A separate pool is a separate backend, which is what an operator + // running `settings set` is. This first cost the check its whole point: the + // same assertion through `pool` passed with the rollback taken out. + const operator = await operators.connect() + try { + await operator.query('BEGIN') + // Bounded, so a lock nobody will release fails this test rather than + // hanging it until the suite's own timeout. + await operator.query("SET LOCAL lock_timeout = '5s'") + await operator.query('SELECT 1 FROM settings FOR UPDATE') + await operator.query('COMMIT') + } finally { + await operator.query('ROLLBACK').catch(() => {}) + operator.release() } - expect(open, 'a connection went back to the pool inside a transaction').toBe(0) }) it('considers at most as many partitions as it is given, oldest first', async () => { From 26681ff3755660528e2190be5a94c94e864e7ef4 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:20:34 +0000 Subject: [PATCH 47/65] worker: the pass a stopping loop must not run --- packages/worker/src/retention.test.ts | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index 6b31d66..7426594 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -421,6 +421,34 @@ describe('startRetention', () => { expect(passes.length).toBe(after) }) + // The loop has two ways to notice a stop — its own condition, and the check + // between the pass and the next timer — and for *terminating* either one is + // enough, so a test that only requires `stop()` to return cannot tell which + // of them is still there. This is the one that needs the condition: a loop + // woken out of its timer by `stop()` must not run one more pass on the way + // out. That pass is a round of DROP PARTITION issued after SIGTERM. + it('runs no pass after it has been told to stop', async () => { + const passes: string[] = [] + const loop = startRetention({ + pg: pool, + ch, + intervalMs: 3_600_000, + now: () => NOW, + log: (m) => passes.push(m), + }) + for (let i = 0; i < 200 && passes.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 20)) + } + // An hour-long interval is what makes the count below mean anything: the + // loop is asleep on a timer nothing but `stop()` can wake, so any pass + // logged after this line is one it ran on its way out. + const before = passes.length + expect(before).toBeGreaterThan(0) + await loop.stop() + expect(passes.length, 'a pass ran after the loop was told to stop').toBe(before) + await settleMutations() + }) + // `stop()` waits for the pass in flight, and the interval in production is an // hour. So a loop told to stop *during* a pass must not go on to set the next // hour's timer: nothing wakes a timer set after the stop, and `stop()` would From 2c395aeed27a3143626efb9565fc783c7746cc99 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:35:35 +0000 Subject: [PATCH 48/65] worker: a lock the store cannot hold open --- packages/admin/src/settings-routes.test.ts | 31 +++++++ packages/admin/src/settings-routes.ts | 18 +++- packages/cli/src/commands.test.ts | 25 +++++ packages/cli/src/commands.ts | 17 +++- packages/worker/src/config.test.ts | 11 +++ packages/worker/src/config.ts | 15 +++ packages/worker/src/retention.test.ts | 78 +++++++++++++++- packages/worker/src/retention.ts | 48 +++++++++- packages/worker/src/settings.test.ts | 65 +++++++++++++ packages/worker/src/settings.ts | 102 +++++++++++++++++---- 10 files changed, 385 insertions(+), 25 deletions(-) diff --git a/packages/admin/src/settings-routes.test.ts b/packages/admin/src/settings-routes.test.ts index 593a9d1..ba8ad99 100644 --- a/packages/admin/src/settings-routes.test.ts +++ b/packages/admin/src/settings-routes.test.ts @@ -172,6 +172,37 @@ describe('writing the settings', () => { expect(back.json().retention).toEqual({ rawRetentionDays: 30, ipRetentionDays: 90 }) }) + // The retention pass holds this row while it deletes, and the writer stops + // waiting rather than holding this request open for the length of a pass — + // Fastify's own request timeout does not bound a handler, so waiting here is + // waiting with a connection of a pool of four. Nothing is wrong with the + // request, which is why it is 503 with a `retry-after` and not a 4xx. + it('answers 503 while the settings row is held, and writes nothing', async () => { + const holder = await pg.connect() + let r: Awaited> + try { + await holder.query('BEGIN') + await holder.query('SELECT 1 FROM settings FOR UPDATE') + r = await put({ + traffic: { ...DEFAULT_TRAFFIC_SETTINGS, abuserThreshold: 120 }, + retention: { rawRetentionDays: 5, ipRetentionDays: 5 }, + }) + } finally { + await holder.query('ROLLBACK').catch(() => {}) + holder.release() + } + expect(r.statusCode).toBe(503) + expect(r.headers['retry-after']).toBe('5') + expect(r.json()).toEqual({ + error: 'settings_locked', + message: + 'the settings row is held by another writer, most likely the retention pass, which holds it for the length of one pass; nothing was written, so run this again in a moment', + }) + const back = await get() + expect(back.json().traffic.abuserThreshold).toBe(60) + expect(back.json().retention).toEqual({ rawRetentionDays: 90, ipRetentionDays: 30 }) + }) + it('refuses a body whose retention half is invalid, and writes neither half', async () => { const r = await put({ traffic: { ...DEFAULT_TRAFFIC_SETTINGS, abuserThreshold: 120 }, diff --git a/packages/admin/src/settings-routes.ts b/packages/admin/src/settings-routes.ts index ad0d0b2..2cb2376 100644 --- a/packages/admin/src/settings-routes.ts +++ b/packages/admin/src/settings-routes.ts @@ -12,11 +12,11 @@ * validated by its own strict schema. */ import { type InstallSettings, InstallSettingsSchema, retentionNote } from '@clickmonk/core' -import { readSettings, writeSettings } from '@clickmonk/worker/settings' +import { SettingsLockedError, readSettings, writeSettings } from '@clickmonk/worker/settings' import type { FastifyInstance } from 'fastify' import type { AdminContext } from './app.js' import { requireCredential } from './auth.js' -import { readBody } from './http.js' +import { fail, readBody } from './http.js' function body(s: InstallSettings, problem: string | null): Record { return { traffic: s.traffic, retention: s.retention, note: retentionNote(s.retention), problem } @@ -41,7 +41,19 @@ export function registerSettingsRoutes(app: FastifyInstance, ctx: AdminContext): app.put('/api/settings', async (req) => { requireCredential(req) const next = readBody(InstallSettingsSchema, req.body) - await writeSettings(ctx.pg, next, ctx.now()) + try { + await writeSettings(ctx.pg, next, ctx.now()) + } catch (err) { + // The retention pass holds this row while it deletes, and the writer stops + // waiting rather than holding a request open for the length of a pass. + // 503 with `retry-after`, because nothing is wrong with the request and + // sending it again is the whole remedy. `return fail(…)`: a bare call does + // not tell the compiler this path ends. + if (err instanceof SettingsLockedError) { + return fail(503, 'settings_locked', err.message, { 'retry-after': '5' }) + } + throw err + } return body(next, null) }) } diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index 995ef30..f3abcc4 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -305,6 +305,31 @@ describe('clickmonk settings', () => { ]) }) + // The retention pass holds this row while it deletes. A command that waited + // for it would print nothing until somebody killed it, and then they would + // have to guess whether it had written anything — so it stops waiting and says + // what has the row. One line, and an exit code that says this is a refusal. + it('refuses to set the settings while the row is held, and writes nothing', async () => { + // A value of its own, and not the one the command below asks for: the tests + // in this block share the row, so an assertion on a value this test did not + // set would be an assertion about whichever of them ran last. + await pg.query('UPDATE settings SET abuser_threshold = 77') + const holder = await pg.connect() + try { + await holder.query('BEGIN') + await holder.query('SELECT 1 FROM settings FOR UPDATE') + lines.length = 0 + expect(await run('settings', 'set', '--abuser-threshold', '99')).toBe(2) + } finally { + await holder.query('ROLLBACK').catch(() => {}) + holder.release() + } + expect(lines).toEqual([ + 'error: the settings row is held by another writer, most likely the retention pass, which holds it for the length of one pass; nothing was written, so run this again in a moment', + ]) + expect((await settings())?.abuser_threshold).toBe(77) + }) + // The traffic half falls back and the retention half does not, and the // output says both: printing "keep clicks: 90 days" for a row that is not // there would tell the operator a period this install is not enforcing. diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index ad59637..8fc2387 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -46,7 +46,12 @@ import { runDomainChecks, } from '@clickmonk/worker/domains' import { createLink } from '@clickmonk/worker/links' -import { type SettingsSubstitution, readSettings, updateSettings } from '@clickmonk/worker/settings' +import { + SettingsLockedError, + type SettingsSubstitution, + readSettings, + updateSettings, +} from '@clickmonk/worker/settings' import type { ZodError } from 'zod' export interface CliDeps { @@ -860,6 +865,16 @@ export async function runCli(argv: string[], d: CliDeps): Promise { d.out(`error: ${err.message}\n${USAGE}`) return 1 } + // The settings row was held by something else for longer than the writer + // waits — almost always the retention pass, which holds it while it deletes. + // A refusal rather than an unexpected error because the answer is to run the + // command again, and because the alternative was a command that printed + // nothing until somebody killed it and then had to guess what it had done. + // Its message is written for a person; it is printed as it stands. + if (err instanceof SettingsLockedError) { + d.out(`error: ${err.message}`) + return 2 + } // A command run before the migrations. `42P01` is Postgres saying the table // is not there, and the answer is always the same one command, so this is a // refusal rather than an unexpected error. The database's own wording — the diff --git a/packages/worker/src/config.test.ts b/packages/worker/src/config.test.ts index ea06180..ff07990 100644 --- a/packages/worker/src/config.test.ts +++ b/packages/worker/src/config.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { loadConfig } from './config.js' +import { RETENTION_MAX_EXECUTION_SECONDS } from './retention.js' const base = { CLICKMONK_POSTGRES_URL: 'postgres://u:p@db:5432/clickmonk', @@ -57,6 +58,16 @@ describe('loadConfig', () => { ]) }) + // Not a tuning number but the outer bound on how long the retention pass can + // hold the settings row while waiting for ClickHouse, so what is asserted is + // the relation that makes it one: above the bound the pass sends with each + // statement, and present at all. + it('bounds a ClickHouse request above the bound the retention pass sends', () => { + expect(loadConfig(base).ch.requestTimeoutMs).toBeGreaterThan( + RETENTION_MAX_EXECUTION_SECONDS * 1000, + ) + }) + it('looks for clicks past their retention every hour', () => { expect(loadConfig(base).retentionIntervalMs).toBe(3_600_000) }) diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index b07eb79..66bfe02 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -24,6 +24,20 @@ const DnsServers = z } }) +/** + * Client-side bound on one ClickHouse request from this service. + * + * Not a tuning knob: it is the outer half of what stops a store that has + * stopped answering from holding a Postgres row lock open. The retention pass + * takes the settings row's lock and then talks to ClickHouse, so a request that + * never comes back is a settings write that never goes through. Longer than the + * `max_execution_time` the pass sends, so a statement ClickHouse itself ends + * arrives as ClickHouse's error and not as an abort with nothing in it, and long + * enough for the shipper's largest segment — an insert that times out is kept + * and retried, so the cost of being wrong here is a pass, not a click. + */ +export const CH_REQUEST_TIMEOUT_MS = 60_000 + const Schema = z.object({ CLICKMONK_POSTGRES_URL: z.string().url(), CLICKMONK_CLICKHOUSE_URL: z.string().url(), @@ -71,6 +85,7 @@ export function loadConfig(env: NodeJS.ProcessEnv): WorkerConfig { username: e.CLICKMONK_CLICKHOUSE_USER, password: e.CLICKMONK_CLICKHOUSE_PASSWORD, database: e.CLICKMONK_CLICKHOUSE_DB, + requestTimeoutMs: CH_REQUEST_TIMEOUT_MS, }, spoolDir: e.CLICKMONK_SPOOL_DIR, ipdataDir: e.CLICKMONK_IPDATA_DIR, diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index 7426594..c678b85 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -1,7 +1,12 @@ import type { ClickHouseClient, Pool } from '@clickmonk/db' import { resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { partitionEndMs, runRetention, startRetention } from './retention.js' +import { + RETENTION_MAX_EXECUTION_SECONDS, + partitionEndMs, + runRetention, + startRetention, +} from './retention.js' const pool: Pool = testPg() /** @@ -179,6 +184,77 @@ describe('runRetention', () => { ]) }) + // What decides whether to blank is the condition — is there an address left + // in this partition — and not a record of partitions already done. This is the + // difference between the two: a spool segment shipped late lands in the month + // its clicks happened in, which can be a month already blanked, and a ledger + // would have called that month finished and left the address there for ever. + it('blanks an address that arrived after its month was blanked', async () => { + const first = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(first.blanked).toEqual(['202606', '202607']) + await ch.insert({ + table: 'clicks', + values: [ + click('01920000-0000-7000-8000-00000000000a', '2026-06-20 10:00:00.000', '198.51.100.10'), + ], + format: 'JSONEachRow', + }) + const second = await runRetention({ pg: pool, ch, now: NOW }) + await settleMutations() + expect(second.blanked).toEqual(['202606']) + expect(await addresses()).toEqual([ + { partition_id: '202606', ip: '' }, + { partition_id: '202606', ip: '' }, + { partition_id: '202607', ip: '' }, + { partition_id: '202608', ip: '198.51.100.8' }, + { partition_id: '202609', ip: '198.51.100.9' }, + ]) + }) + + // A row lock held across calls to another store is a lock that store can hold + // open, so every statement of a pass carries a bound. Asserted over every + // statement the pass actually sent, not at the call sites: the rule is the + // module's, and a statement added without it is the defect this would catch. + it('bounds every statement it sends to the store', async () => { + const sent: { query: string; clickhouse_settings?: Record }[] = [] + const watched: ClickHouseClient = Object.assign(Object.create(ch) as ClickHouseClient, { + query: (p: { query: string }) => { + sent.push(p) + return ch.query(p as Parameters[0]) + }, + command: (p: { query: string }) => { + sent.push(p) + return ch.command(p as Parameters[0]) + }, + }) + const r = await runRetention({ pg: pool, ch: watched, now: NOW }) + await settleMutations() + // The pass did its whole job through the wrapper, so all five kinds of + // statement it can send are in the list below. Without this the loop under + // it would hold over an empty list. + expect(r.dropped).toEqual(['202605']) + expect(r.blanked).toEqual(['202606', '202607']) + const queries = sent.map((p) => p.query) + for (const fragment of [ + 'system.parts', + 'DROP PARTITION', + 'system.mutations', + "ip != ''", + 'UPDATE ip', + ]) { + expect( + queries.some((q) => q.includes(fragment)), + fragment, + ).toBe(true) + } + for (const p of sent) { + expect(p.clickhouse_settings, p.query).toEqual({ + max_execution_time: RETENTION_MAX_EXECUTION_SECONDS, + }) + } + }) + it('does not blank a partition twice', async () => { await runRetention({ pg: pool, ch, now: NOW }) await settleMutations() diff --git a/packages/worker/src/retention.ts b/packages/worker/src/retention.ts index 4c6768c..151700e 100644 --- a/packages/worker/src/retention.ts +++ b/packages/worker/src/retention.ts @@ -41,6 +41,15 @@ * a moment after the read has it enforced anyway, once, by a pass that read it * as ninety days. The cost is that a settings write waits for a pass in * flight, which is the DDL for at most one pass's worth of partitions. + * + * **Which is why every statement here is bounded.** A row lock held across + * calls to another store is a lock that store can hold open: a ClickHouse that + * accepts the connection and then stops answering would leave this transaction + * open for as long as it stayed that way, with an operator's `settings set` + * waiting behind it. So each statement carries `max_execution_time`, the + * worker's client carries a request timeout longer than that, and the settings + * writers carry a lock timeout of their own — the operator is told to try + * again rather than left holding a command that has printed nothing. */ import type { ClickHouseClient, Pool, PoolClient } from '@clickmonk/db' import { readSettingsLocked } from './settings.js' @@ -48,6 +57,23 @@ import { readSettingsLocked } from './settings.js' /** Partitions one pass looks at, oldest first. Two years of months. */ export const RETENTION_PARTITIONS_PER_PASS = 24 +/** + * The server-side bound on one statement of a pass, in seconds. + * + * Under the worker's own request timeout, so a statement ClickHouse itself ends + * comes back as ClickHouse's error rather than as a client-side abort with + * nothing in it. It is the inner half of what keeps the settings row's lock from + * being held by a store that has stopped answering; the client timeout is the + * outer half, and it covers whatever a server-side bound does not. + */ +export const RETENTION_MAX_EXECUTION_SECONDS = 20 + +/** + * Sent with every statement this pass issues. One object, so that adding a + * statement without the bound is a visible omission rather than a default. + */ +const BOUND = { max_execution_time: RETENTION_MAX_EXECUTION_SECONDS } as const + const DAY_MS = 86_400_000 const PARTITION_ID = /^(\d{4})(\d{2})$/ @@ -95,7 +121,12 @@ async function scalar( query: string, params: Record = {}, ): Promise { - const rs = await ch.query({ query, query_params: params, format: 'JSONEachRow' }) + const rs = await ch.query({ + query, + query_params: params, + format: 'JSONEachRow', + clickhouse_settings: BOUND, + }) const rows = await rs.json<{ n: string }>() return Number(rows[0]?.n ?? 0) } @@ -107,6 +138,7 @@ async function partitionsOfClicks(ch: ClickHouseClient): Promise { WHERE database = currentDatabase() AND table = 'clicks' AND active ORDER BY partition_id`, format: 'JSONEachRow', + clickhouse_settings: BOUND, }) return (await rs.json<{ partition_id: string }>()).map((r) => r.partition_id) } @@ -181,7 +213,10 @@ async function pass( // The drop is checked first: a partition past both periods goes whole // rather than being rewritten and then dropped. if (rawCutoff !== null && end <= rawCutoff) { - await o.ch.command({ query: `ALTER TABLE clicks DROP PARTITION ID '${id}'` }) + await o.ch.command({ + query: `ALTER TABLE clicks DROP PARTITION ID '${id}'`, + clickhouse_settings: BOUND, + }) result.dropped.push(id) continue } @@ -206,6 +241,14 @@ async function pass( // that is deliberate: this is an existence test bounded at one row, and // a duplicate row still means there is an address in this partition. A // dedup would cost a merge to answer a question that does not need one. + // + // **And this is why the condition is not a ledger.** A spool segment + // shipped late lands in whichever month its clicks happened in, which + // can be a month this pass has already blanked. The condition sees the + // address that arrived and blanks it on the next pass. A record of + // partitions already done would say that month was finished and leave + // that address in place for ever, which is the one outcome this whole + // module exists to prevent. const left = await scalar( o.ch, `SELECT count() AS n FROM ( @@ -215,6 +258,7 @@ async function pass( if (left === 0) continue await o.ch.command({ query: `ALTER TABLE clicks UPDATE ip = '' IN PARTITION ID '${id}' WHERE ip != ''`, + clickhouse_settings: BOUND, }) result.blanked.push(id) } diff --git a/packages/worker/src/settings.test.ts b/packages/worker/src/settings.test.ts index d53d6af..eaea40a 100644 --- a/packages/worker/src/settings.test.ts +++ b/packages/worker/src/settings.test.ts @@ -3,6 +3,7 @@ import { type Pool, createPgPool } from '@clickmonk/db' import { TEST_PG_URL, resetDatabases, testCh, testPg } from '@clickmonk/db/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' import { + SettingsLockedError, type SettingsSubstitution, readSettings, updateSettings, @@ -307,3 +308,67 @@ describe('updateSettings', () => { expect(subs).toEqual([]) }) }) + +/** + * The retention pass reads this row under its lock and holds it while it + * deletes, which is what makes a write and a delete serialise. A writer that + * waited for it would be a command, or a request, that hangs for the length of + * a pass with nothing printed — and an operator who kills one of those is left + * guessing what it managed to do. So a writer stops waiting and says what has + * the row. + * + * Held here the way a pass holds it, from another connection of this pool, so + * the writer under test is served a different one and really does wait. + */ +describe('a writer while the row is held', () => { + async function whileHeld(body: () => Promise): Promise { + const holder = await pool.connect() + try { + await holder.query('BEGIN') + await holder.query('SELECT 1 FROM settings FOR UPDATE') + return await body() + } finally { + await holder.query('ROLLBACK').catch(() => {}) + holder.release() + } + } + + it('refuses the merging write, names the pass, and writes nothing', async () => { + const failed = await whileHeld(() => + updateSettings(pool, NOW, (current) => ({ + ...current, + retention: { ...current.retention, rawRetentionDays: 5 }, + })).then( + () => null, + (err: unknown) => err, + ), + ) + expect(failed).toBeInstanceOf(SettingsLockedError) + expect((failed as Error).message).toMatch(/retention pass/) + expect((await readSettings(pool)).retention).toEqual({ + rawRetentionDays: 90, + ipRetentionDays: 30, + }) + }) + + it('refuses the whole-row write the same way, and writes nothing', async () => { + const failed = await whileHeld(() => + writeSettings( + pool, + { + traffic: { ...DEFAULT_TRAFFIC_SETTINGS, abuserThreshold: 120 }, + retention: { rawRetentionDays: 5, ipRetentionDays: 5 }, + }, + NOW, + ).then( + () => null, + (err: unknown) => err, + ), + ) + expect(failed).toBeInstanceOf(SettingsLockedError) + expect((failed as Error).message).toMatch(/retention pass/) + const after = await readSettings(pool) + expect(after.traffic.abuserThreshold).toBe(DEFAULT_TRAFFIC_SETTINGS.abuserThreshold) + expect(after.retention).toEqual({ rawRetentionDays: 90, ipRetentionDays: 30 }) + }) +}) diff --git a/packages/worker/src/settings.ts b/packages/worker/src/settings.ts index 35fd25e..8e6820d 100644 --- a/packages/worker/src/settings.ts +++ b/packages/worker/src/settings.ts @@ -58,6 +58,50 @@ export interface SettingsRead { problem: string | null } +/** + * How long a writer waits for the settings row before refusing, in + * milliseconds. + * + * The retention pass holds this row's lock for the whole of a pass, which is + * what makes a write and a delete serialise, and the pass's own statements are + * bounded so the wait is bounded too. But a command that has printed nothing + * for several seconds is a command an operator kills, and a killed write is one + * they then have to guess about. So a writer stops waiting and says what has + * the row instead. A pass that is doing its usual work holds the lock for + * milliseconds; this is long enough to ride that out and short enough to answer + * while somebody is still watching. + */ +export const SETTINGS_LOCK_TIMEOUT_MS = 2_000 + +/** + * The row was locked by someone else for longer than a writer waits. Named + * rather than anonymous because each surface answers it differently — the + * command line prints it and stops, the API answers 503 — and neither can tell + * it apart from a real fault otherwise. + */ +export class SettingsLockedError extends Error { + constructor() { + super( + 'the settings row is held by another writer, most likely the retention pass, which holds it for the length of one pass; nothing was written, so run this again in a moment', + ) + this.name = 'SettingsLockedError' + } +} + +/** `55P03` is Postgres saying it stopped waiting for a lock. */ +const isLockTimeout = (err: unknown): boolean => + typeof err === 'object' && err !== null && (err as { code?: unknown }).code === '55P03' + +/** + * `set_config(…, true)` rather than `SET LOCAL`: it is the spelling that takes + * a bound parameter, so the bound stays a value and never becomes SQL text. + */ +const boundWait = async (client: PoolClient): Promise => { + await client.query(`SELECT set_config('lock_timeout', $1, true)`, [ + String(SETTINGS_LOCK_TIMEOUT_MS), + ]) +} + const why = (issues: { path: (string | number)[]; message: string }[]): string => issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') @@ -175,27 +219,44 @@ function fromRow(row: SettingsRow | undefined): SettingsRead { * whole snapshot twice for one write. It upserts rather than assuming the row * is there, because it can be deleted by hand and an UPDATE alone would match * nothing and report success. + * + * Still one statement, and now in a transaction of its own so that it can carry + * a lock timeout: the retention pass holds this row while it deletes, and an + * upsert that waited for it would hang the request that asked for it. A wrapped + * single statement notifies exactly once, as it did unwrapped. */ export async function writeSettings(pg: Pool, next: InstallSettings, now: Date): Promise { - await pg.query( - `INSERT INTO settings (id, traffic_actions, safe_url, abuser_threshold, + const client: PoolClient = await pg.connect() + try { + await client.query('BEGIN') + await boundWait(client) + await client.query( + `INSERT INTO settings (id, traffic_actions, safe_url, abuser_threshold, raw_retention_days, ip_retention_days, updated_at) - VALUES (true, $1, $2, $3, $4, $5, $6) - ON CONFLICT (id) DO UPDATE - SET traffic_actions = EXCLUDED.traffic_actions, safe_url = EXCLUDED.safe_url, - abuser_threshold = EXCLUDED.abuser_threshold, - raw_retention_days = EXCLUDED.raw_retention_days, - ip_retention_days = EXCLUDED.ip_retention_days, - updated_at = EXCLUDED.updated_at`, - [ - JSON.stringify(next.traffic.actions), - next.traffic.safeUrl, - next.traffic.abuserThreshold, - next.retention.rawRetentionDays, - next.retention.ipRetentionDays, - now, - ], - ) + VALUES (true, $1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE + SET traffic_actions = EXCLUDED.traffic_actions, safe_url = EXCLUDED.safe_url, + abuser_threshold = EXCLUDED.abuser_threshold, + raw_retention_days = EXCLUDED.raw_retention_days, + ip_retention_days = EXCLUDED.ip_retention_days, + updated_at = EXCLUDED.updated_at`, + [ + JSON.stringify(next.traffic.actions), + next.traffic.safeUrl, + next.traffic.abuserThreshold, + next.retention.rawRetentionDays, + next.retention.ipRetentionDays, + now, + ], + ) + await client.query('COMMIT') + } catch (err) { + await client.query('ROLLBACK').catch(() => {}) + if (isLockTimeout(err)) throw new SettingsLockedError() + throw err + } finally { + client.release() + } } /** @@ -251,6 +312,10 @@ export async function updateSettings( const client: PoolClient = await pg.connect() try { await client.query('BEGIN') + // Before the first statement that can wait for the row, which is the insert: + // an upsert against a locked row blocks exactly as the `FOR UPDATE` below + // does. + await boundWait(client) // Whether this inserted tells us the row was not there, which is the third // substitution and the one the earlier two callbacks could not express: // before this statement the reader answered "nothing is being deleted", @@ -299,6 +364,7 @@ export async function updateSettings( // client released inside an open transaction is a connection the next // caller inherits mid-transaction. await client.query('ROLLBACK').catch(() => {}) + if (isLockTimeout(err)) throw new SettingsLockedError() throw err } finally { client.release() From ef46bae4eb9e2675d0d14f4764f4d06ee17cbd76 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 19:48:45 +0000 Subject: [PATCH 49/65] stack: a click, through everything, and back out as a number --- test/stack/docker-compose.tls.yml | 10 + test/stack/reports.test.ts | 329 ++++++++++++++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 test/stack/reports.test.ts diff --git a/test/stack/docker-compose.tls.yml b/test/stack/docker-compose.tls.yml index c413841..6c986e3 100644 --- a/test/stack/docker-compose.tls.yml +++ b/test/stack/docker-compose.tls.yml @@ -29,6 +29,9 @@ services: # through the API rather than through the CLI. - admin.example.test - pw.example.test + # A link domain the reporting suite clicks through, added with the + # CLI's escape hatch rather than through a published DNS token. + - rep.example.test worker: environment: @@ -36,6 +39,13 @@ services: CLICKMONK_DNS_CHECK: "on" CLICKMONK_DNS_SERVERS: 172.30.0.53 CLICKMONK_DNS_CHECK_INTERVAL_MS: "2000" + # An hour is the shipped period, and an hour means exactly one pass ever + # runs in a suite — the one at start, over an empty table. A test that a + # period is a floor has to watch a pass decide about a partition that + # holds clicks, so a pass has to happen after they arrive. Five seconds + # rather than two: a writer waits two seconds for the settings row the + # pass holds, and one suite does write it. + CLICKMONK_RETENTION_INTERVAL_MS: "5000" # A miniature ACME server. NOSLEEP and NONCEREJECT turn off the randomness # it adds on purpose to shake out client bugs, which here would only make diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts new file mode 100644 index 0000000..9a0efdb --- /dev/null +++ b/test/stack/reports.test.ts @@ -0,0 +1,329 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import { + ADMIN_HOST, + CA_MOUNT, + type CurlResult, + WAIT_TIMEOUT, + cli, + cliWithInput, + compose, + curl, + publishZone, + until, + writeAcmeRoot, + writeIssuingRoot, +} from './stack.js' + +/** + * A click, all the way through, and then read back out. + * + * Every other test of these reports inserts its rows into ClickHouse itself, so + * none of them can fail when the admin container is given the wrong variable + * name for the store, when Compose forgets to give it one at all, or when a + * real click never reaches the rollup. This suite clicks a link through Caddy + * and reads the number back through the admin host over real HTTPS, which is + * the one path where every link in the chain has to be right. + */ +const EMAIL = 'admin@example.com' +const PASSWORD = 'a decent admin password' +const LINK_HOST = 'rep.example.test' +const SLUG = 'r1' +const TARGET = 'https://example.com/report' +/** Clicks to make. Three, so a count of one or of every request is visibly wrong. */ +const CLICKS = 3 + +/** + * What the redirect answers a name it does not serve, in plain text, and the + * header the admin API puts on every one of its own answers including its + * refusals. A status alone cannot tell the two services apart — the redirect's + * 404 for an unknown slug and the admin API's 404 for a host it does not answer + * on are the same number — so the test that no report is served on a link domain + * reads both. + */ +const REDIRECT_404 = 'Not found.\n' +const ADMIN_ONLY_HEADER = /^content-security-policy:/im + +let root = '' +let cookie = '' +let setUp = false +let failures = 0 +/** The address the client container used, read from the client rather than written here. */ +let clientIp = '' +/** + * How many retention passes had logged a result by the time the clicks were + * countable. The pass the retention test reads has to be one that ran with the + * clicks already in the store, and this is the only way to say that without + * sleeping: it waits for the count to grow. + */ +let passesWithClicks = 0 + +function api( + method: string, + path: string, + o: { body?: string; cookie?: string; origin?: string } = {}, +): CurlResult { + const args = ['--cacert', root, '--max-time', '60', '-X', method, '-D', '-'] + if (o.body !== undefined) { + args.push('-H', 'content-type: application/json', '--data-binary', o.body) + } + if (o.cookie) args.push('-H', `cookie: ${o.cookie}`) + if (o.origin !== undefined) args.push('-H', `origin: ${o.origin}`) + args.push(`https://${ADMIN_HOST}${path}`) + return curl(args, CA_MOUNT, { body: true }) +} + +/** The window every request below asks for: a day either side of now, in UTC. */ +function window(): string { + const now = Date.now() + const from = new Date(now - 86_400_000).toISOString() + const to = new Date(now + 86_400_000).toISOString() + return `from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}` +} + +const cookieFrom = (headers: string): string => { + const line = headers.split('\n').find((l) => /^set-cookie:/i.test(l)) ?? '' + return (line.slice(line.indexOf(':') + 1).split(';')[0] ?? '').trim() +} + +/** + * The result line every retention pass writes, one per pass. The line the worker + * writes at start says the pass was *started*; this one says a pass *ran*, and + * what it decided, which is the difference between the two things the retention + * tests below want to know. + */ +const PASS_LINE = /retention: dropped (\d+), blanked (\d+), skipped \d+/g + +const passes = (): string[] => + [...compose('logs', '--no-color', 'worker').matchAll(PASS_LINE)].map((m) => m[0] as string) + +afterEach((ctx) => { + if (ctx.task.result?.state === 'fail') failures++ +}) + +beforeAll(async () => { + compose('down', '-v') + writeAcmeRoot() + publishZone() + compose('up', '-d', '--build', '--wait', ...WAIT_TIMEOUT) + root = writeIssuingRoot() + cliWithInput(PASSWORD, 'admin', 'create', EMAIL) + // No DNS proof for this host: the escape hatch exists for exactly this, and + // the suite that tests verification is the one that publishes a token. + cli('domain', 'add', LINK_HOST, '--verified') + cli('link', 'add', LINK_HOST, SLUG, '--target', TARGET) + + // The certificate for the admin host is obtained on the first HTTPS request, + // so wait for the handshake before signing in. + await until('a certificate for the admin host', 120_000, () => api('GET', '/api/me').exit === 0) + const signedIn = api('POST', '/api/session', { + body: JSON.stringify({ email: EMAIL, password: PASSWORD }), + origin: `https://${ADMIN_HOST}`, + }) + if (signedIn.status !== 200) + throw new Error(`sign-in failed: ${signedIn.status} ${signedIn.body}`) + cookie = cookieFrom(signedIn.headers) + + // The clicks. Plain HTTP, which is what a link domain with no certificate + // answers on, and the client's own address is read back out of the last one. + // + // No wait for the redirect to know the link: a write to `links` notifies, the + // snapshot reloads a quarter of a second later, and the certificate wait and + // the sign-in above both happen in between. A 404 here is therefore a link + // that was never created rather than one that has not arrived yet, which is + // what the message says. + for (let i = 0; i < CLICKS; i++) { + const r = curl(['--max-time', '30', '-D', '-', `http://${LINK_HOST}/${SLUG}`], [], { + body: true, + }) + if (r.status !== 302) throw new Error(`click ${i} answered ${r.status}: ${r.body}`) + clientIp = r.ip + } + + // A segment seals after two seconds or 8 MB, and the shipper takes a pass a + // second: a few seconds in normal operation, waited for rather than slept + // through. + await until('the clicks to reach the rollups', 120_000, () => { + const r = api('GET', `/api/reports/summary?${window()}`, { cookie }) + if (r.status !== 200) return false + return (JSON.parse(r.body) as { clicks: number }).clicks >= CLICKS + }) + passesWithClicks = passes().length + setUp = true +}, 900_000) + +afterAll(() => { + try { + if (failures > 0 || !setUp) console.error(compose('logs', '--no-color', '--tail', '200')) + } finally { + compose('down', '-v') + } +}, 300_000) + +describe('a report of real clicks', () => { + it('counts them, and counts the one client as one visitor', () => { + const r = api('GET', `/api/reports/summary?${window()}`, { cookie }) + expect(r.status, r.body).toBe(200) + const body = JSON.parse(r.body) as { + clicks: number + visitors: number + byOutcome: Record + newestHour: string | null + } + expect(body.clicks).toBe(CLICKS) + // Three requests from one client that kept no cookies: three visitor ids, + // because a visitor without a cookie is a new visitor. Stated rather than + // asserted as one, because it is the honest behaviour and the README says + // so. + expect(body.visitors).toBe(CLICKS) + expect(body.byOutcome.target).toBe(CLICKS) + expect(body.newestHour).not.toBeNull() + }) + + it('puts them in the chart, in the hour they happened', () => { + const r = api('GET', `/api/reports/timeseries?${window()}&bucket=hour`, { cookie }) + expect(r.status, r.body).toBe(200) + const buckets = (JSON.parse(r.body) as { buckets: { clicks: number }[] }).buckets + expect(buckets.reduce((n, b) => n + b.clicks, 0)).toBe(CLICKS) + // Two days of hours, all present, because a chart fills its gaps. + expect(buckets.length).toBeGreaterThan(24) + }) + + it('breaks them down by the target rotation chose', () => { + const r = api('GET', `/api/reports/breakdown?${window()}&dimension=target`, { cookie }) + expect(r.status, r.body).toBe(200) + const rows = (JSON.parse(r.body) as { rows: { value: string; clicks: number }[] }).rows + expect(rows).toHaveLength(1) + expect(rows[0]?.clicks).toBe(CLICKS) + // One target, so its id is the only value: a uuid, and not an empty string. + expect(rows[0]?.value).toMatch(/^[0-9a-f-]{36}$/) + }) + + it('shows them in the log as networks, never as the address they came from', () => { + const r = api('GET', `/api/clicks?${window()}`, { cookie }) + expect(r.status, r.body).toBe(200) + const clicks = (JSON.parse(r.body) as { clicks: { network: string; destination: string }[] }) + .clicks + expect(clicks).toHaveLength(CLICKS) + expect(clicks[0]?.destination).toBe(TARGET) + expect(clicks[0]?.network).toMatch(/\/(24|64)$/) + // The whole address the client really used, which is read from the client + // and never written down here: no response may contain it. + expect(clientIp.length).toBeGreaterThan(0) + expect(r.body).not.toContain(clientIp) + }) + + // One assertion here goes through Caddy rather than through `inject`, and a + // reverse proxy is allowed to buffer a small chunked body and add a length of + // its own. If `content-length` turns up on this response, read the whole + // header block and Caddy's configuration before touching the assertion: the + // property that matters is that this service did not buffer the file, which + // the unit test pins directly. A length added by the proxy is a different + // fact, and it belongs in the README's limits rather than in a relaxed + // assertion here. + it('exports them as a file, streamed, and says it was not cut', () => { + const r = api('GET', `/api/clicks.csv?${window()}`, { cookie }) + expect(r.status, r.body).toBe(200) + expect(r.headers).toMatch(/^content-type: text\/csv; charset=utf-8/im) + expect(r.headers).toMatch(/^content-disposition: attachment; filename="clicks-/im) + expect(r.headers).toMatch(/^x-clickmonk-truncated: false/im) + // Streamed, so no length was known in advance. + expect(r.headers).not.toMatch(/^content-length:/im) + // Split on the newline alone, not on the CRLF this file is really written + // with: the client helper normalises every CRLF in what curl wrote so that + // a header block can be told from a body at all, so the line ending cannot + // be read back out here. It is not unpinned — the unit test compares a + // written line byte for byte. What this counts is the rows. + const lines = r.body.split('\n').filter((l) => l.length > 0) + expect(lines).toHaveLength(CLICKS + 1) + expect(lines[0]).toContain('"network"') + expect(r.body).not.toContain(clientIp) + }) + + it('answers a script holding a key, not only a browser holding a cookie', () => { + const made = api('POST', '/api/keys', { + body: JSON.stringify({ name: 'reporting' }), + cookie, + origin: `https://${ADMIN_HOST}`, + }) + expect(made.status, made.body).toBe(201) + const key = (JSON.parse(made.body) as { key: string }).key + const r = curl( + [ + '--cacert', + root, + '--max-time', + '60', + '-D', + '-', + '-H', + `authorization: Bearer ${key}`, + `https://${ADMIN_HOST}/api/reports/summary?${window()}`, + ], + CA_MOUNT, + { body: true }, + ) + expect(r.status, r.body).toBe(200) + expect((JSON.parse(r.body) as { clicks: number }).clicks).toBe(CLICKS) + }) + + it('does not answer a report on a link domain', () => { + const r = curl(['--max-time', '30', '-D', '-', `http://${LINK_HOST}/api/reports/summary`], [], { + body: true, + }) + // The redirect's own 404 for an unknown slug, not the admin API's anything: + // the status is the same number on both services, so the body and the header + // the admin API never leaves off are what say which one answered. + // + // **This request is itself a click**, recorded with outcome `not_found`, so + // from here on the number of clicks in the window is four and not three. + // Anything after this one reads an outcome rather than the total. + expect(r.status).toBe(404) + expect(r.body).toBe(REDIRECT_404) + expect(r.headers).not.toMatch(ADMIN_ONLY_HEADER) + }) +}) + +describe('the install', () => { + it('says how long it keeps what it records', () => { + const out = cli('settings', 'show') + expect(out).toContain('keep clicks: 90 days') + expect(out).toContain('keep addresses: 30 days') + }) + + it('takes a shorter period and reports it back', () => { + cli('settings', 'set', '--keep-clicks', '30', '--keep-addresses', '7') + const r = api('GET', '/api/settings', { cookie }) + expect(r.status, r.body).toBe(200) + expect((JSON.parse(r.body) as { retention: unknown }).retention).toEqual({ + rawRetentionDays: 30, + ipRetentionDays: 7, + }) + }) + + it('is running the retention pass, and has deleted nothing', async () => { + // A period is a floor and the partition is this month, so the right + // behaviour here is that a pass ran and dropped nothing. Two passes past the + // count taken once the clicks were countable, so at least one of them began + // after they were in the store: a pass logs its result at the end, so the + // first one past that count could have read the partition before them. + await until( + 'a retention pass over a partition holding the clicks', + 120_000, + () => passes().length >= passesWithClicks + 2, + ) + // The line the worker writes at start, which says the pass was started at + // all, and then every pass's own result. Each pass decided about a partition + // whose month has not ended, and every one of them left it alone. + expect(compose('logs', '--no-color', 'worker')).toContain('retention periods') + for (const line of passes()) expect(line).toMatch(/dropped 0, blanked 0,/) + // What would have changed if any of them had been wrong. The clicks that + // reached a target, not the total: the 404 above is a click too, so the total + // here is four, and a number that counts something a test did on its way past + // is a number that changes when the tests are reordered. A dropped partition + // takes every outcome with it, so this sees it either way. + const r = api('GET', `/api/reports/summary?${window()}`, { cookie }) + expect(r.status, r.body).toBe(200) + const body = JSON.parse(r.body) as { byOutcome: Record } + expect(body.byOutcome.target).toBe(CLICKS) + }) +}) From ca67dc8808e5b721c5b98a7a7e8e291b7aa36f9e Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 20:08:54 +0000 Subject: [PATCH 50/65] stack: a wait that names what never happened --- test/stack/reports.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts index 9a0efdb..a599e49 100644 --- a/test/stack/reports.test.ts +++ b/test/stack/reports.test.ts @@ -306,9 +306,14 @@ describe('the install', () => { // count taken once the clicks were countable, so at least one of them began // after they were in the store: a pass logs its result at the end, so the // first one past that count could have read the partition before them. + // Half the file's test timeout, so a worker that never runs a pass fails + // with the sentence naming what was missing rather than with vitest's own + // "test timed out", which says nothing about retention. Measured: with the + // pass not started at all, the two deadlines landed on the same millisecond + // and the timeout won. await until( 'a retention pass over a partition holding the clicks', - 120_000, + 60_000, () => passes().length >= passesWithClicks + 2, ) // The line the worker writes at start, which says the pass was started at From 710d3a46151fb4c86cfaf0e9dc978ad44dda62a8 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:09:39 +0000 Subject: [PATCH 51/65] stack: assertions a stub could not satisfy --- test/stack/reports.test.ts | 267 ++++++++++++++++++++++++++----------- 1 file changed, 192 insertions(+), 75 deletions(-) diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts index a599e49..263c0fd 100644 --- a/test/stack/reports.test.ts +++ b/test/stack/reports.test.ts @@ -31,6 +31,17 @@ const SLUG = 'r1' const TARGET = 'https://example.com/report' /** Clicks to make. Three, so a count of one or of every request is visibly wrong. */ const CLICKS = 3 +const HOUR_MS = 3_600_000 +/** + * An address per click, so that "no response carries the address a click came + * from" is a claim about every row rather than about whichever row happens to + * carry the newest client's. Docker hands a container the next free address in + * the pool, which gave all three of these the same one when the suite let it + * choose — and an older row leaking its address then passed, because the address + * it leaked was the newest one. High in the range the compose file declares, so + * they are clear of the addresses the stack's own containers are allocated. + */ +const CLIENT_IPS = ['172.30.0.201', '172.30.0.202', '172.30.0.203'] /** * What the redirect answers a name it does not serve, in plain text, and the @@ -47,15 +58,25 @@ let root = '' let cookie = '' let setUp = false let failures = 0 -/** The address the client container used, read from the client rather than written here. */ -let clientIp = '' /** - * How many retention passes had logged a result by the time the clicks were - * countable. The pass the retention test reads has to be one that ran with the - * clicks already in the store, and this is the only way to say that without - * sleeping: it waits for the count to grow. + * The addresses the client containers reported for themselves, one per click. + * Read back out of each client rather than trusted to be the ones asked for + * above, and asserted to be three different addresses before anything is + * concluded from them. */ -let passesWithClicks = 0 +const clientIps: string[] = [] +/** + * The hours the clicks could have landed in: the hour the first went out and the + * hour the last did. One hour in every run so far — three requests take under a + * second — and two if a run crosses a boundary between them, which is why it is + * a list. + */ +let clickHours: string[] = [] +/** + * The outcome on the one row the retention test writes for itself, which is how + * that row is counted and told apart from every click this suite really made. + */ +const OLD_OUTCOME = 'expired' function api( method: string, @@ -72,29 +93,76 @@ function api( return curl(args, CA_MOUNT, { body: true }) } -/** The window every request below asks for: a day either side of now, in UTC. */ +/** Hours either side of the hour this run is in that the window below covers. */ +const WINDOW_HOURS_EITHER_SIDE = 24 + +/** + * The window every request below asks for, in UTC, with both ends **on the + * hour**. + * + * Aligned rather than "now, give or take a day", because a report raises an + * unaligned `to` to the next hour and the chart then holds 48 buckets or 49 + * depending on what minute the run started. Aligned, the count is one number and + * the chart test can assert it. + */ function window(): string { - const now = Date.now() - const from = new Date(now - 86_400_000).toISOString() - const to = new Date(now + 86_400_000).toISOString() + const hour = Math.floor(Date.now() / HOUR_MS) * HOUR_MS + const from = new Date(hour - WINDOW_HOURS_EITHER_SIDE * HOUR_MS).toISOString() + const to = new Date(hour + WINDOW_HOURS_EITHER_SIDE * HOUR_MS).toISOString() return `from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}` } +/** The hour an instant falls in, spelled the way a chart bucket spells it. */ +const hourOf = (ms: number): string => new Date(Math.floor(ms / HOUR_MS) * HOUR_MS).toISOString() + const cookieFrom = (headers: string): string => { const line = headers.split('\n').find((l) => /^set-cookie:/i.test(l)) ?? '' return (line.slice(line.indexOf(':') + 1).split(';')[0] ?? '').trim() } /** - * The result line every retention pass writes, one per pass. The line the worker - * writes at start says the pass was *started*; this one says a pass *ran*, and - * what it decided, which is the difference between the two things the retention - * tests below want to know. + * One statement against ClickHouse, run from the worker container and therefore + * through the four variables the install gives it: the store is reached with the + * install's own configuration, and no password is written here. + * + * It exists because no report can answer what the retention test asks. Every + * report reads a rollup; the pass drops partitions of `clicks` and blanks + * addresses in `clicks`, and never touches a rollup. A pass that had deleted + * every raw click would leave the summary, the chart and the breakdown answering + * exactly what they answered before. + * + * POST with the statement as the body rather than a query parameter, because the + * HTTP interface treats a GET as read-only and the retention test writes a row. */ -const PASS_LINE = /retention: dropped (\d+), blanked (\d+), skipped \d+/g +const RUN_IN_STORE = ` +const u = new URL(process.env.CLICKMONK_CLICKHOUSE_URL) +u.searchParams.set('database', process.env.CLICKMONK_CLICKHOUSE_DB) +fetch(u, { + method: 'POST', + body: process.env.CM_SQL, + headers: { + 'x-clickhouse-user': process.env.CLICKMONK_CLICKHOUSE_USER, + 'x-clickhouse-key': process.env.CLICKMONK_CLICKHOUSE_PASSWORD, + }, +}).then(async (r) => { + const text = await r.text() + if (!r.ok) throw new Error(r.status + ' ' + text) + process.stdout.write(text) +}) +` + +const storeRun = (sql: string): string => + compose('exec', '-T', '-e', `CM_SQL=${sql}`, 'worker', 'node', '-e', RUN_IN_STORE) -const passes = (): string[] => - [...compose('logs', '--no-color', 'worker').matchAll(PASS_LINE)].map((m) => m[0] as string) +function storeCount(sql: string): number { + const out = storeRun(sql).trim() + const n = Number(out) + // Never a silent NaN or a silent zero: an unreachable store answers nothing and + // `Number('')` is 0, which would reach an assertion looking like a count of + // none. + if (out === '' || !Number.isInteger(n)) throw new Error(`the store answered "${out}"`) + return n +} afterEach((ctx) => { if (ctx.task.result?.state === 'fail') failures++ @@ -131,13 +199,21 @@ beforeAll(async () => { // the sign-in above both happen in between. A 404 here is therefore a link // that was never created rather than one that has not arrived yet, which is // what the message says. + // + // One address per client, and `-4` so that the address the client reports for + // itself is the one that was asked for: a container on this network also gets + // an IPv6 address, which `--ip` does not set, and curl would prefer it. + const startedMs = Date.now() for (let i = 0; i < CLICKS; i++) { - const r = curl(['--max-time', '30', '-D', '-', `http://${LINK_HOST}/${SLUG}`], [], { - body: true, - }) + const r = curl( + ['-4', '--max-time', '30', '-D', '-', `http://${LINK_HOST}/${SLUG}`], + ['--ip', CLIENT_IPS[i] as string], + { body: true }, + ) if (r.status !== 302) throw new Error(`click ${i} answered ${r.status}: ${r.body}`) - clientIp = r.ip + clientIps.push(r.ip) } + clickHours = [...new Set([hourOf(startedMs), hourOf(Date.now())])] // A segment seals after two seconds or 8 MB, and the shipper takes a pass a // second: a few seconds in normal operation, waited for rather than slept @@ -147,7 +223,6 @@ beforeAll(async () => { if (r.status !== 200) return false return (JSON.parse(r.body) as { clicks: number }).clicks >= CLICKS }) - passesWithClicks = passes().length setUp = true }, 900_000) @@ -182,10 +257,18 @@ describe('a report of real clicks', () => { it('puts them in the chart, in the hour they happened', () => { const r = api('GET', `/api/reports/timeseries?${window()}&bucket=hour`, { cookie }) expect(r.status, r.body).toBe(200) - const buckets = (JSON.parse(r.body) as { buckets: { clicks: number }[] }).buckets - expect(buckets.reduce((n, b) => n + b.clicks, 0)).toBe(CLICKS) - // Two days of hours, all present, because a chart fills its gaps. - expect(buckets.length).toBeGreaterThan(24) + const buckets = (JSON.parse(r.body) as { buckets: { at: string; clicks: number }[] }).buckets + // Two days of hours, every one of them there because a chart fills its gaps. + // The number and not a floor: "more than 24" is satisfied by 25, and the + // window is aligned at both ends precisely so this can be one number. + expect(buckets.length).toBe(48) + // Which bucket carries them, not only that some bucket does. Shifting the + // fill's lookup key by an hour leaves the length and the total untouched and + // draws the clicks in the wrong hour, which is the whole of what a chart + // says. + const carrying = buckets.filter((b) => b.clicks > 0) + expect(carrying.reduce((n, b) => n + b.clicks, 0)).toBe(CLICKS) + for (const b of carrying) expect(clickHours).toContain(b.at) }) it('breaks them down by the target rotation chose', () => { @@ -204,22 +287,32 @@ describe('a report of real clicks', () => { const clicks = (JSON.parse(r.body) as { clicks: { network: string; destination: string }[] }) .clicks expect(clicks).toHaveLength(CLICKS) - expect(clicks[0]?.destination).toBe(TARGET) - expect(clicks[0]?.network).toMatch(/\/(24|64)$/) - // The whole address the client really used, which is read from the client - // and never written down here: no response may contain it. - expect(clientIp.length).toBeGreaterThan(0) - expect(r.body).not.toContain(clientIp) + for (const c of clicks) { + expect(c.destination).toBe(TARGET) + expect(c.network).toMatch(/\/(24|64)$/) + } + // Three different addresses, asserted before anything is concluded from them: + // the check below is only about every row while the rows carry addresses that + // differ, and letting Docker choose gave all three clients the same one. + expect(new Set(clientIps).size).toBe(CLICKS) + for (const ip of clientIps) expect(r.body).not.toContain(ip) }) - // One assertion here goes through Caddy rather than through `inject`, and a - // reverse proxy is allowed to buffer a small chunked body and add a length of - // its own. If `content-length` turns up on this response, read the whole - // header block and Caddy's configuration before touching the assertion: the - // property that matters is that this service did not buffer the file, which - // the unit test pins directly. A length added by the proxy is a different - // fact, and it belongs in the README's limits rather than in a relaxed - // assertion here. + // **What the absent length does and does not prove.** It proves no length was + // decided before the first row, which is what a handler that hands back the + // whole file as a string does: measured, that shape fails here and fails + // exactly one unit test, `holds nothing: no content-length, and a chunked + // body`. It does not prove the file was never held in memory — a body read to + // the end and handed over as a single-chunk stream is still chunked and still + // carries no length, and measured, it passes here and passes all 398 of the + // admin service's unit tests. Nothing catches that shape today. Do not read + // this assertion as more than the header it reads. + // + // This one answer travels through Caddy rather than through `inject`, and a + // reverse proxy may buffer a small chunked body and add a length of its own. + // Measured on this stack, Caddy adds none. If one turns up, read the whole + // header block and Caddy's configuration before touching the assertion: a + // length added by the proxy is a different fact from a length added here. it('exports them as a file, streamed, and says it was not cut', () => { const r = api('GET', `/api/clicks.csv?${window()}`, { cookie }) expect(r.status, r.body).toBe(200) @@ -231,12 +324,13 @@ describe('a report of real clicks', () => { // Split on the newline alone, not on the CRLF this file is really written // with: the client helper normalises every CRLF in what curl wrote so that // a header block can be told from a body at all, so the line ending cannot - // be read back out here. It is not unpinned — the unit test compares a + // be read back out here. It is not unpinned — `csv.test.ts` compares a // written line byte for byte. What this counts is the rows. const lines = r.body.split('\n').filter((l) => l.length > 0) expect(lines).toHaveLength(CLICKS + 1) expect(lines[0]).toContain('"network"') - expect(r.body).not.toContain(clientIp) + expect(new Set(clientIps).size).toBe(CLICKS) + for (const ip of clientIps) expect(r.body).not.toContain(ip) }) it('answers a script holding a key, not only a browser holding a cookie', () => { @@ -291,44 +385,67 @@ describe('the install', () => { }) it('takes a shorter period and reports it back', () => { + // Through the API before the change as well as after. With only the second + // read, two constants in the settings response satisfy this test: nothing + // would have seen this surface answer anything else. + const before = api('GET', '/api/settings', { cookie }) + expect(before.status, before.body).toBe(200) + expect((JSON.parse(before.body) as { retention: unknown }).retention).toEqual({ + rawRetentionDays: 90, + ipRetentionDays: 30, + }) cli('settings', 'set', '--keep-clicks', '30', '--keep-addresses', '7') - const r = api('GET', '/api/settings', { cookie }) - expect(r.status, r.body).toBe(200) - expect((JSON.parse(r.body) as { retention: unknown }).retention).toEqual({ + const after = api('GET', '/api/settings', { cookie }) + expect(after.status, after.body).toBe(200) + expect((JSON.parse(after.body) as { retention: unknown }).retention).toEqual({ rawRetentionDays: 30, ipRetentionDays: 7, }) }) - it('is running the retention pass, and has deleted nothing', async () => { - // A period is a floor and the partition is this month, so the right - // behaviour here is that a pass ran and dropped nothing. Two passes past the - // count taken once the clicks were countable, so at least one of them began - // after they were in the store: a pass logs its result at the end, so the - // first one past that count could have read the partition before them. - // Half the file's test timeout, so a worker that never runs a pass fails - // with the sentence naming what was missing rather than with vitest's own - // "test timed out", which says nothing about retention. Measured: with the - // pass not started at all, the two deadlines landed on the same millisecond - // and the timeout won. - await until( - 'a retention pass over a partition holding the clicks', - 60_000, - () => passes().length >= passesWithClicks + 2, + it('drops a month past the period it was just given, and keeps the month it is in', async () => { + // **A pass needs something to delete before a test can watch it delete.** + // Nothing that can be clicked is past a period on a fresh install: a period is + // a floor, partitions go whole, and the month this run is in has not ended. So + // a pass here has no work, and a test that only watched for one — a log line, + // a count of passes — is satisfied by a pass that prints that it ran and + // returns. This row is the work: a click in a month that ended long before the + // thirty days set above, written straight into the store because no click can + // be made 120 days ago. + // + // Its own outcome, so that it is counted apart from every real click, and an + // address from the documentation range, so that the row the pass considers is + // shaped like the rows beside it. + const oldMs = Date.now() - 120 * 24 * HOUR_MS + const at = new Date(oldMs).toISOString().replace('T', ' ').replace('Z', '') + storeRun( + `INSERT INTO clicks (click_id, time, outcome, ip) + VALUES (generateUUIDv4(), toDateTime64('${at}', 3, 'UTC'), '${OLD_OUTCOME}', '198.51.100.7')`, ) - // The line the worker writes at start, which says the pass was started at - // all, and then every pass's own result. Each pass decided about a partition - // whose month has not ended, and every one of them left it alone. - expect(compose('logs', '--no-color', 'worker')).toContain('retention periods') - for (const line of passes()) expect(line).toMatch(/dropped 0, blanked 0,/) - // What would have changed if any of them had been wrong. The clicks that - // reached a target, not the total: the 404 above is a click too, so the total - // here is four, and a number that counts something a test did on its way past - // is a number that changes when the tests are reordered. A dropped partition - // takes every outcome with it, so this sees it either way. - const r = api('GET', `/api/reports/summary?${window()}`, { cookie }) - expect(r.status, r.body).toBe(200) - const body = JSON.parse(r.body) as { byOutcome: Record } - expect(body.byOutcome.target).toBe(CLICKS) + const old = `SELECT count() FROM clicks WHERE outcome = '${OLD_OUTCOME}'` + expect(storeCount(old)).toBe(1) + + // The pass, watched for by its effect rather than by anything it printed. + // Sixty seconds against a five-second interval, and half the file's test + // timeout so that a worker which never runs a pass fails with the sentence + // naming what was missing: measured, with the pass not started at all the two + // deadlines landed on the same millisecond and vitest's own "test timed out" + // won, which says nothing about retention. + await until('the pass to drop the month past the period', 60_000, () => storeCount(old) === 0) + + // And the month this run is in, which the same pass considered and had to + // leave alone: every raw click still there, and every one of them still + // holding the address it came from. Read out of `clicks` and not out of a + // report, because the pass deletes from `clicks` and no report reads it — a + // pass that dropped every raw click would leave every report answering exactly + // what it answered before, which is how a test that read one of them passed + // with every raw click gone. + // + // Counted by outcome rather than as a total, because the 404 further up is a + // click too: a number that counts what another test did on its way past + // changes when the tests are reordered. + const kept = `FROM clicks WHERE outcome = 'target'` + expect(storeCount(`SELECT uniqExact(click_id) ${kept}`)).toBe(CLICKS) + expect(storeCount(`SELECT uniqExact(click_id) ${kept} AND ip != ''`)).toBe(CLICKS) }) }) From c602b0f4bb20611969b7888e62dcc9e228193b8d Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:12:04 +0000 Subject: [PATCH 52/65] stack: names and comments that match what is proved --- test/stack/reports.test.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts index 263c0fd..67573f0 100644 --- a/test/stack/reports.test.ts +++ b/test/stack/reports.test.ts @@ -235,7 +235,7 @@ afterAll(() => { }, 300_000) describe('a report of real clicks', () => { - it('counts them, and counts the one client as one visitor', () => { + it('counts every click, under the outcome it took, and says how fresh it is', () => { const r = api('GET', `/api/reports/summary?${window()}`, { cookie }) expect(r.status, r.body).toBe(200) const body = JSON.parse(r.body) as { @@ -245,10 +245,16 @@ describe('a report of real clicks', () => { newestHour: string | null } expect(body.clicks).toBe(CLICKS) - // Three requests from one client that kept no cookies: three visitor ids, - // because a visitor without a cookie is a new visitor. Stated rather than - // asserted as one, because it is the honest behaviour and the README says - // so. + // **This number is the one above, and the test is not named for it.** Three + // requests that kept no cookies are three visitors, because a request with no + // visitor cookie is minted a new id — `mints a new visitor when there is no + // cookie` in the redirect's own suite is where that is pinned, not the README, + // which does not say it. So `visitors` here is a second reading of the click + // count and replacing it with that count would pass. It is asserted because + // the field has to be populated at all in the shipped stack; the property + // worth pinning — one visitor merged across buckets rather than summed — needs + // a visitor who clicked twice, which is `gives one bucket a day, and counts a + // visitor across the hours of that day once` in the admin service's suite. expect(body.visitors).toBe(CLICKS) expect(body.byOutcome.target).toBe(CLICKS) expect(body.newestHour).not.toBeNull() From 688d0aacde419ab62a01ffd333721f843cff58a2 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:13:33 +0000 Subject: [PATCH 53/65] stack: a premise a pass could remove before it was read --- test/stack/docker-compose.tls.yml | 7 ------- test/stack/reports.test.ts | 20 ++++++++++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/test/stack/docker-compose.tls.yml b/test/stack/docker-compose.tls.yml index 6c986e3..4bde35e 100644 --- a/test/stack/docker-compose.tls.yml +++ b/test/stack/docker-compose.tls.yml @@ -39,13 +39,6 @@ services: CLICKMONK_DNS_CHECK: "on" CLICKMONK_DNS_SERVERS: 172.30.0.53 CLICKMONK_DNS_CHECK_INTERVAL_MS: "2000" - # An hour is the shipped period, and an hour means exactly one pass ever - # runs in a suite — the one at start, over an empty table. A test that a - # period is a floor has to watch a pass decide about a partition that - # holds clicks, so a pass has to happen after they arrive. Five seconds - # rather than two: a writer waits two seconds for the settings row the - # pass holds, and one suite does write it. - CLICKMONK_RETENTION_INTERVAL_MS: "5000" # A miniature ACME server. NOSLEEP and NONCEREJECT turn off the randomness # it adds on purpose to shake out client bugs, which here would only make diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts index 67573f0..672c515 100644 --- a/test/stack/reports.test.ts +++ b/test/stack/reports.test.ts @@ -431,12 +431,20 @@ describe('the install', () => { const old = `SELECT count() FROM clicks WHERE outcome = '${OLD_OUTCOME}'` expect(storeCount(old)).toBe(1) - // The pass, watched for by its effect rather than by anything it printed. - // Sixty seconds against a five-second interval, and half the file's test - // timeout so that a worker which never runs a pass fails with the sentence - // naming what was missing: measured, with the pass not started at all the two - // deadlines landed on the same millisecond and vitest's own "test timed out" - // won, which says nothing about retention. + // **The pass is provoked, not waited for.** A pass runs at start and then once + // an interval, and the interval is the shipped hour, so restarting the worker + // is a pass on demand — and it is what makes the count above a premise rather + // than a race. On a short interval a pass lands between the insert and that + // count often enough to have failed this test on its second run: the row was + // already gone, and "it is there" is not something a test can check while + // something else is removing it. + // + // Watched for by its effect, never by anything it printed. Sixty seconds, half + // the file's test timeout, so that a worker which never runs a pass fails with + // the sentence naming what was missing: measured, with the two deadlines equal + // they landed on the same millisecond and vitest's own "test timed out" won, + // which says nothing about retention. + compose('restart', 'worker') await until('the pass to drop the month past the period', 60_000, () => storeCount(old) === 0) // And the month this run is in, which the same pass considered and had to From 9a02d39d31910b828210581138288383896f6320 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:29:35 +0000 Subject: [PATCH 54/65] admin: an export that buffers cannot answer when the caller got bytes --- packages/admin/src/clicks.test.ts | 128 ++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index b30707c..84948d1 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -1540,3 +1540,131 @@ describe('GET /api/clicks.csv, a store that fails mid-file', () => { } }) }) + +/** + * A caller reading while the store is still producing. + * + * **The property this endpoint was built for, and the one nothing here could + * see.** The cap is a million rows, so an export that reads its answer to the end + * before sending any of it holds a file's worth of memory in the process — which + * is the whole reason the rows are streamed. Measured before this existed: an + * implementation that concatenated every chunk and handed the result over as a + * single-chunk stream passed all 398 tests of this service and the stack suite's + * `content-length` assertion too, because a one-chunk stream is still chunked and + * still carries no length. A length is a weaker question than this one. + * + * So the question asked here is the ordering: **were bytes given to the caller + * while the store still had blocks to hand over?** No implementation that buffers + * can answer yes, however it is written, and none that streams can answer no. + * + * A stub store, because a healthy ClickHouse hands over a small result set faster + * than a socket can be read and the two events would not be ordered at all; and a + * real socket, because `inject` resolves once with a whole body and has no + * earlier moment to observe. The stub is what makes the ordering deterministic + * rather than a race: it hands over one block and then **waits to be told the + * caller has bytes**. A streaming server tells it within milliseconds. A + * buffering one cannot, because it is itself waiting for the block, so the wait + * expires, the file is produced whole, and the first bytes reach the caller with + * every block already pulled. + */ +describe('GET /api/clicks.csv, a caller reading while the store produces', () => { + /** Blocks the stub hands over. More than one, so "the last block" means something. */ + const BLOCKS = 8 + /** + * How long the stub waits to hear that the caller has bytes before giving up and + * producing the rest. Only ever paid by an implementation that cannot answer, so + * it is the cost of a failure and not of a pass. + */ + const WAIT_MS = 2000 + + const settle = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) + + interface Producing { + ch: ClickHouseClient + /** How many blocks the store has been asked for so far. */ + pulled: () => number + /** Told by the test the moment the caller is given its first bytes. */ + gotBytes: () => void + } + + function producingCh(): Producing { + let pulled = 0 + let tell = (): void => {} + const bytesReached = new Promise((resolve) => { + tell = resolve + }) + async function* blocks(): AsyncGenerator<{ json: () => ClickRow }[]> { + for (let i = 0; i < BLOCKS; i++) { + pulled++ + const row = { + ...click(), + click_id: `01920000-0000-7000-8000-0000000000${String(i).padStart(2, '0')}`, + at_ms: '1758708000000', + } as unknown as ClickRow + yield [{ json: () => row }] + // The whole arrangement, in one line: after the first block the store will + // not produce another until the caller has been given something. + if (i === 0) await Promise.race([bytesReached, settle(WAIT_MS)]) + } + } + return { + pulled: () => pulled, + gotBytes: () => tell(), + ch: { + query: async ({ query }: { query: string }) => + query.includes('count()') + ? { json: async () => [{ n: String(BLOCKS) }] } + : { close: () => {}, stream: () => Readable.from(blocks(), { objectMode: true }) }, + } as unknown as ClickHouseClient, + } + } + + it('gives the caller bytes before the store has produced the last block', async () => { + const store = producingCh() + const served = testApp(pool, clock, { ch: store.ch }) + try { + await served.listen({ host: '127.0.0.1', port: 0 }) + const at = served.server.address() + const port = typeof at === 'object' && at !== null ? at.port : 0 + /** What the store had produced when the caller was first given bytes. */ + let pulledAtFirstBytes = -1 + let status = 0 + const body = await new Promise((resolve, reject) => { + const req = httpRequest( + { + host: '127.0.0.1', + port, + path: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST, cookie }, + }, + (res) => { + status = res.statusCode ?? 0 + let text = '' + res.setEncoding('utf8') + res.on('data', (chunk: string) => { + if (pulledAtFirstBytes === -1) { + pulledAtFirstBytes = store.pulled() + store.gotBytes() + } + text += chunk + }) + res.on('end', () => resolve(text)) + }, + ) + req.on('error', reject) + req.end() + }) + // The whole file arrived, so none of this is about a request that failed + // early: a refusal has no rows and would make the ordering below vacuous. + expect(status).toBe(200) + expect(body.split('\r\n').filter((l) => l.length > 0)).toHaveLength(BLOCKS + 1) + // And the ordering. Not `-1`: that is "no chunk ever arrived", which the + // length above already rules out and which would otherwise satisfy the + // comparison that follows it. + expect(pulledAtFirstBytes).not.toBe(-1) + expect(pulledAtFirstBytes).toBeLessThan(BLOCKS) + } finally { + await served.close() + } + }) +}) From 6d5c551fc5b91d10058daf661e58174df9f8987d Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:32:17 +0000 Subject: [PATCH 55/65] stack: the test that does catch a buffered export --- test/stack/reports.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts index 672c515..c97af07 100644 --- a/test/stack/reports.test.ts +++ b/test/stack/reports.test.ts @@ -306,13 +306,15 @@ describe('a report of real clicks', () => { // **What the absent length does and does not prove.** It proves no length was // decided before the first row, which is what a handler that hands back the - // whole file as a string does: measured, that shape fails here and fails - // exactly one unit test, `holds nothing: no content-length, and a chunked - // body`. It does not prove the file was never held in memory — a body read to - // the end and handed over as a single-chunk stream is still chunked and still - // carries no length, and measured, it passes here and passes all 398 of the - // admin service's unit tests. Nothing catches that shape today. Do not read - // this assertion as more than the header it reads. + // whole file as a string does: measured, that shape fails here and fails the + // unit test `holds nothing: no content-length, and a chunked body`. It does not + // prove the file was never held in memory — a body read to the end and handed + // over as a single-chunk stream is still chunked and still carries no length, + // and measured, it passes this assertion. What catches that shape is the unit + // test `gives the caller bytes before the store has produced the last block`, + // which asks the question a length cannot: whether the caller was reading while + // the store was still producing. Do not read this assertion as more than the + // header it reads. // // This one answer travels through Caddy rather than through `inject`, and a // reverse proxy may buffer a small chunked body and add a length of its own. From a2c1f8e4607091926563e8e89ae44f25fe66850e Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:54:37 +0000 Subject: [PATCH 56/65] docs: the reports, what they count, and how long anything is kept --- .env.example | 5 +- CLAUDE.md | 68 +++++++++++---- README.md | 232 ++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 265 insertions(+), 40 deletions(-) diff --git a/.env.example b/.env.example index 7901a6f..5026f66 100644 --- a/.env.example +++ b/.env.example @@ -21,8 +21,9 @@ CLICKMONK_DNS_CHECK=on # port ([2001:db8::1]:5353 for IPv6). Empty means the host's own. CLICKMONK_DNS_SERVERS= # How often the worker looks for clicks and addresses past their retention -# periods. An hour is plenty; shorten it only if you are watching it work. The -# periods themselves are settings on the install, not here: +# periods. An hour is plenty; shorten it only if you are watching it work, and +# not below one second, which the worker refuses. The periods themselves are +# settings on the install, not here: # clickmonk settings set --keep-clicks 30 --keep-addresses 7 # Each period is a floor rather than a deadline. Clicks are deleted a whole # month at a time, so 30 days keeps between 30 and 61 days of them, and the diff --git a/CLAUDE.md b/CLAUDE.md index df59966..191b856 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,14 +45,27 @@ run it on their own infrastructure; their click data stays theirs. spool before response, click caps, traffic classification and actions, country rules from an in-memory IP lookup, the password gate on a link), Caddy in front of it with on-demand TLS gated on a verified domain or the configured admin host, the worker (spool -to ClickHouse, migrations on boot, IP data updates, domain DNS verification), the admin +to ClickHouse, migrations on boot, IP data updates, domain DNS verification, the retention +pass), the admin API (one account, sessions, TOTP with recovery codes, API keys, domain/link/settings CRUD, answering on `CLICKMONK_ADMIN_HOST` alone — unset, every route but `/health` is a -503 and links serve exactly as before), the CLI (`migrate`, +503 and links serve exactly as before), reporting over that API (a summary, a chart by +hour or day, a breakdown over nine dimensions, the raw click log and a streamed CSV of it, +all reading hourly rollups except the log and its export), retention (two periods on the +settings row, enforced hourly by the worker, dropping raw clicks a partition at a time and +blanking the address on a click in place), the CLI (`migrate`, `domain add|list|verify`, `link add`, `settings show|set`, `ipdata status|update`, `admin create|passwd`, `admin totp disable`, `apikey create|list|revoke`), `install.sh`, a Compose stack, and the restart durability and stack test suites. +**A retention period is a floor, not a deadline, and it is never defaulted by a reader +that deletes.** `clicks` is partitioned by month and dropped whole, so 90 days keeps 90 to +121 days; and a settings row that is missing or cannot be read as retention means the pass +deletes nothing at all, rather than the defaults. **A report counts whole hours** — it +aligns the window to the rollups' grain and echoes what it counted — while the click log +and the export use the window to the millisecond, so the two can answer the same request +differently and both are right. + **The two services read `CLICKMONK_ADMIN_HOST` differently on purpose.** The admin service refuses to boot on a value it cannot parse; the redirect logs `ADMIN_HOST_IGNORED` once and serves links with the certificate check approving verified link domains only. The redirect @@ -70,20 +83,29 @@ What does not exist yet, and must not be implied by any documentation: - **A way back in over the API when both factors are lost.** Removing the second factor there requires the second factor. `clickmonk admin totp disable` is the answer, and it is a command on the server for that reason. -- **Reports and exports, over the API or anywhere else.** Clicks reach ClickHouse and - nothing reads them back out. The API covers domains, links and settings only. +- **Anything drawn.** The reports are JSON and nothing plots them. There is no dashboard, + so how fresh the numbers are is `newestHour` in the summary and the chart rather than + something on a screen. +- **A CSV of a report.** The export is the click log. A summary or a breakdown is one small + answer already, and turning it into a file belongs to the web interface. +- **A time zone.** Every window, bucket and retention period is UTC, and a preset like + "yesterday" is the caller's to compute. The hourly grain leaves a whole-hour offset + possible later without a schema change; a half-hour zone would need one. - **Notifications of any kind.** No mail configuration exists; `GET /api/alerts` is what an operator reads instead. - **Most link settings in the CLI.** `link add` takes `--target`, `--backup`, `--cap`, `--expires`, `--no-passthrough` and `--action` only, and no command changes a link - after `link add`. `settings set` sets the install-wide traffic actions, the safe URL - and the abuser threshold. Device URLs, a returning URL, country rules, a name, a - password and the disabled state are the admin API's, or hand-written SQL without it. + after `link add`. `settings set` sets the install-wide traffic actions, the safe URL, + the abuser threshold and the two retention periods. Device URLs, a returning URL, + country rules, a name, a password and the disabled state are the admin API's, or + hand-written SQL without it. A returning URL also needs HTTPS to do anything: its cookie is marked `Secure`, so a browser drops it over plain HTTP. - **Proxy/VPN detection beyond Tor exits, cloud providers' published ranges, and region or city.** No licensed VPN or proxy list has been found; datacenter traffic is - recognised by ASN only, since no cloud provider's range file states a licence. + recognised by ASN only, since no cloud provider's range file states a licence. The + region and city columns exist on a click and are always empty, and a breakdown by + either would be a new materialized view and a backfill of it. - **Backup and restore.** - Segments ClickHouse rejects are set aside as `.bad` files, and nothing reports them. - One redirect process per spool directory. @@ -110,16 +132,18 @@ reading a record version ships no later than writing it. A pnpm workspace of TypeScript packages, Node 22, ESM throughout. TypeScript is `strict` with `noUncheckedIndexedAccess`; avoid `any`, and justify it inline on the rare occasion it is unavoidable. Biome handles lint and format. Vitest runs the tests. Two stores: -**Postgres** for domains, links, counters, the admin account and its credentials, and the -single migration ledger; **ClickHouse** for click events. +**Postgres** for domains, links, counters, the admin account and its credentials, the one +install-wide settings row (the traffic actions and the two retention periods) and the +single migration ledger; **ClickHouse** for click events and the hourly rollups over them. ``` packages/core/ pure logic, no I/O: link schemas (zod), the redirect evaluator, device, OS and browser detection, traffic classification, destination tokens, passthrough, rotation, click IDs, the click - record, and the credential primitives (password hashing, opaque - tokens, TOTP, recovery codes, attempt counting). Owns - SCHEMA_VERSION. + record, the vocabulary a report is asked in (bucket sizes, + dimensions, window bounds) and the credential primitives + (password hashing, opaque tokens, TOTP, recovery codes, + attempt counting). Owns SCHEMA_VERSION. packages/db/ Postgres and ClickHouse clients and the migrator. Migrations live in packages/db/migrations/{postgres,clickhouse} as one shared version sequence. @@ -131,8 +155,11 @@ packages/redirect/ the service that answers link domains: an in-memory snapshot writer, the click-cap counter, the password gate and its proof cookie. packages/worker/ ships the spool into ClickHouse; runs migrations on boot; - updates the IP data; checks domain DNS verification on a schedule. -packages/admin/ the admin API: sessions, API keys, TOTP, domain/link/settings CRUD. + updates the IP data; checks domain DNS verification on a schedule; + deletes past the retention periods; owns the one reader and + writer of the settings row that the CLI and the admin API share. +packages/admin/ the admin API: sessions, API keys, TOTP, domain/link/settings CRUD, + the reports, the click log and its CSV export. packages/cli/ `clickmonk migrate | domain | link add | settings | ipdata | admin | apikey`. Two commands here are deliberately not API routes: `admin create`, because nothing can authenticate before the account exists, and @@ -147,8 +174,10 @@ install.sh writes .env once, with fresh secrets, and starts the stack. test/stack/ brings the whole stack up against a local certificate authority and DNS server; proves TLS issuance, domain verification, real client addresses over IPv4 and IPv6, and the admin host — its - certificate, a sign-in over HTTPS, and a password-protected link - answered end to end. + certificate, a sign-in over HTTPS, a password-protected link + answered end to end, a click driven through the whole stack and + read back as a report, a log page and a CSV, and a retention + period lowered and then enforced. ``` Three properties of the product shape every change: @@ -222,7 +251,10 @@ are still pulled over the network, once): a domain gets no certificate until it publishes its verification record, the address every visitor arrives from is the visitor's, and `install.sh` writes its secrets once. They also bring the admin service up, sign in to it over HTTPS on the admin host, and drive a password-protected link from -the form to the redirect. They bind **80 and 443**; the +the form to the redirect. One of them clicks a link through Caddy and then reads that +click back through the admin host as a summary, a chart, a breakdown, a log page and a +CSV, and lowers a retention period and watches the worker enforce it — the one path where +the admin service's own ClickHouse configuration has to be right. They bind **80 and 443**; the durability suite binds 8080 and 8123 and the test databases 8123 and 5433. Run one at a time. diff --git a/README.md b/README.md index 1005e89..fba4c87 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ on your own infrastructure, and your click data stays yours. local disk before it answers, and keeps redirecting while Postgres or ClickHouse is down. - **The worker**, which ships spooled clicks into ClickHouse, runs the database - migrations when it starts, and keeps the IP data up to date. + migrations when it starts, keeps the IP data up to date, and deletes what this install + has stopped keeping. - **TLS on every link domain.** Caddy sits in front and obtains a certificate the first time someone asks for a domain over HTTPS, then renews it without anyone touching a file. It asks ClickMonk first, and ClickMonk says yes only for a domain you added and @@ -32,8 +33,9 @@ on your own infrastructure, and your click data stays yours. image and starts the stack. Running it again changes nothing that is already set. - **An admin API.** One admin account, created on the server with `clickmonk admin create`, a session cookie for a browser and API keys for a script, two-factor - authentication with recovery codes, and endpoints for domains, links and the - install-wide settings. It answers on one host name of its own — set + authentication with recovery codes, and endpoints for domains, links, the + install-wide settings and everything under "Reading the clicks back out" + below. It answers on one host name of its own — set `CLICKMONK_ADMIN_HOST` — and Caddy gets it a certificate the first time you visit it. - **Password-protected links.** Set a password on a link and visitors are asked for it on a ClickMonk page on your own domain before they are sent on. Attempts are counted per @@ -50,6 +52,31 @@ on your own infrastructure, and your click data stays yours. as a bot unless another check gives it a class, and never uses up a click cap. Each click also records its country, network (ASN), operating system and browser. - **Country rules** use the country looked up from the visitor's address. +- **Reports over the admin API.** Clicks come back out: a summary of a window, a chart by + hour or by day, and a breakdown by country, device, operating system, browser, referring + host, the target rotation chose, the traffic class, the action taken or the outcome. They + read hourly rollups written as the clicks arrive rather than the clicks themselves, so a + window of a year is read as hours and not as every click inside it, and every number + counts a click once even if the worker shipped its spool segment twice. **A report counts + whole hours** and says in its answer which hours it counted: ask for + `from=2026-09-24T10:30:00Z` and you are told you were given 10:00 onwards. +- **The click log, and a CSV of it.** `GET /api/clicks` lists the clicks themselves, newest + first, filterable by link, traffic class, outcome, country and time, and exact to the + millisecond. `GET /api/clicks.csv` is the same rows as a file: it streams, so the window + can be as wide as you like, and when it reaches its row cap it says so in a header rather + than handing you a prefix that looks complete. The file identifies visitors, which + "Reading the clicks back out" spells out before you send one to anybody. +- **Addresses are shown as networks, always.** The log and the export show + `198.51.100.0/24` and `2001:db8:1234:5678::/64`, never the address itself — enough to see + a pattern or a bot, and not a file of addresses. The whole address stays in the database + until the address retention period blanks it, because the traffic classification and the + per-client counters need it, and no response reads it back out. +- **Retention.** Raw clicks are kept for 90 days on a new install and the address on them + for 30, both changeable with `clickmonk settings set --keep-clicks` and + `--keep-addresses`, and either can be `never`. The rollups are never deleted, so a chart + or a breakdown still answers for a window whose raw clicks are gone. **Both periods are + floors, not deadlines**: clicks are stored a month at a time and dropped the same way, so + 90 days keeps 90 to 121 days of clicks and 30 days of addresses keeps them 30 to 61. - **The CLI**: `clickmonk migrate`, `clickmonk domain add|list|verify`, `clickmonk link add`, `clickmonk settings show|set`, `clickmonk ipdata status|update`, `clickmonk admin create`, `clickmonk admin passwd`, `clickmonk admin totp disable`, `clickmonk apikey create|list|revoke`. @@ -74,8 +101,28 @@ What does not work yet: every browser out. An API key is the only credential you can hand over and revoke on its own, and a key is not a person — it cannot sign in, and nothing it does is attributed to anyone. The admin API also cannot be tried without a real domain name: see that section. -- **Reports and the click log over the API.** The API covers domains, links and settings. - Clicks reach ClickHouse and nothing reads them back out yet. +- **Anything drawn.** The reports are JSON; nothing plots them. There is no dashboard, so + how fresh the numbers are is a field in the answer — `newestHour`, on the summary and the + chart — rather than something on a screen. +- **A CSV of a report.** The export is the click log. A summary or a breakdown is already one + small answer, and turning it into a file is a job for the web interface rather than + something this API streams. +- **A time zone.** Every window, bucket and retention period is UTC. A preset like + "yesterday" is for whoever is asking to work out. +- **A breakdown by two things at once.** One dimension per request: clicks by country, or + clicks by browser, never clicks by country by browser. +- **A report of a window older than this install's rollups.** They are written as the clicks + arrive and there is nothing to backfill them from, so a window from before this install + had them reads as zeroes rather than as a refusal. +- **A report is only as fresh as the shipper.** Nothing reads the spool: a click that has + not been shipped is in no report, and the shipper stops while ClickHouse is unreachable. + The redirect keeps answering and keeps spooling throughout, so nothing is lost — it has + simply not arrived yet. +- **Reports answer nothing while ClickHouse is down.** `GET /api/reports/…`, + `GET /api/clicks` and `GET /api/clicks.csv` answer 503; domains, links and settings keep + working. +- **An export that fails part way through ends the connection** rather than finishing the + file, because a file that looks complete and is not is worse than a download that broke. - **Any notification.** Nothing is emailed, posted or pushed anywhere: there is no mail configuration and no secret for one. `GET /api/alerts` lists every domain whose last DNS check did not find its token, and every domain no check has reached, which is what an @@ -92,7 +139,8 @@ What does not work yet: used. - **Cloud providers' published address ranges.** Datacenter traffic is recognised by its network (ASN) only. The providers' range files state no licence, so they are not used. -- **Region and city.** A click records its country only. +- **Region and city.** A click records its country only. The two columns exist and are + always empty, and a breakdown by either would need a rollup of its own. - **Backup and restore.** - **Rejected clicks are not reported.** A batch of clicks ClickHouse refuses is set aside as a `.bad` file in the spool, and nothing tells you it is there. @@ -329,15 +377,14 @@ periods cannot be read — and nothing is deleted while it reads that way. A mis the likelier of the two: it is one `DELETE` away, and the defaults would answer it with a period nobody chose. -The worker enforces both periods once an hour, and **each one is a floor rather than a -deadline.** Clicks are stored a month at a time and deleted the same way, so a month of -them goes only once its newest possible click is past the period: 90 days keeps between 90 -and 121 days, and 30 days of addresses keeps them between 30 and 61. Lowering a period -takes effect on the next pass and never rewrites anything already deleted. What is deleted -is the raw clicks and the addresses on them; the hourly totals every report but the click -log reads are kept, which is why a chart still answers for a window older than the clicks -behind it. A blanked address leaves an empty network in an export, the same as an address -that could never be looked up. +The worker enforces both periods, and **each one is a floor rather than a deadline** — +[How long ClickMonk keeps things](#how-long-clickmonk-keeps-things) below has the numbers +and what a missing row does. What is deleted is the raw clicks and the addresses on them; +the hourly rollups — which every report but the click log and its export reads — are kept, +which is why a chart still answers for a window whose raw clicks are gone. A blanked +address reads as no `network` at all, `null` in the log and an empty cell in the export, +the same as an address this build's parser does not recognise: the answer does not tell +those two apart, because the row does not either. For a script, mint an API key and send it as `Authorization: Bearer …`: @@ -345,11 +392,12 @@ For a script, mint an API key and send it as `Authorization: Bearer …`: docker compose exec -T worker node packages/cli/dist/index.js apikey create reporting ``` -A key can read and write domains, links and settings, and it can read the account's -identity at `GET /api/me`: the email address, whether two-factor authentication is on, and -how many recovery codes are unspent. It can change no credential at all. Every route that -touches one wants a session instead: listing or ending sessions, changing the password, -anything to do with two-factor authentication, and minting or revoking a key. So a key +A key can read and write domains, links and settings, read the reports, the click log and +the CSV export, and read the account's identity at `GET /api/me`: the email address, +whether two-factor authentication is on, and how many recovery codes are unspent. It can +change no credential at all. Every route that touches one wants a session instead: listing +or ending sessions, changing the password, anything to do with two-factor authentication, +and minting or revoking a key. So a key that leaks cannot lock you out of your own install, and cannot make itself a second key. `GET /api/me` also reports the failed sign-in count and any standing lockout, and those two it reports only to a session: they say whether someone is attacking the account right now, @@ -370,6 +418,116 @@ install query DNS in a loop. stack starts, Caddy routes every name to the redirect exactly as it did before, the admin service answers `503` on every route but `/health`, and the CLI is the only way in. +## Reading the clicks back out + +Every one of these is a read, so an API key reaches all of them and no `Origin` header is +needed. Windows are UTC, `from` is included and `to` is not, and a window may be at most +400 days. + +```sh +# Everything, for a day +curl -H "authorization: Bearer $KEY" \ + "https://admin.example.com/api/reports/summary?from=2026-09-23T00:00:00Z&to=2026-09-24T00:00:00Z" + +# A chart: bucket=hour or bucket=day, at most 2000 buckets in one answer +curl -H "authorization: Bearer $KEY" \ + "https://admin.example.com/api/reports/timeseries?from=2026-09-01T00:00:00Z&to=2026-09-24T00:00:00Z&bucket=day" + +# One dimension: country, device, os, browser, referrer, target, class, action, outcome +curl -H "authorization: Bearer $KEY" \ + "https://admin.example.com/api/reports/breakdown?from=2026-09-23T00:00:00Z&to=2026-09-24T00:00:00Z&dimension=country&limit=20" + +# The log itself, newest first, paged with the cursor the previous answer gave +curl -H "authorization: Bearer $KEY" \ + "https://admin.example.com/api/clicks?from=2026-09-23T00:00:00Z&to=2026-09-24T00:00:00Z&class=bot&limit=100" + +# And as a file +curl -H "authorization: Bearer $KEY" -OJ \ + "https://admin.example.com/api/clicks.csv?from=2026-09-01T00:00:00Z&to=2026-09-24T00:00:00Z" +``` + +Add `&link=` to any of them for one link instead of all of them. **Nothing looks the +link up**, so an id that belongs to no link is an empty answer rather than an error, on all +five of these. + +**A report counts whole hours; the log counts milliseconds.** Before a report is counted, +`from` is floored to the hour and `to` raised to the next one, because that is the grain the +rollups hold; the log and the export use the two instants exactly as they were sent. So the +same window can answer differently through the two surfaces and neither is broken — a window +starting one millisecond after the newest click was measured giving three clicks from the +summary and none from the log, because the summary counted the whole hour that click is in. +Every one of these five answers repeats the window it actually counted, in its own `window` +field. Read that before comparing two numbers. + +**What the numbers mean.** `clicks` counts distinct clicks and `visitors` distinct visitor +cookies, and both are counted as sets rather than added up: a click the worker shipped twice +is one click, and a visitor who clicked two links is one visitor for the install and one for +each link. The per-link visitor numbers therefore do not add up to the install-wide one, on +purpose. And `visitors` counts only what a cookie can see. A client that keeps no cookie is +a new visitor on every click, so for bot traffic `visitors` comes out close to `clicks`, and +that is the truth rather than a fault. The cookie is also `Secure` and set for one host +name: over a plain-HTTP trial every click is a new visitor, and one person using two of your +link domains is two visitors. + +**An empty value is a row, not a gap.** A breakdown by country, by referrer or by target has +a row whose `value` is `""`, and it means something different in each: an address that could +not be looked up, a visitor who arrived with no referrer, and a click that reached no target. +They are kept so that the rows add up to the number beside them. A breakdown returns 100 +rows unless you ask for more, at most 500, ordered by clicks, and sets `"truncated": true` +when there were more. + +**Reports lag the clicks** by the time a spool segment takes to seal plus the shipper's next +pass — a couple of seconds in normal operation. `newestHour`, on the summary and the chart, +is the newest hour the rollups hold anywhere, not the newest hour in the window you asked +for. + +**Two reports or log pages at a time, and one export.** Past that the answer is `429` with +`retry-after` rather than a queued query: a credential is not permission to scan the table +in a loop. The export's slot is its own, so a download does not lock a dashboard out. + +### The export + +`GET /api/clicks.csv` answers `content-disposition: attachment` with a filename built from +the window — `clicks-20260901T000000Z-20260924T000000Z.csv` — and a chunked body with no +`content-length`, because the rows are read from the store and written out as they arrive +rather than built in memory first. The filename drops milliseconds, so two windows a +millisecond apart produce the same name; what distinguishes them is the request, not the +file. + +**One export writes at most 1,000,000 rows**, which is just under what a spreadsheet will +open. Every export carries `x-clickmonk-row-cap` with that number and +`x-clickmonk-truncated`, `true` or `false`, and both are sent before the first byte of the +body — which is why a probe query runs in front of the export rather than a trailer being +appended, since the clients an operator actually uses do not show trailers. Narrow the +window and ask again; nothing is lost. The cap can be built as high as 10,000,000, and +there is no environment variable for it yet. + +**A cell that would otherwise look like a formula is prefixed with an apostrophe.** A +spreadsheet runs a cell beginning `=`, `+`, `-` or `@`, or with whitespace in front of one, +whether the cell is quoted or not. The two columns a visitor's own browser chooses the +contents of — `referrer` and `userAgent` — are where that turns up, so those are the ones +that can come back with an apostrophe in front of them. + +**The file identifies people, and the truncated network is the least of it.** Every row +carries `visitorId`, the 128-bit value from a cookie that lives a year and does not change +between sessions, so one visitor's rows reconstruct everything that person did through this +install's links on that domain. Every row also carries the whole `userAgent` and the whole +`referrer`, query string included, which is whatever the referring site chose to put there. +Decide who receives the file with that in mind: it is not an anonymous export with one +field redacted. `GET /api/clicks` hands over the same three fields, and the reports hand +over none of them. + +**A slow reader holds the export's slot**, because the slot is given back when the last row +has been written rather than when the request was accepted. A second export started while +one is still being read is refused. + +**A store failure after the first byte is a short file and nothing else.** The 200 and every +header have already gone, so there is no status left to change and nothing honest to append: +the download stops mid-file, inside a chunked body that never gets its last chunk, which is +a transfer error the client can see rather than a file that looks whole. The only record on +the server is one error line in the admin service's log, and **a download cannot be +retracted** — so check that a file you are about to pass on ends where you expect it to. + ## Password-protected links Set a password when you create a link, or later with `PATCH /api/links/:id`. It is at @@ -413,6 +571,40 @@ backup URL. That ordering is deliberate: the page a protected link shows says no about the link's state, and posting a guess at one must not say anything either. Use it to keep a link out of casual hands, not to protect something that matters if it gets out. +## How long ClickMonk keeps things + +```sh +docker compose exec -T worker node packages/cli/dist/index.js settings show +docker compose exec -T worker node packages/cli/dist/index.js settings set \ + --keep-clicks 180 --keep-addresses 7 +``` + +- **Raw clicks**: 90 days on a new install. `--keep-clicks never` keeps them for ever. +- **The address on a click**: 30 days, after which that one column is blanked and the click + keeps everything else. `--keep-addresses never` keeps them for ever. +- **The hourly rollups**: for ever. They are small, and they are what answers a question + about a window whose raw clicks are gone. + +Clicks are stored a month at a time and dropped a month at a time, and a month goes only +once every click it could hold is past the period — so 90 days means 90 to 121 days of +clicks, and 30 days means 30 to 61 days of addresses: the period, plus the longest month. +That is the trade for dropping a month in one statement instead of rewriting the table +continuously. The worker checks once an hour, or as often as +`CLICKMONK_RETENTION_INTERVAL_MS` says. Lowering a period applies on the next check; raising +one brings nothing back. + +**Nothing is deleted while this install cannot say what it asked for.** If the settings row +is missing, or the two periods on it cannot be read, the pass does nothing at all and says +why in the worker's log, `settings show` prints `keep clicks: unknown, so nothing is being +deleted`, and `GET /api/settings` answers `"retention": null` with a `problem`. That is +deliberate rather than a gap: applying the defaults there would delete clicks for an +operator who never chose a period, including one who had chosen to keep them for ever and +whose row was since deleted. `settings set` writes the row back, and says that it did. + +Setting the address period longer than the click period does nothing: the address goes when +the click does. `settings show` and `GET /api/settings` say so in a note rather than letting +you find out. + ## IP data The worker downloads four lists to your server, and `clickmonk ipdata update` fetches From 0e1da46ddad48fb376641a8bff1d1fa5598ad6db Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 21:59:55 +0000 Subject: [PATCH 57/65] cli: a period that reads as a sentence, and a lag we have measured --- README.md | 11 +++++++---- packages/cli/src/commands.test.ts | 17 +++++++++++++++++ packages/cli/src/commands.ts | 16 +++++++++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fba4c87..c9d61dd 100644 --- a/README.md +++ b/README.md @@ -476,10 +476,13 @@ They are kept so that the rows add up to the number beside them. A breakdown ret rows unless you ask for more, at most 500, ordered by clicks, and sets `"truncated": true` when there were more. -**Reports lag the clicks** by the time a spool segment takes to seal plus the shipper's next -pass — a couple of seconds in normal operation. `newestHour`, on the summary and the chart, -is the newest hour the rollups hold anywhere, not the newest hour in the window you asked -for. +**Reports lag the clicks**, and the mechanism is the whole of it: a click is countable once +the spool segment holding it has been sealed and the shipper's next pass has taken it, which +with the intervals as shipped is a small number of seconds. Nothing here reads the spool, so +until both of those have happened the click is in no report and in no export. +`newestHour`, on the summary and the chart, is the newest hour the rollups hold anywhere, +not the newest hour in the window you asked for, and it is what says whether a run of zeroes +at the end of a chart is "nobody clicked" or "not arrived yet". **Two reports or log pages at a time, and one export.** Past that the answer is `429` with `retry-after` rather than a queued query: a credential is not permission to scan the table diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index f3abcc4..d7f41aa 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -375,6 +375,23 @@ describe('clickmonk settings', () => { expect(lines).toContain('abuser threshold: 61 requests a minute from one client') }) + // The same note with the other value of both periods, which is the case the + // test above cannot reach: "for ever" already carries its own preposition, so + // a sentence that writes one as well says "kept for for ever". A number needs + // the preposition and the word does not, which is why the formatter decides + // it rather than the sentence. + it('says a period of never without stuttering, in the note about a row it wrote', async () => { + await pg.query('DELETE FROM settings') + lines.length = 0 + expect( + await run('settings', 'set', '--keep-clicks', 'never', '--keep-addresses', 'never'), + ).toBe(0) + expect(lines[0]).toBe( + 'note: there was no settings row, so this command has written one; nothing was being deleted before, and clicks are now kept for ever and addresses for ever', + ) + expect(await stored()).toEqual({ raw_retention_days: null, ip_retention_days: null }) + }) + it('prints the retention periods, in days and as for ever', async () => { await setRetention(90, null) lines.length = 0 diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 8fc2387..3be5e02 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -636,8 +636,19 @@ function printSettings(s: InstallSettings, d: CliDeps): void { if (note) d.out(`note: ${note}`) } +/** A period where a label wants one: `90 days`, or `for ever` when there is none. */ const days = (n: number | null): string => (n === null ? 'for ever' : `${n} days`) +/** + * The same period where a sentence wants one, which needs the preposition a + * label does not: `for 90 days`, and `for ever` rather than "for for ever". + * + * A wrapper over `days` and not a second formatter. The number is still spelled + * in one place, so the two cannot come to disagree about what `90` reads as; + * what is decided here is only whether a `for` goes in front of it. + */ +const keptFor = (n: number | null): string => (n === null ? days(n) : `for ${days(n)}`) + async function settingsShow(d: CliDeps): Promise { const read = await readSettings(d.pg) if (read.problem) d.out(`note: ${read.problem}`) @@ -715,7 +726,10 @@ async function settingsSet(args: string[], d: CliDeps): Promise { // The kept periods are named in full because there are two of them; the // traffic settings point at the lines printed below instead, because there // are five and they are about to be printed anyway. - const kept = `clicks are now kept for ${days(next.retention.rawRetentionDays)} and addresses for ${days(next.retention.ipRetentionDays)}` + // `keptFor`, not `days`: this is a sentence rather than a label, so the + // preposition comes from the formatter. With `days` and a `for` written here, + // `--keep-clicks never` printed "kept for for ever". + const kept = `clicks are now kept ${keptFor(next.retention.rawRetentionDays)} and addresses ${keptFor(next.retention.ipRetentionDays)}` for (const sub of substituted) { if (sub.what === 'row') { d.out( From bd0af312cf9fe79adecef0399f41e2696d2352e2 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:02:20 +0000 Subject: [PATCH 58/65] ci: forty minutes for six stack suites --- .github/workflows/ci.yml | 5 ++++- packages/cli/src/commands.test.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 306c819..9bbf4f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,10 @@ jobs: # `durability` both call `down -v`. stack: runs-on: ubuntu-latest - timeout-minutes: 30 + # Six suites now, each bringing the stack up for itself, one of them + # obtaining certificates from a local authority. Three and a half minutes + # with warm images locally; a cold runner builds the image first. + timeout-minutes: 40 steps: - uses: actions/checkout@v7 - uses: pnpm/action-setup@v6 diff --git a/packages/cli/src/commands.test.ts b/packages/cli/src/commands.test.ts index d7f41aa..2a20c02 100644 --- a/packages/cli/src/commands.test.ts +++ b/packages/cli/src/commands.test.ts @@ -813,7 +813,10 @@ describe('clickmonk domain list and verify', () => { expect(await withResolver(fakeResolver({}), 'domain', 'verify', 'stillok.example.test')).toBe(0) const out = lines.join('\n') expect(out).toContain('stillok.example.test: still verified') - expect(out).not.toContain('404') + // The whole transcript embeds a 32-character hex token, and about one run in + // a few hundred puts `404` inside it — this asserts the warning line is + // absent, so it matches the line rather than the digits. + expect(out.split('\n').filter((l) => l.startsWith('Until the TXT record is found'))).toEqual([]) expect(out).toContain('_clickmonk.stillok.example.test TXT') const after = await pg.query<{ verified: boolean }>( 'SELECT verified FROM domains WHERE host = $1', From 5f3765584f21085500b0231e09930b15adcef795 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:40:46 +0000 Subject: [PATCH 59/65] worker: a pass that keeps making progress --- packages/worker/src/retention.test.ts | 41 ++++++++- packages/worker/src/retention.ts | 128 ++++++++++++++++++-------- 2 files changed, 129 insertions(+), 40 deletions(-) diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index c678b85..073b27a 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -444,13 +444,52 @@ describe('runRetention', () => { } }) - it('considers at most as many partitions as it is given, oldest first', async () => { + it('acts on at most as many partitions as it is given, oldest first', async () => { const r = await runRetention({ pg: pool, ch, now: NOW, maxPartitions: 1 }) await settleMutations() + // The drop spends the whole budget, so the two months past the address + // period are left for the next pass rather than blanked in this one. expect(r.dropped).toEqual(['202605']) expect(r.blanked).toEqual([]) }) + // **The bound is on work, not on position.** With no raw period nothing is + // ever dropped, so the list of partitions never shrinks — and a pass that took + // the oldest few of them would consider the same few every hour and never + // reach the months behind them. Measured on the code before this: the first + // pass blanked its budget's worth and every pass after it blanked nothing, + // leaving every further month holding whole addresses for the life of the + // install while the pass logged a success line each hour. + // + // So this runs more passes than one and asserts the outcome the periods + // promise — no address left past the cutoff — rather than which partitions + // either pass named. Two passes and a budget of two against three months of + // work: the budget has to be spent twice for the third month to be reached. + it('keeps making progress across passes when nothing is ever dropped', async () => { + await pool.query('UPDATE settings SET raw_retention_days = NULL, ip_retention_days = 30') + const first = await runRetention({ pg: pool, ch, now: NOW, maxPartitions: 2 }) + await settleMutations() + // The premise: the first pass filled its budget and stopped, so there is + // still work left for the second. Without this the assertion below would + // pass against a build that did everything in one pass and against one whose + // budget was never reached. + expect(first.blanked).toEqual(['202605', '202606']) + const second = await runRetention({ pg: pool, ch, now: NOW, maxPartitions: 2 }) + await settleMutations() + expect(second.blanked).toEqual(['202607']) + // The address cutoff is 2026-08-25, so 202605, 202606 and 202607 are all + // past it and 202608 and 202609 are not. Nothing was dropped: five months + // are still here, and the two inside the period still hold what they came + // with. + expect(await addresses()).toEqual([ + { partition_id: '202605', ip: '' }, + { partition_id: '202606', ip: '' }, + { partition_id: '202607', ip: '' }, + { partition_id: '202608', ip: '198.51.100.8' }, + { partition_id: '202609', ip: '198.51.100.9' }, + ]) + }) + it('logs and skips a partition id that is not a month rather than putting it in a statement', async () => { const said: string[] = [] // A second table partitioned by something else cannot exist here, so the diff --git a/packages/worker/src/retention.ts b/packages/worker/src/retention.ts index 151700e..0cdeb10 100644 --- a/packages/worker/src/retention.ts +++ b/packages/worker/src/retention.ts @@ -24,6 +24,19 @@ * is still running on the table, or a month that takes twenty minutes to * rewrite would collect a new mutation every hour. * + * **A pass is bounded by the work it does, never by where that work sits.** The + * bound below is on partitions acted on, and what fills it is chosen by asking + * which partitions still need something done to them — not by taking the oldest + * few. Taking the oldest few is correct only while the oldest keep being + * dropped, which is what would make the window advance; with no raw period at + * all, or a raw period longer than the bound's worth of months, nothing is ever + * dropped, the same oldest months are considered every hour, and every month + * behind them keeps its addresses for ever while the pass logs a success. That + * is the ledger's failure reached from the other end, so the selection is the + * condition too: the months past the address cutoff that still hold an address, + * asked for in one statement rather than one per partition, because a probe per + * partition is itself work that grows with the months this install has kept. + * * **A partition id is the one value in this codebase that reaches SQL as * text.** DDL takes no bound parameters. So the id has to come from * ClickHouse's own `system.parts`, never from anything a request can reach, and @@ -54,7 +67,14 @@ import type { ClickHouseClient, Pool, PoolClient } from '@clickmonk/db' import { readSettingsLocked } from './settings.js' -/** Partitions one pass looks at, oldest first. Two years of months. */ +/** + * Partitions one pass acts on — dropped or blanked — oldest first. Two years of + * months, so an install that has fallen behind catches up over a few hours + * rather than issuing every mutation it owes at once. + * + * It is a bound on work and not on position: a partition this pass decides needs + * nothing done to it does not fill it. + */ export const RETENTION_PARTITIONS_PER_PASS = 24 /** @@ -143,10 +163,36 @@ async function partitionsOfClicks(ch: ClickHouseClient): Promise { return (await rs.json<{ partition_id: string }>()).map((r) => r.partition_id) } +/** + * The partitions among `ids` that still hold an address, oldest first. + * + * One statement for the whole set rather than one per partition, and it is the + * condition itself — not a record of what has been blanked. A `_partition_id` + * filter prunes to those partitions, and the only rows the aggregation sees are + * the ones that still carry an address, so a month already blanked costs the read + * of a column of empty strings and nothing else. + * + * No `FINAL`, unlike every other raw read in this service, and that is + * deliberate: this is an existence question, and a duplicate row still means + * there is an address in this partition. A dedup would cost a merge to answer a + * question that does not need one. + */ +async function partitionsHoldingAddress(ch: ClickHouseClient, ids: string[]): Promise> { + const rs = await ch.query({ + query: `SELECT DISTINCT _partition_id AS p FROM clicks + WHERE ip != '' AND _partition_id IN {ids:Array(String)}`, + query_params: { ids }, + format: 'JSONEachRow', + clickhouse_settings: BOUND, + }) + return new Set((await rs.json<{ p: string }>()).map((r) => r.p)) +} + export interface RetentionOptions { pg: Pool ch: ClickHouseClient now: Date + /** Partitions this pass may act on. Defaults to `RETENTION_PARTITIONS_PER_PASS`. */ maxPartitions?: number /** A seam for the list of partitions only, so a test can reach the skip branch. */ partitionSource?: () => Promise @@ -199,31 +245,48 @@ async function pass( const ipCutoff = ipRetentionDays === null ? null : nowMs - ipRetentionDays * DAY_MS const all = await (o.partitionSource ?? (() => partitionsOfClicks(o.ch)))() - const considered = all.slice(0, o.maxPartitions ?? RETENTION_PARTITIONS_PER_PASS) + const budget = o.maxPartitions ?? RETENTION_PARTITIONS_PER_PASS const result = empty() - const toBlank: string[] = [] - for (const id of considered) { + // Oldest first, and only the ids this table's own partition key could have + // produced. An id that is not a month is skipped here whatever the budget is: + // the budget bounds statements, and this decides never to write one. + const months: { id: string; end: number }[] = [] + for (const id of all) { const end = partitionEndMs(id) if (end === null) { result.skipped.push(id) log(`skipped partition ${id}: not a month this table could have produced`) continue } - // The drop is checked first: a partition past both periods goes whole - // rather than being rewritten and then dropped. - if (rawCutoff !== null && end <= rawCutoff) { - await o.ch.command({ - query: `ALTER TABLE clicks DROP PARTITION ID '${id}'`, - clickhouse_settings: BOUND, - }) - result.dropped.push(id) - continue - } - if (ipCutoff !== null && end <= ipCutoff) toBlank.push(id) + months.push({ id, end }) } - if (toBlank.length > 0) { + // The drop is checked first: a partition past both periods goes whole rather + // than being rewritten and then dropped. A drop needs no condition asked of + // the store — the partition is either past the period or it is not — and it + // takes the oldest of them because dropping is what makes the list shrink. + const toDrop = rawCutoff === null ? [] : months.filter((m) => m.end <= rawCutoff) + for (const m of toDrop.slice(0, budget)) { + await o.ch.command({ + query: `ALTER TABLE clicks DROP PARTITION ID '${m.id}'`, + clickhouse_settings: BOUND, + }) + result.dropped.push(m.id) + } + + const dropped = new Set(result.dropped) + // Past the address period, and not one this pass has just dropped whole. A + // partition the budget did not reach is left in, so the next pass sees it + // again: it is a candidate because of where it sits in time, never because of + // what an earlier pass did or did not get to. + const candidates = + ipCutoff === null + ? [] + : months.filter((m) => m.end <= ipCutoff && !dropped.has(m.id)).map((m) => m.id) + const left = budget - result.dropped.length + + if (candidates.length > 0 && left > 0) { const running = await scalar( o.ch, `SELECT count() AS n FROM system.mutations @@ -233,29 +296,16 @@ async function pass( result.mutationInFlight = true log(`${running} mutation(s) still running on clicks; no addresses blanked this pass`) } else { - for (const id of toBlank) { - // The condition itself, and it stops at the first row it finds rather - // than counting a month of them. - // - // No `FINAL` here, unlike every other raw read in this service, and - // that is deliberate: this is an existence test bounded at one row, and - // a duplicate row still means there is an address in this partition. A - // dedup would cost a merge to answer a question that does not need one. - // - // **And this is why the condition is not a ledger.** A spool segment - // shipped late lands in whichever month its clicks happened in, which - // can be a month this pass has already blanked. The condition sees the - // address that arrived and blanks it on the next pass. A record of - // partitions already done would say that month was finished and leave - // that address in place for ever, which is the one outcome this whole - // module exists to prevent. - const left = await scalar( - o.ch, - `SELECT count() AS n FROM ( - SELECT 1 FROM clicks WHERE _partition_id = {p:String} AND ip != '' LIMIT 1)`, - { p: id }, - ) - if (left === 0) continue + // **This is why the selection is not a ledger.** A spool segment shipped + // late lands in whichever month its clicks happened in, which can be a + // month this pass has already blanked. The condition sees the address that + // arrived and blanks it on the next pass. A record of partitions already + // done would say that month was finished and leave that address in place + // for ever, which is the one outcome this whole module exists to prevent — + // and so would a bound that only ever looked at the oldest months, which + // is the same failure with the ledger implied by the arithmetic. + const holding = await partitionsHoldingAddress(o.ch, candidates) + for (const id of candidates.filter((c) => holding.has(c)).slice(0, left)) { await o.ch.command({ query: `ALTER TABLE clicks UPDATE ip = '' IN PARTITION ID '${id}' WHERE ip != ''`, clickhouse_settings: BOUND, From a3a4b9dc5ecb608e61e3ee3ec451c196231de846 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:45:27 +0000 Subject: [PATCH 60/65] core: a click time the rollups can hold --- packages/admin/src/reports.ts | 15 ++++++-- packages/core/src/click-record.test.ts | 36 +++++++++++++++++++ packages/core/src/click-record.ts | 38 +++++++++++++++++++- packages/worker/src/shipper.test.ts | 50 ++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 4 deletions(-) diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 8b15a62..2e5da8f 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -67,9 +67,18 @@ export const WINDOW_FIELDS = { } /** - * The instants the store can hold: the range `DateTime64(3, 'UTC')` represents, - * which is the type every click time, every rollup hour and both ends of every - * window are compared as. + * The instants a window may name: the range `DateTime64(3, 'UTC')` represents, + * which is the type both ends of every window are bound into and the type + * `clicks.time` is stored as. + * + * **It is not the rollups' range, and the two must not be confused.** The + * rollups' `hour` is `DateTime('UTC')` — 1970-01-01 to 2106-02-07 — so the + * instants a rollup row can sit at are a subset of the instants a window can + * name. What keeps that from mattering is a bound at the other end: a click time + * outside the narrower range is refused where the spool is read + * (`MIN_CLICK_TIME_MS`/`MAX_CLICK_TIME_MS`), so no rollup row exists outside it + * and a window that reaches past it is an empty answer rather than a wrong one. A + * window reaching past this range is the different problem below. * * **A bound on a window's length is not a bound on where it sits.** Four hundred * days in the year 9999 is inside every length and bucket ceiling here, and what diff --git a/packages/core/src/click-record.test.ts b/packages/core/src/click-record.test.ts index f593de3..1a36534 100644 --- a/packages/core/src/click-record.test.ts +++ b/packages/core/src/click-record.test.ts @@ -183,6 +183,42 @@ describe('version 3: the password step', () => { }) }) +/** + * The click time against the range the rollups' `hour` column holds, on every + * version, because `common` is where the field is written once. + * + * The instants are written out rather than computed from the constants: a bound + * derived from the value it is testing holds whatever that value becomes. A time + * outside this range is not refused by the store — the materialized views wrap it + * into an hour decades away, which is missing from every rollup-backed report and + * permanently wrong in the freshness field the reports answer with. + */ +describe('the click time against what a rollup hour holds', () => { + it('takes the instants that column holds, and nothing outside them', () => { + for (const time of ['1970-01-01T00:00:00.000Z', '2106-02-07T06:28:15.000Z']) { + expect(SpoolRecordSchema.safeParse(sampleRecord({ time })).success, time).toBe(true) + } + for (const time of [ + '1969-12-31T23:59:59.999Z', + '1950-06-15T10:00:00.000Z', + '2106-02-07T06:28:16.000Z', + '2150-06-15T10:00:00.000Z', + ]) { + expect(SpoolRecordSchema.safeParse(sampleRecord({ time })).success, time).toBe(false) + } + }) + + // Refused on the time and not on something else the fixture happens to break, + // and on all three versions: the field is written once in `common`, so a bound + // added to one version only would leave the other two open. + it('refuses it on every record version, and refuses it for the time', () => { + const time = '1950-06-15T10:00:00.000Z' + expect(refusedFor(sampleRecord({ time }))).toEqual(['custom:time']) + expect(refusedFor(sampleV2({ time }))).toEqual(['custom:time']) + expect(refusedFor({ ...sampleV2(), v: 3, time })).toEqual(['custom:time']) + }) +}) + describe('spool segment names', () => { it('are what the reader matches, and sort by time as strings', () => { const name = segmentName(1_700_000_000_000, 42, 0) diff --git a/packages/core/src/click-record.ts b/packages/core/src/click-record.ts index 2e04f36..a1a95d3 100644 --- a/packages/core/src/click-record.ts +++ b/packages/core/src/click-record.ts @@ -9,10 +9,46 @@ export const MAX_PATH_LENGTH = 2048 /** The referrer is recorded up to this many characters. */ export const MAX_REFERRER_LENGTH = 2048 +/** + * The instants a click time may name: the range the **rollups' `hour` column** + * can hold, which is narrower than the one the click itself is stored in. + * + * Three columns and two ranges, and the gap between them is the whole reason + * this exists. `clicks.time` is `DateTime64(3, 'UTC')`, which reaches from 1900 + * to 2299. `clicks_hourly.hour` and `clicks_hourly_dim.hour` are + * `DateTime('UTC')`: four bytes of seconds, 1970-01-01 to 2106-02-07 06:28:15. + * The materialized views write `toStartOfHour(time)` into that narrower column, + * and ClickHouse does not refuse a value outside it there — it **wraps**. A click + * at 1950 lands in a rollup hour in 2086, a click at 2150 in one in 2014, and + * both are then missing from every rollup-backed report while present in the log + * and the export. Worse, `max(hour)` is the only freshness field the reports + * have, nothing ever deletes a rollup row, and one such click makes it answer an + * hour sixty years out for the life of the install. + * + * So the bound is on the way in. `toStartOfHour` only ever moves an instant + * backwards, so a time inside this range has an hour inside it too, which is + * what makes one range enough. + * + * The same rule as the ASN below, and it is reachable the same way: the time is + * the redirect's own clock, so a container with a wrong year or a spool line + * written by hand is all it takes. No request can reach it. + */ +export const MIN_CLICK_TIME_MS = Date.parse('1970-01-01T00:00:00.000Z') +export const MAX_CLICK_TIME_MS = Date.parse('2106-02-07T06:28:15.999Z') + /** The fields every record version carries. */ const common = { clickId: z.string().uuid(), - time: z.string().datetime({ offset: false }), + time: z + .string() + .datetime({ offset: false }) + // On the parsed instant and not on the text: `.datetime()` has already said + // the text is an ISO timestamp, and every year a rollup hour cannot hold is + // a perfectly well-formed one. + .refine((s) => { + const ms = Date.parse(s) + return ms >= MIN_CLICK_TIME_MS && ms <= MAX_CLICK_TIME_MS + }, 'must name an instant the rollups can hold'), host: z.string().max(253), path: z.string().max(MAX_PATH_LENGTH), domainId: z.string().uuid(), diff --git a/packages/worker/src/shipper.test.ts b/packages/worker/src/shipper.test.ts index 65762f7..1354fe2 100644 --- a/packages/worker/src/shipper.test.ts +++ b/packages/worker/src/shipper.test.ts @@ -35,6 +35,11 @@ beforeAll(async () => { }) beforeEach(async () => { await ch.command({ query: 'TRUNCATE TABLE clicks' }) + // The rollups too: a materialized view fires on every insert this file makes, + // nothing in the product ever deletes a rollup row, and a test that reads one + // must see this test's clicks rather than the whole file's. + await ch.command({ query: 'TRUNCATE TABLE clicks_hourly' }) + await ch.command({ query: 'TRUNCATE TABLE clicks_hourly_dim' }) }) afterAll(async () => { await pg.end() @@ -102,6 +107,20 @@ async function rawRowCount(): Promise { const [r] = await rs.json<{ n: string }>() return Number(r?.n) } +/** + * Every hour the hourly rollup holds, oldest first. + * + * The alias is `at` rather than `hour`: an alias that shadows the column is what + * the `ORDER BY` would then read, and a string where a `DateTime` belongs is an + * illegal-type error rather than an ordering. + */ +async function rollupHours(): Promise { + const rs = await ch.query({ + query: 'SELECT DISTINCT toString(hour) AS at FROM clicks_hourly ORDER BY at', + format: 'JSONEachRow', + }) + return (await rs.json<{ at: string }>()).map((r) => r.at) +} describe('toClickhouseRow', () => { it('formats time for DateTime64 and flattens nulls and booleans', () => { @@ -215,6 +234,37 @@ describe('shipOnce', () => { expect(await clicks()).toBe(1) }) + /** + * A click time outside what the rollups' `hour` column holds is a malformed + * line, and this is the end of the chain the bound exists for. + * + * The store refuses none of it. `clicks.time` holds 1900 to 2299, the rollups' + * `hour` holds 1970 to 2106, and the materialized views write + * `toStartOfHour(time)` into the narrower one — where a click at 1950 wraps + * into an hour in 2086. Nothing deletes a rollup row, so `max(hour)`, the only + * freshness field the reports have, would then answer that hour for the life of + * the install. + * + * So what is asserted is the rollup, not the count of malformed lines alone: a + * line refused by the schema is one that never reached a materialized view. + */ + it('counts a click time no rollup hour can hold as malformed, and lets none of it reach the rollup', async () => { + const dir = tmp() + const good = rec({ time: '2026-09-24T10:00:00.000Z' }) + segment(dir, [ + JSON.stringify(rec({ time: '1950-06-15T10:00:00.000Z' })), + JSON.stringify(rec({ time: '2150-06-15T10:00:00.000Z' })), + JSON.stringify(good), + ]) + const r = await shipOnce({ dir, ch, skipNewer: new Set() }) + expect(r).toEqual({ segments: 1, rows: 1, malformed: 2 }) + expect(await clicks()).toBe(1) + // The one hour the good click is in, and no other: the two wrapped hours + // would be 2086 and 2014, so a build without the bound answers one of them + // here and holds three rows rather than one. + expect(await rollupHours()).toEqual(['2026-09-24 10:00:00']) + }) + it('skips a torn final line (no trailing newline, cut mid-write) and still ships the rest', async () => { const dir = tmp() const name = segmentName(Date.now(), 1, seq++) From 9bdd4c97075de28edd6b8f7e3276c7e16e306ce1 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:52:12 +0000 Subject: [PATCH 61/65] admin: an export the caller cannot hold open --- README.md | 7 +- packages/admin/src/app.ts | 20 ++++- packages/admin/src/clicks.test.ts | 143 ++++++++++++++++++++++++++++++ packages/admin/src/clicks.ts | 65 ++++++++++++-- 4 files changed, 223 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c9d61dd..b6d33a9 100644 --- a/README.md +++ b/README.md @@ -522,7 +522,12 @@ over none of them. **A slow reader holds the export's slot**, because the slot is given back when the last row has been written rather than when the request was accepted. A second export started while -one is still being read is refused. +one is still being read is refused. **That hold ends after ten minutes**, whether the caller +has finished or not: a client that takes the headers and then stops reading would otherwise +hold the only slot for as long as it kept the socket open, and nothing else would end it. +The cost falls on an honest slow reader — a very large file over a slow link is cut off the +same way a store failure cuts one off, mid-body, which the client sees as a failed transfer. +Ask for a narrower window. **A store failure after the first byte is a short file and nothing else.** The 200 and every header have already gone, so there is no status left to change and nothing honest to append: diff --git a/packages/admin/src/app.ts b/packages/admin/src/app.ts index 0a39aa5..e730ad2 100644 --- a/packages/admin/src/app.ts +++ b/packages/admin/src/app.ts @@ -8,7 +8,12 @@ import Fastify, { type FastifyServerOptions, } from 'fastify' import { type Credential, authenticate, checkCsrf, hasBearer } from './auth.js' -import { EXPORT_ROW_CAP, checkExportRowCap, registerClickRoutes } from './clicks.js' +import { + EXPORT_BODY_DEADLINE_MS, + EXPORT_ROW_CAP, + checkExportRowCap, + registerClickRoutes, +} from './clicks.js' import { registerDomainRoutes } from './domains.js' import { HttpError, MAX_BODY_BYTES, securityHeaders } from './http.js' import { registerKeyRoutes } from './keys.js' @@ -37,8 +42,9 @@ export const DNS_CHECKS_IN_FLIGHT = 2 * for the same reason the on-demand DNS check refuses rather than queues. * * The export gate is separate from the report gate, and smaller, because an - * export holds a ClickHouse result open for as long as its client takes to read - * it, which is not a bound this process can set. Separate so that a download an + * export holds a ClickHouse result open while its client reads it — for as long + * as `EXPORT_BODY_DEADLINE_MS`, which is the bound this process sets on it, and + * far longer than a report's own query bound. Separate so that a download an * operator started does not lock their own dashboard out. */ export const REPORT_QUERIES_IN_FLIGHT = 2 @@ -96,6 +102,12 @@ export interface AdminDeps { * is refused at boot rather than trusted — see `checkExportRowCap`. */ exportRowCap?: number + /** + * How long one export may hold the export slot while writing its body. + * Injectable so a test can reach the deadline in milliseconds instead of ten + * minutes — `EXPORT_BODY_DEADLINE_MS` says what it is for and what it costs. + */ + exportDeadlineMs?: number /** * Where a link with no slug of its own gets one. `newSlug` unless a caller * says otherwise, and nothing in the product says otherwise: it is here so @@ -121,6 +133,7 @@ export interface AdminContext extends AdminDeps { reportGate: ConcurrencyGate exportGate: ConcurrencyGate exportRowCap: number + exportDeadlineMs: number slugSource: () => string } @@ -172,6 +185,7 @@ export function buildAdminApp( // how this service was built, so it belongs at boot with the name of the // thing that is wrong, not in the 503 every export would answer with. exportRowCap: checkExportRowCap(deps.exportRowCap ?? EXPORT_ROW_CAP), + exportDeadlineMs: deps.exportDeadlineMs ?? EXPORT_BODY_DEADLINE_MS, slugSource: deps.slugSource ?? newSlug, } diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 84948d1..2c13e3e 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -1668,3 +1668,146 @@ describe('GET /api/clicks.csv, a caller reading while the store produces', () => } }) }) + +/** + * A caller that takes the headers and then stops reading. + * + * This is the one hold in the service that nothing else bounds. A ClickHouse + * query blocked writing into a socket nobody drains is not *executing*, so + * `max_execution_time` never fires on it, the client's request timeout never + * fires either, and Fastify's `connectionTimeout` is 0. Measured before the + * deadline existed: the slot was still held after eighty seconds, and every + * later export answered 429 until the stalled socket was destroyed. One leaked + * key, one request. + * + * A real socket and a store with more rows than the buffers between here and the + * kernel can hold: with the three-row fixture the whole file fits in one write + * and finishes whether the caller reads it or not, which would make the case + * vacuous. + */ +describe('GET /api/clicks.csv, a caller that stops reading', () => { + const settle = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) + + /** + * A body of tens of megabytes, as a few rows carrying a long cell rather than + * very many short ones. + * + * The size is what the case needs: measured, the buffers between the generator + * and a paused reader — the stream's, Node's socket, and the kernel's windows on + * loopback — swallowed two megabytes whole, so a body that size finished on its + * own and proved nothing. The shape is what keeps it quick: a hundred thousand + * one-row blocks through an object-mode stream took longer than the bytes did. + */ + const ROWS = 4000 + const LONG_CELL = 'u'.repeat(8000) + /** The deadline this app is built with, short enough to wait out in a test. */ + const DEADLINE_MS = 500 + + /** A store that hands over `ROWS` rows, one block each, as fast as it is asked. */ + const bigCh = (): ClickHouseClient => { + async function* blocks(): AsyncGenerator<{ json: () => ClickRow }[]> { + for (let i = 0; i < ROWS; i++) { + const row = { + ...click(), + click_id: `01920000-0000-7000-8000-${String(i).padStart(12, '0')}`, + at_ms: '1758708000000', + user_agent: LONG_CELL, + } as unknown as ClickRow + yield [{ json: () => row }] + } + } + return { + query: async ({ query }: { query: string }) => + query.includes('count()') + ? { json: async () => [{ n: String(ROWS) }] } + : { close: () => {}, stream: () => Readable.from(blocks(), { objectMode: true }) }, + } as unknown as ClickHouseClient + } + + it('cuts the body off at the deadline, gives the slot back, and says so once', async () => { + const gate = new CountingGate(1) + const logged: string[] = [] + const served = testApp(pool, clock, { + ch: bigCh(), + exportGate: gate, + exportDeadlineMs: DEADLINE_MS, + log: { + level: 'warn', + stream: { + write(line: string) { + logged.push(line) + }, + }, + }, + }) + try { + await served.listen({ host: '127.0.0.1', port: 0 }) + const at = served.server.address() + const port = typeof at === 'object' && at !== null ? at.port : 0 + let status = 0 + const req = httpRequest( + { + host: '127.0.0.1', + port, + path: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST, cookie }, + }, + (res) => { + status = res.statusCode ?? 0 + // The whole arrangement: the headers are taken and the body is never + // read. Nothing is resumed and nothing is destroyed. + res.pause() + }, + ) + // The server destroying the stream ends this request, which is not a + // failure of anything here. + req.on('error', () => {}) + req.end() + // Waited for the headers rather than for the slot: the slot is taken before + // the status line goes out, so a loop on the gate alone would reach the + // assertion below before the caller had been answered at all. + for (let waited = 0; waited < 5000 && status === 0; waited += 5) await settle(5) + // The premise: the export took the slot and answered 200, so what follows + // is about a body in flight rather than about a refusal. + expect(status, 'the export was never answered').toBe(200) + expect(gate.entered).toBe(1) + + // A fixed wait several times the deadline, not a poll: a loop that waited + // for the gate to empty would pass whatever the release took, including + // never — which is precisely the behaviour being fixed. Eighty seconds was + // measured and was not the end of it. + await settle(DEADLINE_MS * 6) + expect(gate.stats().inFlight, 'the slot was still held past the deadline').toBe(0) + // And the line, which is the only record an operator has: the 200 and every + // header went out before the first row, so nothing about the answer can say + // the file is short. + expect(logged.filter((l) => l.includes('the caller stopped reading its body'))).toHaveLength( + 1, + ) + + // The consequence, end to end: with one export slot, a slot still held + // answers every later export 429 for the life of the process. Over a socket + // and drained, because this store's file is tens of megabytes and `inject` + // would hold all of it as one string. + const second = await new Promise((resolve, reject) => { + const other = httpRequest( + { + host: '127.0.0.1', + port, + path: `/api/clicks.csv?${WINDOW}`, + headers: { host: ADMIN_HOST, cookie }, + }, + (res) => { + res.on('data', () => {}) + res.on('end', () => resolve(res.statusCode ?? 0)) + }, + ) + other.on('error', reject) + other.end() + }) + expect(second).toBe(200) + } finally { + await served.close() + } + }) +}) diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 2f4f9e7..43ebf0c 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -282,6 +282,29 @@ export function clickFilter( export const EXPORT_ROW_CAP = 1_000_000 export const MAX_EXPORT_ROW_CAP = 10_000_000 +/** + * How long one export may hold the only export slot while writing its body. + * + * **What it is for.** Nothing else bounds the body. `max_execution_time` bounds a + * query that is executing; a query blocked writing into a socket nobody drains is + * not executing, so neither it nor the client's request timeout ends it, and + * Fastify's `connectionTimeout` is 0 by default. Measured before this existed: a + * credentialed client that took the headers and then stopped reading held the slot + * for eighty seconds and was still holding it — every later export answered 429 — + * until its socket was destroyed. One leaked key and one request, and the operator + * cannot download their own log, with nothing in the log to say why. + * + * **What it costs.** It is a deadline on the whole body, not an idle timer, so an + * honest slow reader on a poor connection loses a large export: the default cap is + * a million rows, which is a few hundred megabytes, and ten minutes of it is + * roughly what a five-megabit link delivers. That reader gets a file of whole lines + * that stops early inside a chunked body with no terminating chunk — the same thing + * a store failure mid-stream gives them, and visible as a failed transfer rather + * than a short file that looks complete. They can ask for a narrower window, or the + * install can raise this. An unbounded hold has no such remedy, which is the trade. + */ +export const EXPORT_BODY_DEADLINE_MS = 600_000 + /** * The cap, checked against the range it is bound into and not only its type. * @@ -422,14 +445,18 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo * report gate, so that a download an operator started does not lock their * dashboard out. * - * **What bounds the slot.** `max_execution_time` on both statements, and - * nothing else: Fastify's `requestTimeout` does not bound a handler — a - * three-second handler was measured answering after 3,029 ms under a - * one-second setting — and the handler has returned before most of this work - * happens anyway. The slot is held for the probe plus the export, so it is - * held for the sum of the two bounds; a client that reads slowly holds it - * until the store ends the query, which is why the ceiling on how many run at - * once is one. + * **What bounds the slot.** Two different things, because the slot is held + * across two phases that fail differently. The probe is bounded by + * `max_execution_time` and, outside it, by the client's own request timeout: + * that one is a query this process is waiting for. The body is bounded by + * `EXPORT_BODY_DEADLINE_MS`, a timer armed when the stream is handed over, + * because **nothing else bounds it at all** — a query blocked writing into a + * socket nobody drains is not executing, so no server-side execution bound + * applies to it, and Fastify's `requestTimeout` does not bound a handler + * either: a three-second handler was measured answering after 3,029 ms under a + * one-second setting, and this handler has returned before most of the work + * happens anyway. So the slot is held for the probe's bound plus this + * deadline, and never for as long as a caller cares to keep a socket open. * * **Why a probe query.** An export cut at the cap says so in a header, and a * header has to be decided before the first byte of the body. A trailer would @@ -464,6 +491,8 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo * type, which nothing else in this service names. */ let result: { close: () => void } | null = null + /** The body's deadline, once there is a body. Cleared by `release`. */ + let deadline: NodeJS.Timeout | null = null /** * Gives the slot back, once, from whichever path reaches it first. @@ -486,6 +515,10 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo const release = (): void => { if (released) return released = true + // Before the slot goes back, and on every path: a timer left armed after + // an export finished would fire against a stream that is already gone, and + // would keep this process awake for the rest of the deadline. + if (deadline !== null) clearTimeout(deadline) result?.close() ctx.exportGate.leave() } @@ -574,6 +607,22 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo // `error` is a stream that failed, which on a hangup can arrive first. file.on('close', release) file.on('error', release) + // The body's deadline, armed here because here is where the unbounded wait + // begins. Destroying the stream is what ends it: that is the same path a + // caller hanging up takes, so `close` fires and `release` gives the slot + // back through the one function every path already goes through. + // + // The line is the only record that an export was cut off — the 200 and every + // header went out before the first row, so there is no status left to change + // — and it names the caller's behaviour rather than a fault here, because + // that is what it is. + deadline = setTimeout(() => { + req.log.warn( + { deadlineMs: ctx.exportDeadlineMs }, + 'an export was cut off: the caller stopped reading its body', + ) + file.destroy() + }, ctx.exportDeadlineMs) started = true return reply.send(file) } finally { From af569e007434ee89e1fe32e06927182bd9388222 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:54:40 +0000 Subject: [PATCH 62/65] docs: what a day-bucket window counts --- CLAUDE.md | 9 +++++---- README.md | 34 +++++++++++++++++++++++----------- packages/admin/src/clicks.ts | 5 +++-- packages/admin/src/reports.ts | 14 ++++++++------ 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 191b856..4670994 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,10 +61,11 @@ Compose stack, and the restart durability and stack test suites. **A retention period is a floor, not a deadline, and it is never defaulted by a reader that deletes.** `clicks` is partitioned by month and dropped whole, so 90 days keeps 90 to 121 days; and a settings row that is missing or cannot be read as retention means the pass -deletes nothing at all, rather than the defaults. **A report counts whole hours** — it -aligns the window to the rollups' grain and echoes what it counted — while the click log -and the export use the window to the millisecond, so the two can answer the same request -differently and both are right. +deletes nothing at all, rather than the defaults. **A report counts whole buckets** — it +aligns the window to the grain it answers at, the hour for the summary and the breakdown +and the requested bucket for the chart, so `bucket=day` counts whole days — and echoes what +it counted, while the click log and the export use the window to the millisecond. So one +request can be answered differently by all three and every answer is right. **The two services read `CLICKMONK_ADMIN_HOST` differently on purpose.** The admin service refuses to boot on a value it cannot parse; the redirect logs `ADMIN_HOST_IGNORED` once and diff --git a/README.md b/README.md index b6d33a9..44c2680 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,9 @@ on your own infrastructure, and your click data stays yours. read hourly rollups written as the clicks arrive rather than the clicks themselves, so a window of a year is read as hours and not as every click inside it, and every number counts a click once even if the worker shipped its spool segment twice. **A report counts - whole hours** and says in its answer which hours it counted: ask for - `from=2026-09-24T10:30:00Z` and you are told you were given 10:00 onwards. + whole buckets** and says in its answer which window it counted: ask for + `from=2026-09-24T10:30:00Z` and the summary tells you you were given 10:00 onwards, while + a chart by day tells you it gave you the whole day. - **The click log, and a CSV of it.** `GET /api/clicks` lists the clicks themselves, newest first, filterable by link, traffic class, outcome, country and time, and exact to the millisecond. `GET /api/clicks.csv` is the same rows as a file: it streams, so the window @@ -377,6 +378,13 @@ periods cannot be read — and nothing is deleted while it reads that way. A mis the likelier of the two: it is one `DELETE` away, and the defaults would answer it with a period nobody chose. +**`clickmonk settings set` is what repairs that row, and the `PUT` is not.** The `PUT` +requires a `retention` object, so the body a `GET` hands back in this state — with +`"retention": null` — is one the `PUT` refuses: a script that reads, changes a field and +writes back cannot fix the state the read just told it about. Use the CLI, which writes the +periods you name over the defaults for the rest and prints what it wrote; after that the +`GET` and the `PUT` agree again. + The worker enforces both periods, and **each one is a floor rather than a deadline** — [How long ClickMonk keeps things](#how-long-clickmonk-keeps-things) below has the numbers and what a missing row does. What is deleted is the raw clicks and the addresses on them; @@ -450,14 +458,16 @@ Add `&link=` to any of them for one link instead of all of them. **Nothing l link up**, so an id that belongs to no link is an empty answer rather than an error, on all five of these. -**A report counts whole hours; the log counts milliseconds.** Before a report is counted, -`from` is floored to the hour and `to` raised to the next one, because that is the grain the -rollups hold; the log and the export use the two instants exactly as they were sent. So the -same window can answer differently through the two surfaces and neither is broken — a window -starting one millisecond after the newest click was measured giving three clicks from the -summary and none from the log, because the summary counted the whole hour that click is in. -Every one of these five answers repeats the window it actually counted, in its own `window` -field. Read that before comparing two numbers. +**A report counts whole buckets; the log counts milliseconds.** Before a report is counted, +`from` is floored and `to` raised to the next boundary, and **the boundary is the grain that +report answers at**: the hour for the summary and the breakdown, because that is the grain +the rollups hold, and the bucket you asked for on the chart — so `bucket=day` counts whole +days. The log and the export use the two instants exactly as they were sent. So the same +window can answer differently through these surfaces and none of them is broken: one +`from=2026-09-24T10:30:00Z&to=2026-09-24T10:50:00Z` gives two clicks from the summary, the +hourly chart and the breakdown, three from a chart by day, and one from the log. Every one +of these five answers repeats the window it actually counted, in its own `window` field. +Read that before comparing two numbers. **What the numbers mean.** `clicks` counts distinct clicks and `visitors` distinct visitor cookies, and both are counted as sets rather than added up: a click the worker shipped twice @@ -607,7 +617,9 @@ why in the worker's log, `settings show` prints `keep clicks: unknown, so nothin deleted`, and `GET /api/settings` answers `"retention": null` with a `problem`. That is deliberate rather than a gap: applying the defaults there would delete clicks for an operator who never chose a period, including one who had chosen to keep them for ever and -whose row was since deleted. `settings set` writes the row back, and says that it did. +whose row was since deleted. `settings set` writes the row back, and says that it did — and +it is the only thing that does: `PUT /api/settings` will not take a body with +`"retention": null` in it, so the repair is the CLI's. Setting the address period longer than the click period does nothing: the address goes when the click does. `settings show` and `GET /api/settings` say so in a note rather than letting diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 43ebf0c..16e8827 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -392,8 +392,9 @@ export function registerClickRoutes(app: FastifyInstance, ctx: AdminContext): vo * One page of the log, newest first. * * The window is used exactly as it was sent — to the millisecond, half-open — - * unlike a report, which can only answer whole hours. Both say what they - * counted, because the two answers over the same ragged window are allowed to + * unlike a report, which answers whole buckets: whole hours for the summary and + * the breakdown, and whole days for a chart asked for by day. All of them say + * what they counted, because the answers over one ragged window are allowed to * differ and an operator comparing them deserves to know why. */ app.get('/api/clicks', async (req) => { diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 2e5da8f..2d2fdfe 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -13,12 +13,14 @@ * the answer is a refusal with `retry-after` rather than a queued query. The * same argument as the on-demand DNS check, for the same reason. * - * **A report counts whole hours.** The rollups are hourly, so `from` is - * floored to the hour and `to` is raised to the next one, and the response - * says which window was counted rather than echoing the one that was asked - * for. The click log reads raw clicks and uses the window exactly as given; - * the two can therefore differ over a ragged window, which is why both of them - * say what they counted. + * **A report counts whole buckets**, and the bucket is the grain that report + * answers at: the hour for the summary and the breakdown, because the rollups + * are hourly, and the requested bucket size for the chart, because a chart by + * day has to align to days. So `from` is floored and `to` raised to the next + * boundary of that grain, and the response says which window was counted rather + * than echoing the one that was asked for. The click log reads raw clicks and + * uses the window exactly as given; all of them can therefore differ over a + * ragged window, which is why every one of them says what it counted. */ import { BUCKET_MS, From 599c8f4a3ba36fb7e1ad6f865771cb32b3346e30 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:55:02 +0000 Subject: [PATCH 63/65] docs: the day-bucket window without an unstated fixture --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 44c2680..942f3ea 100644 --- a/README.md +++ b/README.md @@ -463,11 +463,12 @@ five of these. report answers at**: the hour for the summary and the breakdown, because that is the grain the rollups hold, and the bucket you asked for on the chart — so `bucket=day` counts whole days. The log and the export use the two instants exactly as they were sent. So the same -window can answer differently through these surfaces and none of them is broken: one -`from=2026-09-24T10:30:00Z&to=2026-09-24T10:50:00Z` gives two clicks from the summary, the -hourly chart and the breakdown, three from a chart by day, and one from the log. Every one -of these five answers repeats the window it actually counted, in its own `window` field. -Read that before comparing two numbers. +window can answer differently through these surfaces and none of them is broken: ask +`from=2026-09-24T10:30:00Z&to=2026-09-24T10:50:00Z` and the summary, the hourly chart and the +breakdown all count 10:00 to 11:00, a chart by day counts the whole of the 24th, and the log +counts the twenty minutes you named — three different numbers from one request. Every one of +these five answers repeats the window it actually counted, in its own `window` field. Read +that before comparing two numbers. **What the numbers mean.** `clicks` counts distinct clicks and `visitors` distinct visitor cookies, and both are counted as sets rather than added up: a click the worker shipped twice From cefbd9c75361157e240c0361a44c57062e3f4138 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 22:58:40 +0000 Subject: [PATCH 64/65] worker: an interval a timer can hold --- .env.example | 5 +++- packages/core/src/index.ts | 1 + packages/core/src/interval.test.ts | 32 ++++++++++++++++++++++ packages/core/src/interval.ts | 39 +++++++++++++++++++++++++++ packages/worker/src/config.test.ts | 19 ++++++++++++- packages/worker/src/config.ts | 24 ++++++++++++++--- packages/worker/src/domains.test.ts | 14 ++++++++++ packages/worker/src/domains.ts | 6 ++++- packages/worker/src/retention.test.ts | 20 ++++++++++++++ packages/worker/src/retention.ts | 7 ++++- 10 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/interval.test.ts create mode 100644 packages/core/src/interval.ts diff --git a/.env.example b/.env.example index 5026f66..820a544 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,10 @@ CLICKMONK_DNS_CHECK=on CLICKMONK_DNS_SERVERS= # How often the worker looks for clicks and addresses past their retention # periods. An hour is plenty; shorten it only if you are watching it work, and -# not below one second, which the worker refuses. The periods themselves are +# not below one second, which the worker refuses. It refuses the other end too: +# at most 2147483647 ms, a little under 25 days, because that is the longest +# delay a timer holds — so a month typed as 2147483648 is refused at boot rather +# than becoming a pass that runs continuously. The periods themselves are # settings on the install, not here: # clickmonk settings set --keep-clicks 30 --keep-addresses 7 # Each period is a floor rather than a deadline. Clicks are deleted a whole diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ba82c28..1449057 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -12,6 +12,7 @@ export * from './config-error.js' export * from './device.js' export * from './domain-verification.js' export * from './evaluate.js' +export * from './interval.js' export * from './link.js' export * from './passthrough.js' export * from './proxies.js' diff --git a/packages/core/src/interval.test.ts b/packages/core/src/interval.test.ts new file mode 100644 index 0000000..953ac07 --- /dev/null +++ b/packages/core/src/interval.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import { MAX_TIMER_MS, checkIntervalMs } from './interval.js' + +/** + * The numbers here are written out rather than derived from `MAX_TIMER_MS`: a + * bound computed from the constant it is testing accepts whatever that constant + * becomes. What is asserted about the constant itself is the one fact that makes + * it the right number — it is the largest signed 32-bit integer, which is what + * `setTimeout` holds a delay in. + */ +describe('checkIntervalMs', () => { + it('is the largest delay a timer holds', () => { + expect(MAX_TIMER_MS).toBe(2 ** 31 - 1) + }) + + it('takes a whole number of milliseconds inside the range', () => { + expect(checkIntervalMs('intervalMs', 1)).toBe(1) + expect(checkIntervalMs('intervalMs', 3_600_000)).toBe(3_600_000) + expect(checkIntervalMs('intervalMs', 2147483647)).toBe(2147483647) + }) + + // 2,147,483,648 first, because it is the one that arrives by accident: it is a + // little under twenty-five days, which is what someone asking for a monthly + // pass types, and `setTimeout` answers it with a delay of 1 ms rather than an + // error. + it.each([2147483648, 2_592_000_000, 0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'refuses %p, naming the thing that is wrong', + (ms) => { + expect(() => checkIntervalMs('intervalMs', ms)).toThrow(/intervalMs/) + }, + ) +}) diff --git a/packages/core/src/interval.ts b/packages/core/src/interval.ts new file mode 100644 index 0000000..0611f3d --- /dev/null +++ b/packages/core/src/interval.ts @@ -0,0 +1,39 @@ +/** + * How long a loop may wait, against what a timer can actually hold. + * + * `setTimeout` keeps its delay in a signed 32-bit integer. Given more it does + * not throw and does not clamp to the maximum: it prints + * `TimeoutOverflowWarning` and **substitutes 1**. So an interval of + * 2,147,483,648 ms — a little over the ceiling, and a little under twenty-five + * days, which is what someone asking for a monthly pass types — turns a loop + * that was meant to run once a month into one that runs about a hundred and + * seventy times a second. Measured on the retention pass: 342 passes in two + * seconds, each one a Postgres transaction taking the settings row's lock and a + * `system.parts` query, with a process warning as the only signal. + * + * It is the same rule as every other bound in this codebase — a value is checked + * against the range of the thing it is handed to, not only against its shape — + * with the thing being a Node API rather than a store column, and the cost being + * sustained load rather than a bad request answered as an outage. + */ +export const MAX_TIMER_MS = 2_147_483_647 + +/** + * The interval, or a throw naming it. + * + * In the loop and not only in the configuration schema, because a caller reaching + * a loop directly is how this was measured in the first place, and because a bound + * that lives only in a schema is a bound the next caller does not have. The floor + * here is one millisecond rather than the second an operator's configuration is + * held to: this is the range a timer can hold, and the shortest interval an + * install may ask for is a separate decision made where that configuration is + * read. + */ +export function checkIntervalMs(name: string, ms: number): number { + if (!Number.isInteger(ms) || ms < 1 || ms > MAX_TIMER_MS) { + throw new Error( + `${name}: must be a whole number of milliseconds from 1 to ${MAX_TIMER_MS}, not ${ms}`, + ) + } + return ms +} diff --git a/packages/worker/src/config.test.ts b/packages/worker/src/config.test.ts index ff07990..8258fd7 100644 --- a/packages/worker/src/config.test.ts +++ b/packages/worker/src/config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { loadConfig } from './config.js' +import { type WorkerConfig, loadConfig } from './config.js' import { RETENTION_MAX_EXECUTION_SECONDS } from './retention.js' const base = { @@ -81,6 +81,23 @@ describe('loadConfig', () => { ).toBe(1000) }) + /** + * And the other end, which is not a preference: `setTimeout` holds a signed + * 32-bit delay and replaces anything larger with 1 ms, so an interval a little + * over twenty-five days is a pass running continuously. + * + * The two numbers are written out rather than computed from `MAX_TIMER_MS`: a + * bound derived from the constant it is testing accepts whatever that constant + * becomes. `2147483648` is the value an operator asking for a month types. + */ + it.each([ + ['CLICKMONK_RETENTION_INTERVAL_MS', (c: WorkerConfig) => c.retentionIntervalMs], + ['CLICKMONK_DNS_CHECK_INTERVAL_MS', (c: WorkerConfig) => c.dnsCheckIntervalMs], + ])('refuses %s past the longest delay a timer holds', (name, read) => { + expect(() => loadConfig({ ...base, [name]: '2147483648' })).toThrow(new RegExp(name)) + expect(read(loadConfig({ ...base, [name]: '2147483647' }))).toBe(2147483647) + }) + it('turns the checks off, and refuses any value but on or off', () => { expect(loadConfig({ ...base, CLICKMONK_DNS_CHECK: 'off' }).dnsCheck).toBe(false) expect(() => loadConfig({ ...base, CLICKMONK_DNS_CHECK: 'no' })).toThrow(/CLICKMONK_DNS_CHECK/) diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index 66bfe02..8349c2c 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -1,4 +1,4 @@ -import { DEFAULT_SPOOL_DIR, formatConfigError } from '@clickmonk/core' +import { DEFAULT_SPOOL_DIR, MAX_TIMER_MS, formatConfigError } from '@clickmonk/core' import type { ChConfig } from '@clickmonk/db' import { DEFAULT_IPDATA_DIR } from '@clickmonk/ipdata' import { z } from 'zod' @@ -51,7 +51,17 @@ const Schema = z.object({ // Off on an install whose domains are verified some other way, or in a // test stack with no DNS to ask. CLICKMONK_DNS_CHECK: z.enum(['on', 'off']).default('on'), - CLICKMONK_DNS_CHECK_INTERVAL_MS: z.coerce.number().int().min(1000).default(300_000), + // Bounded at both ends. The ceiling is `setTimeout`'s, not a preference: past + // it Node substitutes a delay of 1 ms, so an interval a little over twenty-five + // days becomes a check running a hundred and seventy times a second. The loop + // this reaches refuses the same range, because a bound that lives only here is + // one a direct caller does not have. + CLICKMONK_DNS_CHECK_INTERVAL_MS: z.coerce + .number() + .int() + .min(1000) + .max(MAX_TIMER_MS) + .default(300_000), // Empty: the host's own resolvers. Otherwise the addresses to ask, each // validated here rather than inside node:dns, which throws at the first // query with a message that names nothing. @@ -59,7 +69,15 @@ const Schema = z.object({ // How often the retention pass runs. There is no on/off here on purpose: // "never" is a value the setting takes, and two ways to turn one thing off // is how they come to disagree. - CLICKMONK_RETENTION_INTERVAL_MS: z.coerce.number().int().min(1000).default(3_600_000), + // The same ceiling as the DNS interval above, and for the same reason: this one + // is where it was measured. A monthly pass typed as 2,147,483,648 ms ran 342 + // times in two seconds, each pass taking the settings row's lock. + CLICKMONK_RETENTION_INTERVAL_MS: z.coerce + .number() + .int() + .min(1000) + .max(MAX_TIMER_MS) + .default(3_600_000), }) export interface WorkerConfig { diff --git a/packages/worker/src/domains.test.ts b/packages/worker/src/domains.test.ts index fea55a3..f763344 100644 --- a/packages/worker/src/domains.test.ts +++ b/packages/worker/src/domains.test.ts @@ -636,6 +636,20 @@ describe('startDomainChecker', () => { await pool.query('TRUNCATE domains CASCADE') }) + /** + * The same bound the retention loop takes, and this is the neighbour it was + * inherited from: `setTimeout` keeps its delay in a signed 32-bit integer and + * replaces anything larger with 1 ms, so an interval a little over twenty-five + * days is a resolver query every few milliseconds rather than one a month. + * + * Nothing is started, so there is nothing for `afterEach` to stop. + */ + it('refuses an interval no timer can hold', () => { + expect(() => + startDomainChecker({ pg: pool, resolver: fakeResolver({}), intervalMs: 2147483648 }), + ).toThrow(/intervalMs/) + }) + it('runs a pass at once and stops without waiting for the interval', async () => { const d = await addDomain('go.example.test') const resolver = fakeResolver({ txt: { [NAME]: [[verificationRecordValue(d.token)]] } }) diff --git a/packages/worker/src/domains.ts b/packages/worker/src/domains.ts index 9198fc0..2f08739 100644 --- a/packages/worker/src/domains.ts +++ b/packages/worker/src/domains.ts @@ -1,6 +1,7 @@ import { Resolver } from 'node:dns/promises' import { isIP } from 'node:net' import { + checkIntervalMs, isDomainUrl, newVerificationToken, normaliseHost, @@ -375,7 +376,10 @@ export function startDomainChecker(o: { limit?: number log?: (msg: string, err?: unknown) => void }): { stop(): Promise } { - const interval = o.intervalMs ?? 300_000 + // The same bound the retention loop takes, for the same reason: past + // `setTimeout`'s 32-bit ceiling the delay is replaced by 1 ms, so an interval a + // little over twenty-five days is a resolver query every few milliseconds. + const interval = checkIntervalMs('intervalMs', o.intervalMs ?? 300_000) // The loop must never reject, so a logger that throws is ignored. const log = (msg: string, err?: unknown) => { try { diff --git a/packages/worker/src/retention.test.ts b/packages/worker/src/retention.test.ts index 073b27a..e58b21c 100644 --- a/packages/worker/src/retention.test.ts +++ b/packages/worker/src/retention.test.ts @@ -513,6 +513,26 @@ describe('runRetention', () => { }) describe('startRetention', () => { + /** + * The interval against what a timer can hold, in the loop rather than only in + * the configuration schema. + * + * `setTimeout` keeps its delay in a signed 32-bit integer and replaces anything + * larger with 1 ms, so 2,147,483,648 — a little under twenty-five days, which is + * what someone asking for a monthly pass types — was measured running 342 passes + * in two seconds, each one taking the settings row's lock. It throws rather than + * clamping: a loop that quietly ran at a different period than it was asked for + * is the same surprise in a smaller font. + * + * Nothing is started, which is the point — so there is no loop to stop and no + * pass to settle. + */ + it('refuses an interval no timer can hold', () => { + expect(() => startRetention({ pg: pool, ch, intervalMs: 2147483648, now: () => NOW })).toThrow( + /intervalMs/, + ) + }) + it('runs a pass and stops when it is told to', async () => { const passes: string[] = [] const loop = startRetention({ diff --git a/packages/worker/src/retention.ts b/packages/worker/src/retention.ts index 0cdeb10..8597a36 100644 --- a/packages/worker/src/retention.ts +++ b/packages/worker/src/retention.ts @@ -64,6 +64,7 @@ * writers carry a lock timeout of their own — the operator is told to try * again rather than left holding a command that has printed nothing. */ +import { checkIntervalMs } from '@clickmonk/core' import type { ClickHouseClient, Pool, PoolClient } from '@clickmonk/db' import { readSettingsLocked } from './settings.js' @@ -334,7 +335,11 @@ export function startRetention(o: { now?: () => Date log?: (msg: string, err?: unknown) => void }): { stop(): Promise } { - const interval = o.intervalMs ?? 3_600_000 + // Against what a timer can hold, here and not only in the configuration + // schema: an interval past `setTimeout`'s 32-bit ceiling is not clamped, it is + // replaced by 1 ms, and a monthly pass then runs a hundred and seventy times a + // second — each one taking the settings row's lock. + const interval = checkIntervalMs('intervalMs', o.intervalMs ?? 3_600_000) const now = o.now ?? (() => new Date()) const log = (msg: string, err?: unknown) => { try { From 9681552e76b1313d601c0e2931a16f267f0d4160 Mon Sep 17 00:00:00 2001 From: Cem Hurturk Date: Thu, 24 Sep 2026 23:05:18 +0000 Subject: [PATCH 65/65] admin: a cursor this service will take back --- packages/admin/src/clicks.test.ts | 86 ++++++++++++++++++++++++++++++- packages/admin/src/clicks.ts | 36 ++++++++----- packages/admin/src/reports.ts | 11 +++- test/stack/reports.test.ts | 52 +++++++++++++++++++ 4 files changed, 171 insertions(+), 14 deletions(-) diff --git a/packages/admin/src/clicks.test.ts b/packages/admin/src/clicks.test.ts index 2c13e3e..88ad27d 100644 --- a/packages/admin/src/clicks.test.ts +++ b/packages/admin/src/clicks.test.ts @@ -521,6 +521,11 @@ describe('GET /api/clicks', () => { ['a cursor one millisecond past the last instant', `cursor=10413792000000.${CURSOR_ID}`], ['a cursor in year ten thousand', `cursor=253402300800000.${CURSOR_ID}`], ['a cursor of nothing but nines', `cursor=999999999999999.${CURSOR_ID}`], + // And the other end, which the pattern's minus sign makes reachable: one + // millisecond before 1900, where the store clamps rather than refusing in the + // same way it does above 2299. + ['a cursor one millisecond before the first instant', `cursor=-2208988800001.${CURSOR_ID}`], + ['a cursor of nothing but nines with a minus sign', `cursor=-999999999999999.${CURSOR_ID}`], ['a class nobody has', 'class=spider'], ['an outcome nobody has', 'outcome=maybe'], ['a lower-case country', 'country=de'], @@ -565,6 +570,25 @@ describe('GET /api/clicks', () => { expect(r.json().clicks).toHaveLength(3) }) + /** + * The other end of the same range, and a literal for the same reason. + * + * `-2208988800000` is 1900-01-01, the oldest instant a window may name, and it + * reaches the parser only because the pattern's millisecond field carries a + * sign. Every click in the fixture is newer than it, so the page is empty — the + * 200 is what says the instant was accepted rather than refused, and the empty + * page is what says it was used as the boundary rather than ignored. + */ + it('takes the first instant a click can have', async () => { + const r = await app.inject({ + method: 'GET', + url: `/api/clicks?${WINDOW}&cursor=-2208988800000.${CURSOR_ID}`, + headers: read(cookie), + }) + expect(r.statusCode).toBe(200) + expect(r.json().clicks).toEqual([]) + }) + // The ceiling from both sides. The 200 is what says it is not one row tighter // than the number it promises — tightening it to 150 passed every other test // in this file — and the 201 that it is there at all. Both literals, for the @@ -664,7 +688,11 @@ describe('GET /api/clicks.csv', () => { const HEADER = '"clickId","at","host","path","domainId","linkId","outcome","step","status","destination","targetId","visitorId","returning","country","region","city","geoSource","device","os","browser","asn","class","signals","action","referrer","userAgent","network","capUnchecked"' - it('exports the rows the log would have listed, newest first, as a file', async () => { + // Named for what it asserts: three literal ids in order, with two whole rows. + // It was called "the rows the log would have listed", and it reads no log — + // the two surfaces are held to each other by both pinning the same rows + // literally, not by this test comparing them. + it('writes every click in the window, newest first, with a header row', async () => { const r = await app.inject({ method: 'GET', url: `/api/clicks.csv?${WINDOW}`, @@ -1811,3 +1839,59 @@ describe('GET /api/clicks.csv, a caller that stops reading', () => { } }) }) + +/** + * A cursor this service hands out, handed straight back to it. + * + * Both sides of one seam, and each was reviewed alone: the parser's pattern + * reasoned about the instants a *caller* can name, and the emitter is a caller of + * it too. A window may reach back to 1900 and `toUnixTimestamp64Milli` is signed, + * so a page of clicks before 1970 hands out a cursor with a minus sign in it — + * which an unsigned pattern refused, making the second page of that listing + * unreachable through the cursor the first page gave. + * + * The rows are written straight into the store, which is where such a row comes + * from: the record schema refuses a click time outside what a rollup hour holds, + * so nothing this build ships can produce one — but an install that ran an earlier + * build, or a hand-written row, can already have one, and the log still has to + * page it. + * + * Its own window, decades from every other one in this file, so these two rows are + * invisible to every count above. + */ +describe('GET /api/clicks, a cursor from a page before 1970', () => { + const OLD_WINDOW = 'from=1950-01-01T00:00:00.000Z&to=1951-01-01T00:00:00.000Z' + const OLDER = '01920000-0000-7000-8000-0000000000e1' + const NEWER = '01920000-0000-7000-8000-0000000000e2' + + beforeAll(async () => { + await insert([ + click({ click_id: OLDER, time: '1950-06-15 10:00:00.000' }), + click({ click_id: NEWER, time: '1950-06-15 10:01:00.000' }), + ]) + }) + + it('takes it back and gives the page after it', async () => { + const first = await app.inject({ + method: 'GET', + url: `/api/clicks?${OLD_WINDOW}&limit=1`, + headers: read(cookie), + }) + expect(first.statusCode).toBe(200) + expect(first.json().clicks.map((c: { clickId: string }) => c.clickId)).toEqual([NEWER]) + const cursor = first.json().nextCursor as string + // The sign is the whole point, so it is asserted rather than assumed: a + // fixture that had drifted after 1970 would make the round trip below pass + // without ever exercising it. + expect(cursor).toMatch(/^-\d+\.[0-9a-f-]{36}$/) + + const second = await app.inject({ + method: 'GET', + url: `/api/clicks?${OLD_WINDOW}&limit=1&cursor=${encodeURIComponent(cursor)}`, + headers: read(cookie), + }) + expect(second.statusCode).toBe(200) + expect(second.json().clicks.map((c: { clickId: string }) => c.clickId)).toEqual([OLDER]) + expect(second.json().nextCursor).toBeNull() + }) +}) diff --git a/packages/admin/src/clicks.ts b/packages/admin/src/clicks.ts index 16e8827..ea72dbf 100644 --- a/packages/admin/src/clicks.ts +++ b/packages/admin/src/clicks.ts @@ -47,12 +47,15 @@ import { fail } from './http.js' import { CH_MAX_EXECUTION_SECONDS, MAX_STORE_MS, + MIN_STORE_MS, type ReportWindow, WINDOW_FIELDS, chRows, parseWindow, readQuery, requireCh, + windowClause, + windowParams, withSlot, } from './reports.js' @@ -95,7 +98,7 @@ const ListQuery = z * A page boundary: the instant and the id of the last row of the page before. * Opaque to the caller, and parsed strictly — it reaches a comparison against a * `DateTime64` and a `UUID`, so the pattern refuses anything that is not those - * two shapes and `MAX_STORE_MS` refuses an instant outside what the first of + * two shapes and the range below refuses an instant outside what the first of * them holds. * * That range is the window's, imported rather than written out again here: one @@ -103,8 +106,15 @@ const ListQuery = z * it is how the two come to disagree. What the store does either side of it — a * clamp below year 10000 and a refused parameter above — is why the check * exists, and the constant's own comment says it. + * + * **The millisecond field is signed**, because `toUnixTimestamp64Milli` is: a + * window may reach back to 1900, so a page of clicks before 1970 hands out a + * cursor with a minus sign in it. With an unsigned pattern this service emitted a + * `nextCursor` its own parser refused, which is a listing that cannot be paged + * rather than a request that was wrong. What a caller can reach is not the whole + * of what has to be parsed here: this service is a caller of it too. */ -const CURSOR_RE = /^(\d{1,15})\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/ +const CURSOR_RE = /^(-?\d{1,15})\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/ export function parseCursor(value: string): { atMs: number; clickId: string } { const m = CURSOR_RE.exec(value) @@ -114,10 +124,11 @@ export function parseCursor(value: string): { atMs: number; clickId: string } { // pattern has two groups. if (!m) return fail(400, 'invalid_query', 'cursor: not a cursor from a previous page') const atMs = Number(m[1] as string) - // The pattern's digits are unsigned, so the oldest instant a cursor can name - // is the epoch, which is inside the type's range: only the newest end is - // reachable and only it is checked. - if (atMs > MAX_STORE_MS) { + // Both ends, and neither is unreachable: the newest by a caller inventing + // digits, the oldest by a minus sign in front of them. A window's bounds are + // pinned at both ends or at neither, and the cursor is the same rule on the + // same instants. + if (atMs < MIN_STORE_MS || atMs > MAX_STORE_MS) { return fail(400, 'invalid_query', 'cursor: names an instant no click can have') } return { atMs, clickId: m[2] as string } @@ -243,12 +254,13 @@ export function clickFilter( w: ReportWindow, cursor: { atMs: number; clickId: string } | null, ): { where: string; params: Record } { - const parts = [`time >= {from:DateTime64(3,'UTC')}`, `time < {to:DateTime64(3,'UTC')}`] - const params: Record = { from: w.from, to: w.to } - if (w.linkId !== null) { - parts.push('link_id = {link:UUID}') - params.link = w.linkId - } + // The window and the link through `windowClause`, on the `time` column, which + // is what its second parameter is for. Written out here once, it was the same + // half-open rule in a second spelling — and a rule in two spellings is a rule + // that comes to disagree with itself. The parameter had no caller until this + // one, which is how the duplicate survived three rounds of pinning the bound. + const parts = [windowClause(w, 'time')] + const params: Record = windowParams(w) if (q.class !== undefined) { parts.push('traffic_class = {class:String}') params.class = q.class diff --git a/packages/admin/src/reports.ts b/packages/admin/src/reports.ts index 2d2fdfe..92e2c7b 100644 --- a/packages/admin/src/reports.ts +++ b/packages/admin/src/reports.ts @@ -268,7 +268,16 @@ export function newestHourOrNull(text: string | undefined): string | null { return `${text.replace(' ', 'T')}.000Z` } -/** The window clause every report shares, with the link filter only when there is one. */ +/** + * The window clause every report shares, with the link filter only when there is + * one. + * + * `column` is what makes it shared rather than nearly shared: the rollups filter + * `hour` and the click log filters `time`, and those are the same half-open rule + * on two columns. It had no caller once, and the click log wrote the rule out + * again — which is how one rule comes to have two spellings and then two + * meanings. + */ export function windowClause(w: ReportWindow, column = 'hour'): string { const parts = [`${column} >= {from:DateTime64(3,'UTC')}`, `${column} < {to:DateTime64(3,'UTC')}`] if (w.linkId !== null) parts.push('link_id = {link:UUID}') diff --git a/test/stack/reports.test.ts b/test/stack/reports.test.ts index c97af07..2f1010c 100644 --- a/test/stack/reports.test.ts +++ b/test/stack/reports.test.ts @@ -77,6 +77,12 @@ let clickHours: string[] = [] * that row is counted and told apart from every click this suite really made. */ const OLD_OUTCOME = 'expired' +/** + * And the outcome on the row the blanking test writes for itself. Its own value, + * so the two retention tests cannot read each other's row: the first drops its + * row and the second must find exactly one. + */ +const BLANK_OUTCOME = 'capped' function api( method: string, @@ -464,4 +470,50 @@ describe('the install', () => { expect(storeCount(`SELECT uniqExact(click_id) ${kept}`)).toBe(CLICKS) expect(storeCount(`SELECT uniqExact(click_id) ${kept} AND ip != ''`)).toBe(CLICKS) }) + + /** + * The pass's other half, end to end: the address blanked in place, the click + * kept. + * + * It is the half that had unit cover and no cover here, in the one suite whose + * reason for existing is that the others cannot see a real chain — a real + * worker, reading a real settings row, issuing a real mutation against a real + * partition. The unit tests mock nothing either, but they call the pass; this + * one only restarts the worker and watches the store. + * + * **The periods are chosen so that the arithmetic does not depend on today's + * date.** A month goes only once every click it could hold is past the period, + * so which month is past a 30-day period depends on how far into the month the + * run happens to be. Four hundred days ago against `--keep-addresses 7` is past + * the address period by about a year whatever the date, and against + * `--keep-clicks 3650` is nowhere near the click period — so the row must be + * blanked and must not be dropped, and the difference between those two is what + * this test is for. + */ + it('blanks the address on a month past the address period, and keeps the click', async () => { + cli('settings', 'set', '--keep-clicks', '3650', '--keep-addresses', '7') + const oldMs = Date.now() - 400 * 24 * HOUR_MS + const at = new Date(oldMs).toISOString().replace('T', ' ').replace('Z', '') + storeRun( + `INSERT INTO clicks (click_id, time, outcome, ip) + VALUES (generateUUIDv4(), toDateTime64('${at}', 3, 'UTC'), '${BLANK_OUTCOME}', '198.51.100.8')`, + ) + const mine = `FROM clicks WHERE outcome = '${BLANK_OUTCOME}'` + // The premise, and it is a premise rather than a race: the pass that would + // touch this row is an hour away, and the restart below is what brings the + // next one forward. + expect(storeCount(`SELECT count() ${mine} AND ip != ''`)).toBe(1) + + compose('restart', 'worker') + await until( + 'the pass to blank the month past the address period', + 60_000, + () => storeCount(`SELECT count() ${mine} AND ip = ''`) === 1, + ) + // And the click itself is still there: a dropped partition would satisfy an + // assertion about no address being left, which is the wrong outcome reached by + // the same number. Blanking and dropping are the two halves of this pass and + // the whole point of the periods is that they are different. + expect(storeCount(`SELECT count() ${mine}`)).toBe(1) + }) })