From d9ea14766b025c955d007d50c22e879116a59d2f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 21:16:01 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20docs:=20name=20the=20scalar=20re?= =?UTF-8?q?tention=20bound=20in=20its=20proof?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Kosiewski --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ Change-Id: Icc62d1216d2e11eb8d2c7a77f065ba875a6f8805 --- .../services/historyScalarEvidence.test.ts | 244 ++++++++++++++++++ src/node/services/historyScalarEvidence.ts | 243 +++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 src/node/services/historyScalarEvidence.test.ts create mode 100644 src/node/services/historyScalarEvidence.ts diff --git a/src/node/services/historyScalarEvidence.test.ts b/src/node/services/historyScalarEvidence.test.ts new file mode 100644 index 0000000000..3ab7aa8dc2 --- /dev/null +++ b/src/node/services/historyScalarEvidence.test.ts @@ -0,0 +1,244 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, open, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { scanHistoryRows } from "./historyRowScanner"; +import { + createHistoryCanonicalEvidence, + createHistoryNumberEvidence, + createHistoryStringEvidence, +} from "./historyScalarEvidence"; + +const directories: string[] = []; +afterEach(async () => { + for (const directory of directories.splice(0)) + await rm(directory, { recursive: true, force: true }); +}); + +function numberEvidence(raw: string, chunkSize: number) { + const evidence = createHistoryNumberEvidence(raw.length); + for (let i = 0; i < raw.length; i += chunkSize) evidence.push(raw.slice(i, i + chunkSize)); + return evidence.finish(); +} + +async function canonicalRows(raw: string | Buffer, normalizationChanged = false) { + const directory = await mkdtemp(join(tmpdir(), "history-scalar-test-")); + directories.push(directory); + const file = join(directory, "rows.jsonl"); + await writeFile(file, raw); + const result: boolean[] = []; + await scanHistoryRows(file, () => { + const evidence = createHistoryCanonicalEvidence(Buffer.byteLength(raw)); + return { + token: evidence.token.bind(evidence), + finish: (row) => { + result.push(evidence.finish(row, normalizationChanged)); + }, + }; + }); + return result; +} + +function decimalDyadic(numerator: bigint, denominatorPower: number) { + const digits = (numerator * 5n ** BigInt(denominatorPower)) + .toString() + .padStart(denominatorPower + 1, "0"); + return `${digits.slice(0, -denominatorPower)}.${digits.slice(-denominatorPower)}`; +} + +function changeLastDecimalDigit(value: string, delta: bigint) { + const [whole, fraction] = value.split("."); + const digits = (BigInt(whole + fraction) + delta) + .toString() + .padStart(whole.length + fraction.length, "0"); + return `${digits.slice(0, -fraction.length)}.${digits.slice(-fraction.length)}`; +} + +describe("bounded history scalar evidence", () => { + test.each([1, 2, 7, 63, 4096])( + "native number conversion and canonicality agree across %s-character chunks", + (chunkSize) => { + const values = [ + "0", + "-0", + "0.0", + "-0e99999", + "1", + "-1", + "1e0", + "1E+0", + "1e309", + "-1e309", + "1e-99999", + "-1e-99999", + "1.7976931348623157e308", + "5e-324", + "2.2250738585072014e-308", + "9007199254740993", + "9007199254740995", + "1.00000000000000011102230246251565404236316680908203125", + "0." + "0".repeat(5000) + "12345e5004", + "12345" + "0".repeat(5000) + "e-5004", + "1e+" + "0".repeat(5000) + "308", + "1e-" + "9".repeat(5000), + "1e" + "9".repeat(5000), + "1." + "0".repeat(7000) + "1", + ]; + for (const raw of values) { + const actual = numberEvidence(raw, chunkSize); + const expected: unknown = JSON.parse(raw); + expect(Object.is(actual.value, expected)).toBe(true); + expect(actual.canonical).toBe(JSON.stringify(expected) === raw); + expect(actual.retainedSignificantDigits).toBeLessThanOrEqual(2048); + } + } + ); + + test("native rounding preserves midpoint ties and sticky tails beyond retained digits", () => { + const midpoints = [ + decimalDyadic((1n << 53n) + 1n, 53), + decimalDyadic((1n << 53n) + 3n, 53), + decimalDyadic(1n, 1075), + decimalDyadic(3n, 1075), + decimalDyadic((1n << 53n) - 1n, 1075), + ]; + for (const midpoint of midpoints) { + const tie = midpoint + "0".repeat(3000); + for (const raw of [tie, changeLastDecimalDigit(tie, -1n), changeLastDecimalDigit(tie, 1n)]) { + for (const sign of ["", "-"]) { + const literal = sign + raw; + expect(Object.is(numberEvidence(literal, 13).value, JSON.parse(literal))).toBe(true); + } + } + } + const overflow = ((1n << 1024n) - (1n << 970n)).toString() + "." + "0".repeat(3000); + for (const raw of [ + overflow, + changeLastDecimalDigit(overflow, -1n), + changeLastDecimalDigit(overflow, 1n), + ]) + expect(Object.is(numberEvidence(raw, 19).value, JSON.parse(raw))).toBe(true); + }); + + test("many generated finite numbers and noncanonical spellings use native conversion", () => { + let seed = 123456789; + for (let i = 0; i < 2000; i++) { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + const value = (seed - 0x80000000) * 10 ** ((i % 600) - 300); + const raw = JSON.stringify(value); + const expected: unknown = JSON.parse(raw); + expect(Object.is(numberEvidence(raw, 3).value, expected)).toBe(true); + expect(numberEvidence(raw, 3).canonical).toBe(true); + } + }); + + test("giant numeric and string inputs retain only bounded prefixes", () => { + const number = createHistoryNumberEvidence(16 * 1024 * 1024); + number.push("1."); + const zeros = "0".repeat(8192); + for (let i = 0; i < 1024; i++) number.push(zeros); + number.push("1"); + expect(number.finish()).toMatchObject({ + value: 1, + canonical: false, + retainedSignificantDigits: 2048, + }); + const string = createHistoryStringEvidence(128, "different"); + for (let i = 0; i < 1024; i++) string.push(zeros); + const result = string.finish(); + expect(result.prefix).toHaveLength(128); + expect(result.length).toBe(8 * 1024 * 1024); + expect(result.matchesExpected).toBe(false); + }); + + test("decoded string equality and hashing preserve split surrogate pairs and lone surrogates", () => { + const value = "a😀\ud800b\udfff"; + const probe = createHistoryStringEvidence(2, value); + for (const unit of value.split("")) probe.push(unit); + const actual = probe.finish(); + expect(actual.matchesExpected).toBe(true); + expect(actual.prefix).toBe(value.slice(0, 2)); + expect(actual.sha256).toBe( + createHash("sha256").update(Buffer.from(value, "utf16le")).digest("hex") + ); + const other = createHistoryStringEvidence(0, value); + other.push(value.replace("\ud800", "\ud801")); + expect(other.finish().matchesExpected).toBe(false); + }); + + test("source limits are enforced rather than silently saturating an undersized promise", () => { + expect(() => createHistoryNumberEvidence(-1)).toThrow(); + expect(() => createHistoryNumberEvidence(1).push("10")).toThrow(); + expect(() => createHistoryStringEvidence(-1)).toThrow(); + }); +}); + +describe("streamed canonical JSON evidence", () => { + test.each([ + "null", + "true", + "false", + "0", + "-0", + "1e0", + "1e400", + '""', + '"hello"', + "[]", + "{}", + '[1,{"text":"hello"},null]', + '{"2":0,"10":0,"name":0}', + '{"10":0,"2":0}', + '{"name":0,"2":0}', + '{"01":0,"1":0}', + '{"1":0,"01":0}', + '{"4294967294":0,"4294967295":0}', + '{"4294967295":0,"4294967294":0}', + '{"a":0,"a":1}', + '{"a":0,"\\u0061":1}', + '{"nested":{"b":1,"b":2}}', + '{ "a": 1 }', + '"\\u0061"', + '"a/b"', + '"a\\/b"', + '"\\ud800"', + '"\\ud800\\udc00"', + JSON.stringify("😀\ud800\n\t\u2028\u2029"), + '{"__proto__":{"a":1},"constructor":null}', + ])("matches native JSON.stringify round trip for %s", async (raw) => { + const expected = JSON.stringify(JSON.parse(raw)) === raw; + expect(await canonicalRows(raw)).toEqual([expected]); + expect(await canonicalRows(raw + "\n")).toEqual([expected]); + }); + + test("invalid UTF-8, malformed JSON and a declared legacy normalization never prove canonical bytes", async () => { + expect(await canonicalRows(Buffer.from([0x22, 0xff, 0x22]))).toEqual([false]); + expect(await canonicalRows('{"unfinished":\n{"valid":true}\n')).toEqual([false, true]); + expect(await canonicalRows('{"metadata":{"cmuxMetadata":{}}}', true)).toEqual([false]); + }); + + test("giant canonical scalars cross raw read and Unicode boundaries without row assembly", async () => { + const directory = await mkdtemp(join(tmpdir(), "history-canonical-giant-")); + directories.push(directory); + const file = join(directory, "row.jsonl"); + const chunk = "x".repeat(65529) + "😀\n"; + await using handle = await open(file, "w"); + await handle.write('{"text":"'); + for (let i = 0; i < 129; i++) await handle.write(JSON.stringify(chunk).slice(1, -1)); + await handle.write('","tail":"\\ud800"}\n'); + const size = (await handle.stat()).size; + const results: boolean[] = []; + await scanHistoryRows(file, () => { + const evidence = createHistoryCanonicalEvidence(size); + return { + token: evidence.token.bind(evidence), + finish: (row) => { + results.push(evidence.finish(row)); + }, + }; + }); + expect(size).toBeGreaterThan(8 * 1024 * 1024); + expect(results).toEqual([true]); + }); +}); diff --git a/src/node/services/historyScalarEvidence.ts b/src/node/services/historyScalarEvidence.ts new file mode 100644 index 0000000000..7f9096fb08 --- /dev/null +++ b/src/node/services/historyScalarEvidence.ts @@ -0,0 +1,243 @@ +import { createHash } from "node:crypto"; +import type { HistoryRowDescriptor, HistoryRowToken } from "./historyRowScanner"; + +const SIGNIFICANT_DIGITS = 2048; +const CANONICAL_NUMBER_CHARS = 32; + +/** Bounded decoded-string facts; the prefix is never a substitute for validating the full value. */ +export function createHistoryStringEvidence(prefixChars: number, expected?: string) { + if (!Number.isSafeInteger(prefixChars) || prefixChars < 0) + throw new Error("Invalid prefix bound"); + const hash = createHash("sha256"); + let prefix = ""; + let length = 0; + let matches = expected !== undefined; + return { + push(chunk: string) { + prefix += chunk.slice(0, Math.max(0, prefixChars - prefix.length)); + matches &&= expected!.startsWith(chunk, length); + length += chunk.length; + // Preserve JS string equality, including distinct lone surrogates and chunk splits. + hash.update(Buffer.from(chunk, "utf16le")); + }, + finish() { + return { + prefix, + length, + sha256: hash.digest("hex"), + matchesExpected: matches && length === expected!.length, + }; + }, + }; +} + +/** + * Consume one parser-validated JSON number. sourceByteLimit bounds the complete source row/file. + * Keep a prefix and sticky discarded tail, then let native Number perform all binary rounding. + * Every binary64 rounding boundary is dyadic: at most 309 integer and 1075 fractional decimal + * positions suffice (including zero/subnormal and finite/Infinity boundaries). The + * SIGNIFICANT_DIGITS retention bound exceeds every boundary's finite decimal expansion. + * Replacing a discarded nonzero tail with any positive tail preserves the side of a boundary; + * a zero tail preserves exact ties. No binary rounding is implemented here. + */ +export function createHistoryNumberEvidence(sourceByteLimit: number) { + if (!Number.isSafeInteger(sourceByteLimit) || sourceByteLimit < 1) + throw new Error("Invalid numeric source bound"); + const exponentLimit = BigInt(sourceByteLimit) + 4096n; + let exponent = 0n; + let exponentNegative = false; + let inExponent = false; + let negative = false; + let fraction = false; + let fractionDigits = 0; + let significantDigits = 0; + let digits = ""; + let discardedNonzero = false; + let raw = ""; + let length = 0; + return { + push(chunk: string) { + length += chunk.length; + if (length > sourceByteLimit) throw new Error("Number exceeds its captured source bound"); + if (raw.length <= CANONICAL_NUMBER_CHARS) + raw = (raw + chunk).slice(0, CANONICAL_NUMBER_CHARS + 1); + for (const character of chunk) { + if (character === "e" || character === "E") inExponent = true; + else if (character === "-") { + if (inExponent) exponentNegative = true; + else negative = true; + } else if (character === ".") fraction = true; + else if (character !== "+") { + const digit = character.charCodeAt(0) - 48; + if (inExponent) { + // An exponent beyond source length + 4096 cannot be canceled by its mantissa. + exponent = exponent * 10n + BigInt(digit); + if (exponent > exponentLimit) exponent = exponentLimit; + } else { + if (fraction) fractionDigits++; + if (digit !== 0 || significantDigits > 0) { + significantDigits++; + if (digits.length < SIGNIFICANT_DIGITS) digits += character; + else discardedNonzero ||= digit !== 0; + } + } + } + } + }, + finish() { + const scale = + (exponentNegative ? -exponent : exponent) - + BigInt(fractionDigits) + + BigInt(significantDigits - digits.length - (discardedNonzero ? 1 : 0)); + const value = Number( + `${negative ? "-" : ""}${digits || "0"}${discardedNonzero ? "1" : ""}e${scale}` + ); + return { + value, + canonical: length <= CANONICAL_NUMBER_CHARS && JSON.stringify(value) === raw, + retainedSignificantDigits: digits.length, + }; + }, + }; +} + +interface Frame { + array: boolean; + count: number; + lastIndex: number; + sawStringKey: boolean; +} + +/** + * Prove raw JSON.stringify equivalence without materializing a row or scalar. This proves bytes, + * not message readability or authority. The history caller must report normalizationChanged + * when its legacy normalizer changes a row; this module deliberately knows no message fields. + */ +export function createHistoryCanonicalEvidence(sourceByteLimit: number) { + const hash = createHash("sha256"); + const frames: Frame[] = []; + let byteLength = 0; + let canonical = true; + let rootValues = 0; + let surrogate = ""; + let key: ReturnType | undefined; + let number: ReturnType | undefined; + const write = (text: string) => { + const bytes = Buffer.from(text); + hash.update(bytes); + byteLength += bytes.length; + }; + const beforeValue = () => { + const frame = frames.at(-1); + if (!frame) rootValues++; + else if (frame.array && frame.count++ > 0) write(","); + }; + const stringChunk = (chunk: string) => { + let text = surrogate + chunk; + const last = text.charCodeAt(text.length - 1); + surrogate = last >= 0xd800 && last <= 0xdbff ? text.slice(-1) : ""; + if (surrogate) text = text.slice(0, -1); + write(JSON.stringify(text).slice(1, -1)); + }; + const endString = () => { + write(JSON.stringify(surrogate).slice(1, -1)); + surrogate = ""; + write('"'); + }; + return { + token(token: HistoryRowToken) { + switch (token.name) { + case "startObject": + case "startArray": + beforeValue(); + write(token.name === "startArray" ? "[" : "{"); + frames.push({ + array: token.name === "startArray", + count: 0, + lastIndex: -1, + sawStringKey: false, + }); + break; + case "endObject": + case "endArray": + write(token.name === "endArray" ? "]" : "}"); + frames.pop(); + break; + case "startKey": + if (frames.at(-1)!.count++ > 0) write(","); + key = createHistoryStringEvidence(10); + write('"'); + break; + case "endKey": { + const value = key!.finish(); + const index = Number(value.prefix); + const isIndex = + value.length <= 10 && + Number.isInteger(index) && + index >= 0 && + index < 0xffffffff && + String(index) === value.prefix; + const frame = frames.at(-1)!; + if (isIndex) { + canonical &&= !frame.sawStringKey && index > frame.lastIndex; + frame.lastIndex = index; + } else frame.sawStringKey = true; + key = undefined; + endString(); + write(":"); + break; + } + case "startString": + beforeValue(); + write('"'); + break; + case "stringChunk": + key?.push(token.value); + stringChunk(token.value); + break; + case "endString": + endString(); + break; + case "startNumber": + beforeValue(); + number = createHistoryNumberEvidence(sourceByteLimit); + break; + case "numberChunk": + number!.push(token.value); + break; + case "endNumber": { + const value = number!.finish(); + canonical &&= value.canonical; + write(JSON.stringify(value.value)); + number = undefined; + break; + } + case "trueValue": + case "falseValue": + case "nullValue": + beforeValue(); + write(JSON.stringify(token.value)); + break; + case "whitespace": + break; + default: + throw new Error("Canonical evidence requires unpacked parser tokens"); + } + }, + finish(row: HistoryRowDescriptor, normalizationChanged = false) { + const sha256 = hash.digest("hex"); + return ( + canonical && + !normalizationChanged && + rootValues === 1 && + frames.length === 0 && + row.validJson && + row.validUtf8 && + !row.hasDuplicateKeys && + row.byteLength <= sourceByteLimit && + row.byteLength === byteLength && + row.sha256 === sha256 + ); + }, + }; +}