From a2957117cff6cf89c746018c263fa92810d01365 Mon Sep 17 00:00:00 2001 From: Patrick Schmitt <45056826+patschmittdev@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:06:13 -0400 Subject: [PATCH] Fix Bronze duplicate detection for CRLF records Reuse the canonical corpus parser for Bronze reads, hash collection, and verification while preserving lone-CR hashes and corruption refusal. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../content/docs/guides/ingest-and-refine.md | 5 +- src/bronze/store.ts | 56 ++----- test/bronze-line-endings.test.ts | 145 ++++++++++++++++++ 3 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 test/bronze-line-endings.test.ts diff --git a/site/src/content/docs/guides/ingest-and-refine.md b/site/src/content/docs/guides/ingest-and-refine.md index 4262228..00dcfd0 100644 --- a/site/src/content/docs/guides/ingest-and-refine.md +++ b/site/src/content/docs/guides/ingest-and-refine.md @@ -16,7 +16,8 @@ ziggurat ingest --root --file inbox/some-note.md For a new capture, `ingest` creates and verifies a Bronze record before deleting the inbox source. If the canonical body already exists, it returns `duplicate` -and leaves the inbox source untouched. +and leaves the inbox source untouched, even when the existing Bronze record uses +CRLF line endings on disk or the duplicate has a different inbox filename. Because a new capture reads and deletes its source, an escaping path would be a combined arbitrary-read and arbitrary-delete primitive. It is validated hard: the source must @@ -37,6 +38,8 @@ into `inbox/` as a fresh copy rather than linking it. ::: The result is canonical UTF-8 text after CRLF-to-LF normalization in a Bronze record. +Reading, hash verification, and duplicate detection use that same normalization; +lone carriage returns are preserved and remain significant to the body hash. Ingest creates it atomically without overwriting; SHA-256 verification detects later body mutation. Fresh captures default to `sensitivity: restricted` and `pii: unknown`. diff --git a/src/bronze/store.ts b/src/bronze/store.ts index b72e18d..678f0e3 100644 --- a/src/bronze/store.ts +++ b/src/bronze/store.ts @@ -13,6 +13,7 @@ import { dirname, join, relative } from 'node:path'; import * as YAML from 'yaml'; import { BronzeRecordSchema } from '../contracts/index.js'; import type { BronzeRecord } from '../contracts/index.js'; +import { parseCorpusDocument } from '../corpus/documents.js'; import { sha256Text } from './canonical.js'; export interface VerifyResult { @@ -32,36 +33,19 @@ export class BronzeCorruptionError extends Error { } } -interface BronzeSplit { - yamlText: string; - body: string; -} - -/** - * Splits a Bronze file into its YAML frontmatter text and body. - * Format: ---\n\n---\n - */ -function splitBronzeFile(content: string): BronzeSplit | null { - if (!content.startsWith('---\n')) return null; - const afterOpen = content.slice(4); - const closeIdx = afterOpen.indexOf('\n---\n'); - if (closeIdx === -1) return null; - return { - yamlText: afterOpen.slice(0, closeIdx), - body: afterOpen.slice(closeIdx + 5), - }; -} - /** Parses a Bronze file's frontmatter + validates through BronzeRecordSchema. */ export function parseBronzeRecord(content: string): BronzeRecord { return parseBronzeFile(content).record; } export function parseBronzeFile(content: string): { record: BronzeRecord; body: string } { - const split = splitBronzeFile(content); - if (split === null) throw new Error('not a valid Bronze file: missing frontmatter'); - const parsed = YAML.parse(split.yamlText) as unknown; - return { record: BronzeRecordSchema.parse(parsed), body: split.body }; + const parsed = parseCorpusDocument(content, BronzeRecordSchema, Object.keys(BronzeRecordSchema.shape)); + if (!parsed.valid) { + const detail = parsed.failure.reason === 'missing-frontmatter' + ? 'missing frontmatter' : parsed.failure.detail; + throw new Error(`not a valid Bronze file: ${detail}`); + } + return { record: parsed.data, body: parsed.body }; } /** Serializes a BronzeRecord + canonical body into the on-disk file format. */ @@ -95,18 +79,13 @@ export async function collectBronzeHashes(root: string): Promise { const content = await readFile(filePath, 'utf8'); - const split = splitBronzeFile(content); - if (split === null) throw new Error(`${filePath}: not a valid Bronze file (no frontmatter)`); - const parsed = YAML.parse(split.yamlText) as unknown; - const record = BronzeRecordSchema.parse(parsed); - const actual = sha256Text(split.body); + const { record, body } = parseBronzeFile(content); + const actual = sha256Text(body); return { valid: actual === record.sha256, expected: record.sha256, actual }; } diff --git a/test/bronze-line-endings.test.ts b/test/bronze-line-endings.test.ts new file mode 100644 index 0000000..92ceee8 --- /dev/null +++ b/test/bronze-line-endings.test.ts @@ -0,0 +1,145 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { canonicalBronzeBody, sha256Text } from '../src/bronze/canonical.js'; +import { ingestCapture } from '../src/bronze/ingest.js'; +import { + BronzeCorruptionError, + collectBronzeHashes, + parseBronzeFile, + parseBronzeRecord, + verifyBronzeFile, +} from '../src/bronze/store.js'; +import { collectBronzeFilesDetailed } from '../src/corpus/collect.js'; +import { createVerifiedBronzeReader } from '../src/refine/evidence.js'; + +const OPTIONS = { now: new Date('2026-01-02T00:00:00Z'), sourceKind: 'article' }; +const BODIES = [ + { name: 'ordinary body', body: '# Record\n\nEvidence stays unchanged.\n' }, + { name: 'lone CR body', body: '# Record\n\nEvidence\rstays unchanged.\n' }, +]; +const ENDINGS = [ + { name: 'LF', value: '\n' }, + { name: 'CRLF', value: '\r\n' }, +]; + +async function withCapture( + body: string, + ending: string, + fn: (root: string, sourcePath: string, stored: string) => Promise, +): Promise { + const root = await mkdtemp(join(tmpdir(), 'ziggurat-bronze-endings-')); + try { + await mkdir(join(root, 'inbox')); + await writeFile(join(root, 'inbox', 'original.md'), body, 'utf8'); + const first = await ingestCapture(root, 'inbox/original.md', OPTIONS); + assert.equal(first.status, 'created'); + assert.equal(first.sha256, sha256Text(body)); + const filePath = join(root, first.source_path); + const stored = (await readFile(filePath, 'utf8')).replace(/\n/g, ending); + await writeFile(filePath, stored, 'utf8'); + await fn(root, first.source_path, stored); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +async function bronzeFiles(root: string): Promise { + const entries = await readdir(join(root, 'bronze'), { recursive: true, withFileTypes: true }); + return entries.filter(entry => entry.isFile()).map(entry => entry.name).sort(); +} + +for (const ending of ENDINGS) { + for (const fixture of BODIES) { + test(`Bronze ${ending.name}: parsing, live reading and dedup agree for ${fixture.name}`, async () => { + const { body } = fixture; + const expected = sha256Text(body); + await withCapture(body, ending.value, async (root, sourcePath, stored) => { + const filePath = join(root, sourcePath); + const live = await createVerifiedBronzeReader(root).read(root, sourcePath); + assert.equal(live.body, body); + assert.equal(live.record.sha256, expected); + assert.deepEqual(parseBronzeFile(stored), { record: live.record, body }); + assert.deepEqual(parseBronzeRecord(stored), live.record); + assert.deepEqual(await verifyBronzeFile(filePath), { valid: true, expected, actual: expected }); + assert.deepEqual(await collectBronzeHashes(root), new Map([[expected, sourcePath]])); + + const { records, rejected } = await collectBronzeFilesDetailed(root); + assert.deepEqual(rejected, []); + assert.equal(records.length, 1); + assert.equal(records[0]?.body, body); + assert.equal(records[0]?.sha256, expected); + assert.equal(records[0]?.hashVerified, true); + assert.equal(canonicalBronzeBody(body.replace(/\n/g, '\r\n')), body); + if (body.includes('\r')) { + assert.notEqual(expected, sha256Text(body.replace(/\r/g, '\n'))); + } + + const before = await bronzeFiles(root); + assert.equal(before.length, 1); + // A distinct filename prevents the no-overwrite guard from masking failed deduplication. + const inboxPath = join(root, 'inbox', 'different-name.md'); + await writeFile(inboxPath, body, 'utf8'); + const result = await ingestCapture(root, 'inbox/different-name.md', OPTIONS); + assert.deepEqual(result, { status: 'duplicate', source_path: sourcePath, sha256: expected }); + assert.deepEqual(await bronzeFiles(root), before); + assert.equal(await readFile(inboxPath, 'utf8'), body); + assert.equal(await readFile(filePath, 'utf8'), stored); + }); + }); + + test(`Bronze ${ending.name}: corrupted ${fixture.name} still blocks ingestion`, async () => { + const { body } = fixture; + const expected = sha256Text(body); + const actual = sha256Text(body.replace('unchanged.', 'unchanged!')); + await withCapture(body, ending.value, async (root, sourcePath, stored) => { + const filePath = join(root, sourcePath); + const mutated = stored.replace('unchanged.', 'unchanged!'); + assert.notEqual(mutated, stored); + await writeFile(filePath, mutated, 'utf8'); + assert.deepEqual(await verifyBronzeFile(filePath), { valid: false, expected, actual }); + const isCorruption = (error: unknown): boolean => + error instanceof BronzeCorruptionError + && error.filePath === filePath && error.expected === expected && error.actual === actual; + await assert.rejects(collectBronzeHashes(root), isCorruption); + await assert.rejects(createVerifiedBronzeReader(root).read(root, sourcePath), isCorruption); + const { records } = await collectBronzeFilesDetailed(root); + assert.equal(records.length, 1); + assert.equal(records[0]?.sha256, expected); + assert.equal(records[0]?.hashVerified, false); + + const before = await bronzeFiles(root); + const inboxPath = join(root, 'inbox', 'different-name.md'); + await writeFile(inboxPath, body, 'utf8'); + await assert.rejects(ingestCapture(root, 'inbox/different-name.md', OPTIONS), isCorruption); + assert.deepEqual(await bronzeFiles(root), before); + assert.equal(await readFile(inboxPath, 'utf8'), body); + assert.equal(await readFile(filePath, 'utf8'), mutated); + }); + }); + } + + test(`Bronze ${ending.name}: malformed frontmatter still rejects direct reads and is not deduplicated`, async () => { + await withCapture(BODIES[0]!.body, ending.value, async (root, sourcePath, stored) => { + const filePath = join(root, sourcePath); + const malformed = [ + stored.slice(3), + stored.replace('source_id: original', 'source_id: ['), + stored.replace('source_id: original', `source_id: original${ending.value}source_id: duplicate`), + stored.replace('schema_version: 1', 'schema_version: 2'), + stored.replace('schema_version: 1', `schema_version: 1${ending.value}unexpected: true`), + ]; + for (const content of malformed) { + assert.notEqual(content, stored); + await writeFile(filePath, content, 'utf8'); + assert.throws(() => parseBronzeFile(content)); + assert.throws(() => parseBronzeRecord(content)); + await assert.rejects(verifyBronzeFile(filePath)); + await assert.rejects(createVerifiedBronzeReader(root).read(root, sourcePath)); + assert.deepEqual(await collectBronzeHashes(root), new Map()); + } + }); + }); +}