Skip to content

Fix Bronze duplicate detection for CRLF records #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
patschmittdev merged 2 commits into from
Sep 12, 2026
Merged

Fix Bronze duplicate detection for CRLF records #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a295711
Fix Bronze duplicate detection for CRLF records
patschmittdev Sep 12, 2026
6c6369e
Merge branch 'main' into patschmittdev-bronze-crlf-ingestion
patschmittdev Sep 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion site/src/content/docs/guides/ingest-and-refine.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ ziggurat ingest --root <vault> --file inbox/some-note.md

For a new capture, `ingest` creates and verifies a Bronze record before deleting
the inbox source. If the canonical body already exists, it returns `duplicate`
and leaves the inbox source untouched.
and leaves the inbox source untouched, even when the existing Bronze record uses
CRLF line endings on disk or the duplicate has a different inbox filename.

Because a new capture reads and deletes its source, an escaping path would be a combined
arbitrary-read and arbitrary-delete primitive. It is validated hard: the source must
Expand All @@ -37,6 +38,8 @@ into `inbox/` as a fresh copy rather than linking it.
:::

The result is canonical UTF-8 text after CRLF-to-LF normalization in a Bronze record.
Reading, hash verification, and duplicate detection use that same normalization;
lone carriage returns are preserved and remain significant to the body hash.
Ingest creates it atomically without overwriting; SHA-256 verification detects later body
mutation. Fresh captures default to `sensitivity: restricted` and `pii: unknown`.

Expand Down
56 changes: 16 additions & 40 deletions src/bronze/store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { dirname, join, relative } from 'node:path';
import * as YAML from 'yaml';
import { BronzeRecordSchema } from '../contracts/index.js';
import type { BronzeRecord } from '../contracts/index.js';
import { parseCorpusDocument } from '../corpus/documents.js';
import { sha256Text } from './canonical.js';

export interface VerifyResult {
Expand All @@ -32,36 +33,19 @@ export class BronzeCorruptionError extends Error {
}
}

interface BronzeSplit {
yamlText: string;
body: string;
}

/**
* Splits a Bronze file into its YAML frontmatter text and body.
* Format: ---\n<yaml>\n---\n<body>
*/
function splitBronzeFile(content: string): BronzeSplit | null {
if (!content.startsWith('---\n')) return null;
const afterOpen = content.slice(4);
const closeIdx = afterOpen.indexOf('\n---\n');
if (closeIdx === -1) return null;
return {
yamlText: afterOpen.slice(0, closeIdx),
body: afterOpen.slice(closeIdx + 5),
};
}

/** Parses a Bronze file's frontmatter + validates through BronzeRecordSchema. */
export function parseBronzeRecord(content: string): BronzeRecord {
return parseBronzeFile(content).record;
}

export function parseBronzeFile(content: string): { record: BronzeRecord; body: string } {
const split = splitBronzeFile(content);
if (split === null) throw new Error('not a valid Bronze file: missing frontmatter');
const parsed = YAML.parse(split.yamlText) as unknown;
return { record: BronzeRecordSchema.parse(parsed), body: split.body };
const parsed = parseCorpusDocument(content, BronzeRecordSchema, Object.keys(BronzeRecordSchema.shape));
if (!parsed.valid) {
const detail = parsed.failure.reason === 'missing-frontmatter'
? 'missing frontmatter' : parsed.failure.detail;
throw new Error(`not a valid Bronze file: ${detail}`);
}
return { record: parsed.data, body: parsed.body };
}

/** Serializes a BronzeRecord + canonical body into the on-disk file format. */
Expand Down Expand Up @@ -95,18 +79,13 @@ export async function collectBronzeHashes(root: string): Promise<Map<string, str
if (!filePath.endsWith('.md')) return;
try {
const content = await readFile(filePath, 'utf8');
const split = splitBronzeFile(content);
if (split === null) return;
const parsed = YAML.parse(split.yamlText) as unknown;
const result = BronzeRecordSchema.safeParse(parsed);
if (result.success) {
const actualHash = sha256Text(split.body);
if (actualHash !== result.data.sha256) {
throw new BronzeCorruptionError(filePath, result.data.sha256, actualHash);
}
const relPath = relative(root, filePath).replace(/\\/gu, '/');
hashes.set(result.data.sha256, relPath);
const { record, body } = parseBronzeFile(content);
const actualHash = sha256Text(body);
if (actualHash !== record.sha256) {
throw new BronzeCorruptionError(filePath, record.sha256, actualHash);
}
const relPath = relative(root, filePath).replace(/\\/gu, '/');
hashes.set(record.sha256, relPath);
} catch (err) {
if (err instanceof BronzeCorruptionError) throw err;
/* skip unreadable or invalid files */
Expand Down Expand Up @@ -181,10 +160,7 @@ export async function atomicWriteBronze(
*/
export async function verifyBronzeFile(filePath: string): Promise<VerifyResult> {
const content = await readFile(filePath, 'utf8');
const split = splitBronzeFile(content);
if (split === null) throw new Error(`${filePath}: not a valid Bronze file (no frontmatter)`);
const parsed = YAML.parse(split.yamlText) as unknown;
const record = BronzeRecordSchema.parse(parsed);
const actual = sha256Text(split.body);
const { record, body } = parseBronzeFile(content);
const actual = sha256Text(body);
return { valid: actual === record.sha256, expected: record.sha256, actual };
}
145 changes: 145 additions & 0 deletions test/bronze-line-endings.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'node:test';
import { canonicalBronzeBody, sha256Text } from '../src/bronze/canonical.js';
import { ingestCapture } from '../src/bronze/ingest.js';
import {
BronzeCorruptionError,
collectBronzeHashes,
parseBronzeFile,
parseBronzeRecord,
verifyBronzeFile,
} from '../src/bronze/store.js';
import { collectBronzeFilesDetailed } from '../src/corpus/collect.js';
import { createVerifiedBronzeReader } from '../src/refine/evidence.js';

const OPTIONS = { now: new Date('2026-01-02T00:00:00Z'), sourceKind: 'article' };
const BODIES = [
{ name: 'ordinary body', body: '# Record\n\nEvidence stays unchanged.\n' },
{ name: 'lone CR body', body: '# Record\n\nEvidence\rstays unchanged.\n' },
];
const ENDINGS = [
{ name: 'LF', value: '\n' },
{ name: 'CRLF', value: '\r\n' },
];

async function withCapture(
body: string,
ending: string,
fn: (root: string, sourcePath: string, stored: string) => Promise<void>,
): Promise<void> {
const root = await mkdtemp(join(tmpdir(), 'ziggurat-bronze-endings-'));
try {
await mkdir(join(root, 'inbox'));
await writeFile(join(root, 'inbox', 'original.md'), body, 'utf8');
const first = await ingestCapture(root, 'inbox/original.md', OPTIONS);
assert.equal(first.status, 'created');
assert.equal(first.sha256, sha256Text(body));
const filePath = join(root, first.source_path);
const stored = (await readFile(filePath, 'utf8')).replace(/\n/g, ending);
await writeFile(filePath, stored, 'utf8');
await fn(root, first.source_path, stored);
} finally {
await rm(root, { recursive: true, force: true });
}
}

async function bronzeFiles(root: string): Promise<string[]> {
const entries = await readdir(join(root, 'bronze'), { recursive: true, withFileTypes: true });
return entries.filter(entry => entry.isFile()).map(entry => entry.name).sort();
}

for (const ending of ENDINGS) {
for (const fixture of BODIES) {
test(`Bronze ${ending.name}: parsing, live reading and dedup agree for ${fixture.name}`, async () => {
const { body } = fixture;
const expected = sha256Text(body);
await withCapture(body, ending.value, async (root, sourcePath, stored) => {
const filePath = join(root, sourcePath);
const live = await createVerifiedBronzeReader(root).read(root, sourcePath);
assert.equal(live.body, body);
assert.equal(live.record.sha256, expected);
assert.deepEqual(parseBronzeFile(stored), { record: live.record, body });
assert.deepEqual(parseBronzeRecord(stored), live.record);
assert.deepEqual(await verifyBronzeFile(filePath), { valid: true, expected, actual: expected });
assert.deepEqual(await collectBronzeHashes(root), new Map([[expected, sourcePath]]));

const { records, rejected } = await collectBronzeFilesDetailed(root);
assert.deepEqual(rejected, []);
assert.equal(records.length, 1);
assert.equal(records[0]?.body, body);
assert.equal(records[0]?.sha256, expected);
assert.equal(records[0]?.hashVerified, true);
assert.equal(canonicalBronzeBody(body.replace(/\n/g, '\r\n')), body);
if (body.includes('\r')) {
assert.notEqual(expected, sha256Text(body.replace(/\r/g, '\n')));
}

const before = await bronzeFiles(root);
assert.equal(before.length, 1);
// A distinct filename prevents the no-overwrite guard from masking failed deduplication.
const inboxPath = join(root, 'inbox', 'different-name.md');
await writeFile(inboxPath, body, 'utf8');
const result = await ingestCapture(root, 'inbox/different-name.md', OPTIONS);
assert.deepEqual(result, { status: 'duplicate', source_path: sourcePath, sha256: expected });
assert.deepEqual(await bronzeFiles(root), before);
assert.equal(await readFile(inboxPath, 'utf8'), body);
assert.equal(await readFile(filePath, 'utf8'), stored);
});
});

test(`Bronze ${ending.name}: corrupted ${fixture.name} still blocks ingestion`, async () => {
const { body } = fixture;
const expected = sha256Text(body);
const actual = sha256Text(body.replace('unchanged.', 'unchanged!'));
await withCapture(body, ending.value, async (root, sourcePath, stored) => {
const filePath = join(root, sourcePath);
const mutated = stored.replace('unchanged.', 'unchanged!');
assert.notEqual(mutated, stored);
await writeFile(filePath, mutated, 'utf8');
assert.deepEqual(await verifyBronzeFile(filePath), { valid: false, expected, actual });
const isCorruption = (error: unknown): boolean =>
error instanceof BronzeCorruptionError
&& error.filePath === filePath && error.expected === expected && error.actual === actual;
await assert.rejects(collectBronzeHashes(root), isCorruption);
await assert.rejects(createVerifiedBronzeReader(root).read(root, sourcePath), isCorruption);
const { records } = await collectBronzeFilesDetailed(root);
assert.equal(records.length, 1);
assert.equal(records[0]?.sha256, expected);
assert.equal(records[0]?.hashVerified, false);

const before = await bronzeFiles(root);
const inboxPath = join(root, 'inbox', 'different-name.md');
await writeFile(inboxPath, body, 'utf8');
await assert.rejects(ingestCapture(root, 'inbox/different-name.md', OPTIONS), isCorruption);
assert.deepEqual(await bronzeFiles(root), before);
assert.equal(await readFile(inboxPath, 'utf8'), body);
assert.equal(await readFile(filePath, 'utf8'), mutated);
});
});
}

test(`Bronze ${ending.name}: malformed frontmatter still rejects direct reads and is not deduplicated`, async () => {
await withCapture(BODIES[0]!.body, ending.value, async (root, sourcePath, stored) => {
const filePath = join(root, sourcePath);
const malformed = [
stored.slice(3),
stored.replace('source_id: original', 'source_id: ['),
stored.replace('source_id: original', `source_id: original${ending.value}source_id: duplicate`),
stored.replace('schema_version: 1', 'schema_version: 2'),
stored.replace('schema_version: 1', `schema_version: 1${ending.value}unexpected: true`),
];
for (const content of malformed) {
assert.notEqual(content, stored);
await writeFile(filePath, content, 'utf8');
assert.throws(() => parseBronzeFile(content));
assert.throws(() => parseBronzeRecord(content));
await assert.rejects(verifyBronzeFile(filePath));
await assert.rejects(createVerifiedBronzeReader(root).read(root, sourcePath));
assert.deepEqual(await collectBronzeHashes(root), new Map());
}
});
});
}