From b27d6671cd94c67c1fab6dbd2ae607d98f90bde5 Mon Sep 17 00:00:00 2001 From: Jason Doyle Date: Wed, 16 Sep 2026 12:50:21 -0700 Subject: [PATCH 1/2] Refine SRE evidence source hashing --- benchmarks/sre/README.md | 6 ++++-- benchmarks/sre/run.ts | 40 +++++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/benchmarks/sre/README.md b/benchmarks/sre/README.md index 7773e7d..815aae5 100644 --- a/benchmarks/sre/README.md +++ b/benchmarks/sre/README.md @@ -34,8 +34,10 @@ Use `BENCHMARK_WRITE_RESULTS=1` to regenerate `results/summary.json` and `summary.json` includes every per-run outcome, audit answer, table count, database footprint, and informational duration used by the aggregate report. -It also includes a portable source hash over the benchmark, production kernel, -migrations, lockfile, and PostgreSQL container configuration. +It also includes a portable source hash over benchmark-relevant package +manifest fields, the lockfile, benchmark, production kernel, migrations, and +PostgreSQL container configuration. Descriptive package metadata such as +author and homepage does not invalidate the evidence. CI runs the comparison and verifies that the committed report is generated from `summary.json`, contains three correct runs per variant, has aggregates diff --git a/benchmarks/sre/run.ts b/benchmarks/sre/run.ts index 14125c6..6159bd0 100644 --- a/benchmarks/sre/run.ts +++ b/benchmarks/sre/run.ts @@ -16,6 +16,7 @@ import { SyntheticRemediationTransport, type ProductionConfig, } from "../../src/production/index.js"; +import { stableStringify } from "../../src/util.js"; import { runConventionalBaseline } from "./baseline.js"; import { runKernelVariant } from "./kernel.js"; import { @@ -623,7 +624,6 @@ function listFiles(directory: string): string[] { function benchmarkSourceHash(): string { const paths = [ - "package.json", "package-lock.json", "docker-compose.yml", ...listFiles("benchmarks/sre").filter((path) => @@ -637,19 +637,45 @@ function benchmarkSourceHash(): string { extname(path) === ".ts" && !path.split(/[\\/]/).includes("test"), ), - ].map((path) => path.replaceAll("\\", "/")).sort(); + ].map((path) => path.replaceAll("\\", "/")); + const inputs = [ + { + path: "package.json", + source: benchmarkPackageManifest(), + }, + ...paths.map((path) => ({ + path, + source: normalizeLineEndings(readFileSync(resolve(path), "utf8")), + })), + ].sort((left, right) => left.path.localeCompare(right.path)); const digest = createHash("sha256"); - for (const path of paths) { - digest.update(path); + for (const input of inputs) { + digest.update(input.path); digest.update("\0"); - digest.update( - normalizeLineEndings(readFileSync(resolve(path), "utf8")), - ); + digest.update(input.source); digest.update("\0"); } return digest.digest("hex"); } +function benchmarkPackageManifest(): string { + const manifest: unknown = JSON.parse( + readFileSync(resolve("package.json"), "utf8"), + ); + if (!isRecord(manifest)) { + throw new Error("package.json must contain an object"); + } + return stableStringify({ + dependencies: manifest.dependencies, + devDependencies: manifest.devDependencies, + engines: manifest.engines, + overrides: manifest.overrides, + packageManager: manifest.packageManager, + scripts: manifest.scripts, + type: manifest.type, + }); +} + function normalizeLineEndings(value: string): string { return value.replaceAll("\r\n", "\n"); } From 9aca127bd98a5047463f7f777b74f778aaac10bb Mon Sep 17 00:00:00 2001 From: Jason Doyle Date: Wed, 16 Sep 2026 12:52:01 -0700 Subject: [PATCH 2/2] Refresh published SRE benchmark evidence --- benchmarks/sre/results/report.md | 10 +++++----- benchmarks/sre/results/summary.json | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/benchmarks/sre/results/report.md b/benchmarks/sre/results/report.md index 647d577..0b6826b 100644 --- a/benchmarks/sre/results/report.md +++ b/benchmarks/sre/results/report.md @@ -2,9 +2,9 @@ Generated from `summary.json`. -Source revision: `478122eee5118456b32bfe71f44d7d58f2a82348` +Source revision: `b27d6671cd94c67c1fab6dbd2ae607d98f90bde5` -Source hash: `01b4f1e8c7fae94284afad63dd8d3f84765423ced3642edef0eb69a8a4a5b68b` +Source hash: `b88e11bfb6346c1200eea4cc027fdfb25ee16291e2602e17bc797f8abf8c8650` ## Correctness @@ -28,7 +28,7 @@ dependency. The full kernel dependency contains 16709 nonblank TypeScript source lines. The benchmark runner and engine-specific audit verification contain -1445 nonblank TypeScript source lines. +1470 nonblank TypeScript source lines. They are excluded from both application columns. Dependency and harness code is not application-authored, but it remains code that must be understood, operated, or upgraded. @@ -44,8 +44,8 @@ operated, or upgraded. | Variant | Median milliseconds | | --- | ---: | -| Conventional PostgreSQL | 50.64 | -| Agentic Data Kernel | 858.53 | +| Conventional PostgreSQL | 57.90 | +| Agentic Data Kernel | 929.60 | Runtime is not a headline metric. The variants perform different work and this deterministic smoke benchmark is not a latency study. diff --git a/benchmarks/sre/results/summary.json b/benchmarks/sre/results/summary.json index a402854..da70e0d 100644 --- a/benchmarks/sre/results/summary.json +++ b/benchmarks/sre/results/summary.json @@ -4,8 +4,8 @@ "environment": { "node": "v22.22.2", "postgres": "18.6 (Debian 18.6-1.pgdg12+2)", - "commit": "478122eee5118456b32bfe71f44d7d58f2a82348", - "sourceHash": "01b4f1e8c7fae94284afad63dd8d3f84765423ced3642edef0eb69a8a4a5b68b" + "commit": "b27d6671cd94c67c1fab6dbd2ae607d98f90bde5", + "sourceHash": "b88e11bfb6346c1200eea4cc027fdfb25ee16291e2602e17bc797f8abf8c8650" }, "runs": [ { @@ -31,7 +31,7 @@ "provider reconciliation": true, "verification and terminal state": true }, - "durationMs": 70.2844 + "durationMs": 81.21829999999954 }, "operatedTables": 8, "databaseBytes": 540672 @@ -59,7 +59,7 @@ "provider reconciliation": true, "verification and terminal state": true }, - "durationMs": 879.4365999999998 + "durationMs": 949.5142999999998 }, "operatedTables": 18, "databaseBytes": 1572864 @@ -87,7 +87,7 @@ "provider reconciliation": true, "verification and terminal state": true }, - "durationMs": 50.64379999999983 + "durationMs": 47.11999999999989 }, "operatedTables": 8, "databaseBytes": 540672 @@ -115,7 +115,7 @@ "provider reconciliation": true, "verification and terminal state": true }, - "durationMs": 858.5281 + "durationMs": 872.3099000000002 }, "operatedTables": 18, "databaseBytes": 1572864 @@ -143,7 +143,7 @@ "provider reconciliation": true, "verification and terminal state": true }, - "durationMs": 49.347099999999955 + "durationMs": 57.89609999999993 }, "operatedTables": 8, "databaseBytes": 540672 @@ -171,7 +171,7 @@ "provider reconciliation": true, "verification and terminal state": true }, - "durationMs": 842.4119000000001 + "durationMs": 929.5964000000004 }, "operatedTables": 18, "databaseBytes": 1572864 @@ -270,15 +270,15 @@ } }, "benchmarkHarness": { - "nonblankLines": 1445 + "nonblankLines": 1470 }, "databaseBytes": { "conventionalPostgresMedian": 540672, "agenticDataKernelMedian": 1572864 }, "runtimeMillisecondsInformational": { - "conventionalPostgresMedian": 50.64379999999983, - "agenticDataKernelMedian": 858.5281 + "conventionalPostgresMedian": 57.89609999999993, + "agenticDataKernelMedian": 929.5964000000004 }, "explanationQuestions": 9, "claims": {