Refresh and harden SRE benchmark evidence

benchmarks/sre/README.md

@@ -34,8 +34,10 @@ Use `BENCHMARK_WRITE_RESULTS=1` to regenerate `results/summary.json` and
 
 `summary.json` includes every per-run outcome, audit answer, table count,
 database footprint, and informational duration used by the aggregate report.
-It also includes a portable source hash over the benchmark, production kernel,
-migrations, lockfile, and PostgreSQL container configuration.
+It also includes a portable source hash over benchmark-relevant package
+manifest fields, the lockfile, benchmark, production kernel, migrations, and
+PostgreSQL container configuration. Descriptive package metadata such as
+author and homepage does not invalidate the evidence.
 
 CI runs the comparison and verifies that the committed report is generated
 from `summary.json`, contains three correct runs per variant, has aggregates

benchmarks/sre/results/report.md

@@ -2,9 +2,9 @@
 
 Generated from `summary.json`.
 
-Source revision: `478122eee5118456b32bfe71f44d7d58f2a82348`
+Source revision: `b27d6671cd94c67c1fab6dbd2ae607d98f90bde5`
 
-Source hash: `01b4f1e8c7fae94284afad63dd8d3f84765423ced3642edef0eb69a8a4a5b68b`
+Source hash: `b88e11bfb6346c1200eea4cc027fdfb25ee16291e2602e17bc797f8abf8c8650`
 
 ## Correctness
 
@@ -28,7 +28,7 @@ dependency. The full kernel dependency contains 16709
 nonblank TypeScript source lines.
 
 The benchmark runner and engine-specific audit verification contain
-1445 nonblank TypeScript source lines.
+1470 nonblank TypeScript source lines.
 They are excluded from both application columns. Dependency and harness code
 is not application-authored, but it remains code that must be understood,
 operated, or upgraded.
@@ -44,8 +44,8 @@ operated, or upgraded.
 
 | Variant | Median milliseconds |
 | --- | ---: |
-| Conventional PostgreSQL | 50.64 |
-| Agentic Data Kernel | 858.53 |
+| Conventional PostgreSQL | 57.90 |
+| Agentic Data Kernel | 929.60 |
 
 Runtime is not a headline metric. The variants perform different work and this
 deterministic smoke benchmark is not a latency study.

benchmarks/sre/results/summary.json

@@ -4,8 +4,8 @@
   "environment": {
     "node": "v22.22.2",
     "postgres": "18.6 (Debian 18.6-1.pgdg12+2)",
-    "commit": "478122eee5118456b32bfe71f44d7d58f2a82348",
-    "sourceHash": "01b4f1e8c7fae94284afad63dd8d3f84765423ced3642edef0eb69a8a4a5b68b"
+    "commit": "b27d6671cd94c67c1fab6dbd2ae607d98f90bde5",
+    "sourceHash": "b88e11bfb6346c1200eea4cc027fdfb25ee16291e2602e17bc797f8abf8c8650"
   },
   "runs": [
     {
@@ -31,7 +31,7 @@
           "provider reconciliation": true,
           "verification and terminal state": true
         },
-        "durationMs": 70.2844
+        "durationMs": 81.21829999999954
       },
       "operatedTables": 8,
       "databaseBytes": 540672
@@ -59,7 +59,7 @@
           "provider reconciliation": true,
           "verification and terminal state": true
         },
-        "durationMs": 879.4365999999998
+        "durationMs": 949.5142999999998
       },
       "operatedTables": 18,
       "databaseBytes": 1572864
@@ -87,7 +87,7 @@
           "provider reconciliation": true,
           "verification and terminal state": true
         },
-        "durationMs": 50.64379999999983
+        "durationMs": 47.11999999999989
       },
       "operatedTables": 8,
       "databaseBytes": 540672
@@ -115,7 +115,7 @@
           "provider reconciliation": true,
           "verification and terminal state": true
         },
-        "durationMs": 858.5281
+        "durationMs": 872.3099000000002
       },
       "operatedTables": 18,
       "databaseBytes": 1572864
@@ -143,7 +143,7 @@
           "provider reconciliation": true,
           "verification and terminal state": true
         },
-        "durationMs": 49.347099999999955
+        "durationMs": 57.89609999999993
       },
       "operatedTables": 8,
       "databaseBytes": 540672
@@ -171,7 +171,7 @@
           "provider reconciliation": true,
           "verification and terminal state": true
         },
-        "durationMs": 842.4119000000001
+        "durationMs": 929.5964000000004
       },
       "operatedTables": 18,
       "databaseBytes": 1572864
@@ -270,15 +270,15 @@
     }
   },
   "benchmarkHarness": {
-    "nonblankLines": 1445
+    "nonblankLines": 1470
   },
   "databaseBytes": {
     "conventionalPostgresMedian": 540672,
     "agenticDataKernelMedian": 1572864
   },
   "runtimeMillisecondsInformational": {
-    "conventionalPostgresMedian": 50.64379999999983,
-    "agenticDataKernelMedian": 858.5281
+    "conventionalPostgresMedian": 57.89609999999993,
+    "agenticDataKernelMedian": 929.5964000000004
   },
   "explanationQuestions": 9,
   "claims": {

benchmarks/sre/run.ts

@@ -16,6 +16,7 @@ import {
   SyntheticRemediationTransport,
   type ProductionConfig,
 } from "../../src/production/index.js";
+import { stableStringify } from "../../src/util.js";
 import { runConventionalBaseline } from "./baseline.js";
 import { runKernelVariant } from "./kernel.js";
 import {
@@ -623,7 +624,6 @@ function listFiles(directory: string): string[] {
 
 function benchmarkSourceHash(): string {
   const paths = [
-    "package.json",
     "package-lock.json",
     "docker-compose.yml",
     ...listFiles("benchmarks/sre").filter((path) =>
@@ -637,19 +637,45 @@ function benchmarkSourceHash(): string {
         extname(path) === ".ts" &&
         !path.split(/[\\/]/).includes("test"),
     ),
-  ].map((path) => path.replaceAll("\\", "/")).sort();
+  ].map((path) => path.replaceAll("\\", "/"));
+  const inputs = [
+    {
+      path: "package.json",
+      source: benchmarkPackageManifest(),
+    },
+    ...paths.map((path) => ({
+      path,
+      source: normalizeLineEndings(readFileSync(resolve(path), "utf8")),
+    })),
+  ].sort((left, right) => left.path.localeCompare(right.path));
   const digest = createHash("sha256");
-  for (const path of paths) {
-    digest.update(path);
+  for (const input of inputs) {
+    digest.update(input.path);
     digest.update("\0");
-    digest.update(
-      normalizeLineEndings(readFileSync(resolve(path), "utf8")),
-    );
+    digest.update(input.source);
     digest.update("\0");
   }
   return digest.digest("hex");
 }
 
+function benchmarkPackageManifest(): string {
+  const manifest: unknown = JSON.parse(
+    readFileSync(resolve("package.json"), "utf8"),
+  );
+  if (!isRecord(manifest)) {
+    throw new Error("package.json must contain an object");
+  }
+  return stableStringify({
+    dependencies: manifest.dependencies,
+    devDependencies: manifest.devDependencies,
+    engines: manifest.engines,
+    overrides: manifest.overrides,
+    packageManager: manifest.packageManager,
+    scripts: manifest.scripts,
+    type: manifest.type,
+  });
+}
+
 function normalizeLineEndings(value: string): string {
   return value.replaceAll("\r\n", "\n");
 }