From 15496fa021c3305bae63e6b2b89e850a32904975 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:39:44 -0400 Subject: [PATCH 1/4] docs(perf): freeze the matched application benchmark protocol and record known startup cost HACK-1169: freeze revision 1 of the v5 matched application benchmark before any trials. docs/reference/application-benchmark-protocol.md and its machine-readable copy fix the exact artifacts (v5.0.0-next.5 at 28de2f89 and stable 4.1.1, 4.2.0, 4.2.1 with release asset SHA-256 digests), the M3 host with Docker CLI 29.4.0, Compose v5.1.2 and OrbStack 2.2.3, Event Agent plus a pinned small fixture, capacities, cache classes, a 10-round Williams rotation, p95 sample floors, observer overhead accounting and the program budgets. scripts/check-application-benchmark-protocol.py validates the frozen file and judges retained summaries offline with closed, value-free codes; anything short of complete, sufficient samples is inconclusive. Offline unittest controls run in CI. docs/performance.md records the user-accepted known v5 limit (HACK-1123): the storage-witness cost of a warm native up after #313, with rationale and revisit trigger. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 + docs/performance.md | 41 ++ docs/reference.md | 1 + .../application-benchmark-protocol.md | 277 +++++++++++ .../native-compose-storage-witness.md | 2 + scripts/application-benchmark-protocol.json | 234 ++++++++++ .../check-application-benchmark-protocol.py | 437 ++++++++++++++++++ .../test_application_benchmark_protocol.py | 297 ++++++++++++ 8 files changed, 1293 insertions(+) create mode 100644 docs/reference/application-benchmark-protocol.md create mode 100644 scripts/application-benchmark-protocol.json create mode 100644 scripts/check-application-benchmark-protocol.py create mode 100644 tests/python/test_application_benchmark_protocol.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 987da5080..5003cc4dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,10 @@ jobs: run: python3 -m unittest discover -s tests/python -p test_native_compose_benchmark.py - name: Check native frontend acceptance controls run: python3 -m unittest discover -s tests/python -p test_native_frontend_acceptance.py + - name: Check application benchmark protocol controls + run: | + python3 scripts/check-application-benchmark-protocol.py + python3 -m unittest discover -s tests/python -p test_application_benchmark_protocol.py runtime-core: name: Candidate core (${{ matrix.os }}) diff --git a/docs/performance.md b/docs/performance.md index be295998a..4953ff2f5 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -707,5 +707,46 @@ bound to its incarnation, so it still acquires and validates every image. Preloa make bases project-specific and require the verifier to check layer contents, so bases stay image-free. The larger remaining costs are image acquisition and per-incarnation revalidation. +## Matched application benchmark protocol + +Revision 1 of the v5 matched application benchmark is frozen in the +[application benchmark protocol](reference/application-benchmark-protocol.md) +(HACK-1169). It fixes the exact candidate and stable artifacts, the host and engine +versions, the Event Agent and small-fixture workloads, capacities, cache classes, a +10-round rotated order, sample counts, observer accounting and the program budgets. +It records no results. + +## Known v5 limit: storage-witness startup cost + +On 2026-10-19 the user accepted the current storage-witness cost as a known v5 limit +for now (HACK-1123). It is part of every native warm `up` and is not removed from +candidate timings under the benchmark protocol. + +**Current cost.** Since #313, a warm, image-only, unrouted native `up` or `restart` +without hooks runs 4 storage proofs instead of 6. Each proof starts 3 helper +containers. Estimated from a traced hosted docker-e2e run, the warm `up` went from +about 22 s to about 17.7 s, and the witness share went from about 13 s to about 8.7 s. +The target is at most 5 s of witness overhead per warm `up`, so the target is not met. +These are hosted-engine estimates, not measurements from the frozen benchmark host. +The proof sequence is described in the +[storage witness reference](reference/native-compose-storage-witness.md). + +**Why it is accepted.** The remaining proofs are the witness safety contract. Cutting +more changes that contract, not just its cost: + +- Reusing a proof receipt across commands was analyzed and found unsafe. The root + identity is only device, inode, uid and gid, so a recreated volume can match it. The + marker is a `user.*` extended attribute that any workload container can remove. + Docker exposes no boot ID or start time, and the engine ID survives restarts. + Checking a stored receipt would itself need a helper container. +- A third in-command cut was left out because it changes when a refusal surfaces. + +Every command therefore still opens with a full proof. + +**Revisit trigger.** Revisit this limit after the matched benchmark protocol +(HACK-1169) is frozen and its first complete revision-1 cohort has run, using that +cohort's measured warm startup and restart results. No proof-policy change may be made +without a separate safety review and approval. + Native recovery investigations can use the explicit, value-free [process census phase diagnostics](guides/native-census-diagnostics.md). diff --git a/docs/reference.md b/docs/reference.md index ee490977b..edca74747 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -21,6 +21,7 @@ - [Native configuration validation](./reference/native-config-compiler.md) — the authored `.hack/hack.project.json` compiler contract - [Native configuration with Compose](./reference/native-compose-commands.md) — the Compose command owner for authored projects - [Authored configuration with the native runtime](./reference/native-authored-commands.md) — the explicit native backend command slices and their support matrix +- [Application benchmark protocol](./reference/application-benchmark-protocol.md) — the frozen matched benchmark artifacts, workloads, order, sample counts and budgets ## Extensions diff --git a/docs/reference/application-benchmark-protocol.md b/docs/reference/application-benchmark-protocol.md new file mode 100644 index 000000000..3489df1ea --- /dev/null +++ b/docs/reference/application-benchmark-protocol.md @@ -0,0 +1,277 @@ +# Matched application benchmark protocol + +This page freezes revision 1 of the protocol for the v5 matched application benchmark +(Linear HACK-1169, under WU10 HACK-1123). It fixes the artifacts, host, workloads, +capacities, cache classes, run order, sample counts, observer accounting and budgets +**before** any trial runs. It records no results. No performance or resource claim +follows from this page. + +The machine-readable copy is +[`scripts/application-benchmark-protocol.json`](../../scripts/application-benchmark-protocol.json). +[`scripts/check-application-benchmark-protocol.py`](../../scripts/check-application-benchmark-protocol.py) +validates it and judges retained summaries offline. It never starts a runtime or calls +Docker. + +A change to anything on this page needs a new protocol revision, made before the +trials it governs. Results from different revisions are never pooled. A cohort that +cannot meet this protocol is reported as inconclusive, not adjusted after the fact. + +## Artifacts + +Every lane runs an exact released artifact, installed into its own isolated home and +state paths. A lane never uses whatever `hack` is on `PATH`; the M3 host currently has +4.2.0 installed, and that installation is not a lane. Before each cohort, the harness +must verify the SHA-256 digest of each downloaded asset and record the executable's +reported version. A mismatch refuses the cohort. + +| Lane | Tag | Commit | Asset | Asset SHA-256 | +| --- | --- | --- | --- | --- | +| `candidate` | `v5.0.0-next.5` (prerelease) | `28de2f8902d3776c3da9326e7ae6c6aa6e7f69bc` | `hack-5.0.0-next.5-darwin-arm64-native.tar.gz` | `ba1da4d349ab70069a1b0775ba671838440b5479cffd1fc2fe41c59816c3df11` | +| `hack-4.1.1` | `v4.1.1` | `74bf0672f41bd3f68a4ce6cba28b9ba15467c2b2` | `hack-4.1.1-darwin-arm64.tar.gz` | `968f75fe66cd84fdf93b0c5b64ca5df02f98c74281b5a9238125b3afd2f3d72b` | +| `hack-4.2.0` | `v4.2.0` | `575847486963db94623e1e7f8cd49acca286c517` | `hack-4.2.0-darwin-arm64.tar.gz` | `c62b533aaddbeaafa594cab682cf10041c9bcaea69e2fe25e33e85df49654a85` | +| `hack-4.2.1` | `v4.2.1` | `5a4c51babb7ef725a30f2f3fd3677081a64bc247` | `hack-4.2.1-darwin-arm64.tar.gz` | `557583d946abcd0dd0a3fa12b2f7b81f18310d594b0699d663ebf239d0d4d729` | +| `compose` | none | none | Docker CLI and Compose on the host | see below | + +The candidate is built from `28de2f89`, the `v5.0.0-next.5` tag commit. A later +candidate needs a new revision; it does not inherit this one's results. + +## Host and engine + +Native stateful benchmarks are host-specific, so one host is frozen. These values were +read from the host on 2026-10-19: + +| Item | Value | +| --- | --- | +| Host | `m3-max`: Apple M3 Max, 16 logical CPUs, 128 GiB memory | +| OS | macOS 27.0 | +| Docker CLI | 29.4.0 | +| Docker Compose | v5.1.2 | +| OrbStack | 2.2.3 (build 20963), configured with 16 CPUs and 71680 MiB memory | + +Before each cohort, the harness records these values again. If any of them differs, +the cohort is refused and a new revision is needed. Results from another host, +including the M5 Max development host, are not revision-1 results. + +Only one stateful benchmark owner may use the host and its runtimes at a time. Other +agents' runtime tests, builds and Docker work must not overlap a cohort. Independent +source-only checks may still run elsewhere. + +## Capacities + +Each lane runs with the capacity a user gets by default. Capacities are recorded and +reported side by side; they are never used to scale results. + +| Lane | Capacity | +| --- | --- | +| `candidate` | native `development` profile: 4 vCPUs, 6144 MiB guest memory, 32 GiB storage, 10 GiB overlay | +| `compose`, `hack-4.1.1`, `hack-4.2.0`, `hack-4.2.1` | the shared OrbStack VM above: 16 CPUs, 71680 MiB memory limit | + +The capacities differ. The compared footprint metric is the measured physical footprint +of the whole runtime, not the configured limit. A lower configured limit is not a +footprint result. Any capacity-matched control is a separate experiment with its own +revision. + +## Workloads + +Two workloads run in every lane. A cohort records each workload's pin, and every lane +in that cohort uses the same pin. + +**Event Agent.** The actual application configuration, not a reduced copy. The cohort +records the application commit and a digest of its configuration. Its expected shape is +14 services, 9 HTTP health checks and 2 initializers. A different shape refuses the +cohort until the protocol is revised. Credentials come only through authorized managed +injection; candidate endpoints and data stay isolated from v4, global state and QA. +A lifecycle counts as correct only when: + +- every declared long-running service is running and every HTTP health check is healthy; +- both initializers exited 0; +- the routed sign-in page returns HTTP 200; +- a per-run data marker written before the restart reads back after it. + +**Small fixture.** A pinned two-service graph: an initializer and a web service, both +on `oven/bun:1.4.2-slim` with `pull never`, the image ID recorded per cohort. The +initializer creates a SQLite schema in a named volume and exits 0. The web service +serves a health endpoint and the current source token, and reloads on a source edit. +A lifecycle counts as correct only when: + +- the initializer exits 0 and the web health endpoint returns HTTP 200; +- the check task inserts a per-run row and reads back the expected count; +- that row reads back after the restart; +- a source edit is served within the watcher acknowledgement window. + +A lane that cannot run a workload correctly records a failure for that lifecycle. It +is not dropped and its timings are not used. A lane is `not_applicable` only when the +lane cannot express the workload at all, and that reason is recorded. + +## Cache classes + +| Class | Meaning | Use | +| --- | --- | --- | +| `warm` | Engine or VM running; images, dependency and build caches filled by an earlier correct lifecycle of the same lane; application volumes kept | Budget class | +| `cold-capacity` | Engine or VM stopped before the trial; images and caches kept | Reported separately | +| `cold-cache` | Engine or VM running; images present; dependency and build caches empty | Reported separately | + +No trial pulls an image, so image download time never enters a lifecycle. The host page +cache is not flushed, and each receipt says so. Every receipt names its cache class. +Classes are never mixed in one statistic. + +## Lifecycle and phases + +One lifecycle is: warm startup to full readiness and correctness, the workload's check +and build/test task, an exec, a source reload, a restart to full readiness and data +readback, a steady idle window, sustained application load, shutdown and cleanup +readback. Each phase is timed and reported on its own: + +`cold-startup`, `warm-startup`, `restart`, `reload`, `build-test`, `exec`, +`steady-idle`, `sustained-load`, `shutdown`, `cleanup`. + +Readiness means the workload's correctness checks pass, not that a process started or +a port opened. Cleanup readback inventories owned containers, processes, sockets and +allocated disk, and keeps persistent data. + +## Order and repetitions + +Each workload runs 10 rounds. Each round runs every lane once, in the order below. This +is a Williams design for five lanes: every lane takes every position twice, and every +lane runs directly after every other lane exactly twice. It balances position and +first-order carryover, so a lane does not benefit from always running after the same +neighbour. The program floor is five rotated lifecycle runs per lane; this protocol uses +10. + +| Round | Order | +| --- | --- | +| 1 | candidate, compose, hack-4.2.1, hack-4.1.1, hack-4.2.0 | +| 2 | compose, hack-4.1.1, candidate, hack-4.2.0, hack-4.2.1 | +| 3 | hack-4.1.1, hack-4.2.0, compose, hack-4.2.1, candidate | +| 4 | hack-4.2.0, hack-4.2.1, hack-4.1.1, candidate, compose | +| 5 | hack-4.2.1, candidate, hack-4.2.0, compose, hack-4.1.1 | +| 6 | hack-4.2.0, hack-4.1.1, hack-4.2.1, compose, candidate | +| 7 | hack-4.2.1, hack-4.2.0, candidate, hack-4.1.1, compose | +| 8 | candidate, hack-4.2.1, compose, hack-4.2.0, hack-4.1.1 | +| 9 | compose, candidate, hack-4.1.1, hack-4.2.1, hack-4.2.0 | +| 10 | hack-4.1.1, compose, hack-4.2.0, candidate, hack-4.2.1 | + +Print the same schedule with +`python3 scripts/check-application-benchmark-protocol.py --schedule`. + +## Sample counts + +- **Lifecycle metrics** (startup, restart, footprint, CPU per task) use the median of + at least 10 lifecycles per lane, with minimum, maximum and interquartile range. +- **Tail metrics** (info, warm admission, cached status, watcher acknowledgement and + request latency under load) use the nearest-rank p95 of at least 60 samples per lane, + per workload. The sample maximum is reported as an upper bound. For any distribution, + the maximum of n samples is at or above the true p95 with probability 1 − 0.95ⁿ; + 59 samples is the smallest count that reaches 95%. +- A statistic with fewer samples than planned, or fewer than these floors, is + inconclusive. It is never a pass. + +## Measurement boundary + +Each lifecycle records: + +- wall time per phase and to full readiness; +- CPU seconds for the CLI process tree, every runtime helper, the whole VM (OrbStack or + the native provider) and the guest, with the sampling window for any CPU percentage; +- CPU per completed task, using a count of completed correct operations; +- physical footprint and peak for the whole runtime, including the VM and every helper. + RSS sums include shared pages and are not reported as physical memory savings; +- logical and allocated disk, before and after cleanup; +- host load, memory pressure and background CPU at the start, end and every second; +- request latency under sustained load; +- every failure, refusal and retry. + +Near-zero CPU percentages are not reported as wins. Unobserved values stay null, and +each summary states its coverage. + +## Observer overhead + +The sampler runs with the same settings in every lane. Before each cohort, it runs +alone for a control window of at least 600 seconds with no workload, to measure its +own CPU and footprint. During the cohort, its own process-tree CPU is recorded for +each lifecycle and reported as its own line. + +Observer cost is never subtracted from a lane. When the candidate and a baseline differ +by less than 2 times the observer's cost for that metric, the comparison is +inconclusive. A summary that does not report observer cost for a compared metric is +inconclusive for that metric. + +## Admission and failures + +A sample is flagged, not dropped, when build tools run, host load or memory pressure is +above the idle baseline, a security scan or indexing burst is attributed to the run, or +any of these could not be observed. Flagged samples are summarized separately. A +background-contention, low-headroom, failed-correctness or stalled-subscriber control +must produce a flag or failure, not a clean sample. + +Every reproducible regression is reported, even where the aggregate looks better. A +regression in a required workflow blocks acceptance until it is fixed or the goal is +explicitly reviewed. + +## Budgets + +**Comparative budgets.** These are judged in the `warm` class, separately against each +applicable baseline: `compose`, `hack-4.1.1`, `hack-4.2.0` and `hack-4.2.1`. There is no +aggregate score. Ratio is candidate divided by baseline; a ratio at the ceiling passes. + +| Metric | Statistic | Ceiling ratio | Program target | +| --- | --- | --- | --- | +| `steady_physical_footprint_bytes` | median | 0.70 | at least 30% lower steady total physical footprint | +| `cpu_seconds_per_task` | median | 0.75 | at least 25% lower CPU per completed matched task | +| `warm_startup_seconds` | median | 0.80 | at least 20% faster median warm startup | +| `warm_restart_seconds` | median | 0.80 | at least 20% faster median warm restart | + +**Absolute budgets.** These are candidate-only. They come from the program acceptance +document. + +| Metric | Statistic | Budget | Workload | +| --- | --- | --- | --- | +| `info_latency_ms` | p95 | ≤ 100 ms | both | +| `warm_admission_ms` | p95 | ≤ 250 ms | both | +| `cached_status_1000_objects_ms` | p95 | ≤ 150 ms, with 1,000 cached objects | small fixture | +| `idle_control_plane_core_fraction` | mean over 600 s | < 1% of one core | small fixture | +| `watcher_ack_ms` | p95 | ≤ 200 ms | small fixture | + +These are acceptance thresholds, not measured achievements. + +## Judging a summary + +`--evaluate SUMMARY` reads one workload's retained summary and returns one verdict per +budget and baseline: `pass`, `fail`, `not_applicable`, `inconclusive_coverage`, +`inconclusive_samples`, `inconclusive_observer` or `inconclusive_missing`. It exits 0 +only when every verdict is `pass` or `not_applicable`. It exits 1 when any verdict +fails or is inconclusive, and 2 when the protocol or summary is invalid. Diagnostics +are closed codes and never repeat input values. + +A summary has this shape: + +```json +{ + "protocol_revision": 1, + "workload": "small-fixture", + "observer_cost": { "cpu_seconds_per_task": 0.02 }, + "lanes": { + "candidate": { "cpu_seconds_per_task": { "samples": [1.2], "expected": 10 } }, + "compose": { "cpu_seconds_per_task": { "not_applicable": true } } + } +} +``` + +## Known limits + +The accepted storage-witness startup cost is recorded in +[performance diagnostics](../performance.md#known-v5-limit-storage-witness-startup-cost). +It is part of every candidate warm `up` measured under this protocol, and is not +removed from candidate timings. + +## Checks + +```sh +python3 scripts/check-application-benchmark-protocol.py +python3 -m unittest discover -s tests/python -p test_application_benchmark_protocol.py +``` + +The unit tests check the frozen file, the rotation balance, the p95 sample floor and +that this page carries every frozen artifact and budget. They also prove that a +weakened budget, a bad digest, an unbalanced order, too few samples or a missing +observer policy is refused. They run offline, locally and in CI. diff --git a/docs/reference/native-compose-storage-witness.md b/docs/reference/native-compose-storage-witness.md index 8146831af..a4ad83673 100644 --- a/docs/reference/native-compose-storage-witness.md +++ b/docs/reference/native-compose-storage-witness.md @@ -137,6 +137,8 @@ proves four times (opening, prepublication, effect-entry and post-workload) rather than six. With `HACK_NATIVE_COMPOSE_PHASE_TRACE=1`, each performed verification proof emits one `storage.proof` span; a reused proof emits none. These counts establish call ordering and helper invocations, not wall time. +The remaining per-command cost is an accepted known v5 limit; see +[performance diagnostics](../performance.md#known-v5-limit-storage-witness-startup-cost). The one-off `run` path separately omits the enrollment-tail read that would immediately repeat its mandatory pre-spawn witness read. `up` and `restart` diff --git a/scripts/application-benchmark-protocol.json b/scripts/application-benchmark-protocol.json new file mode 100644 index 000000000..1823f14b2 --- /dev/null +++ b/scripts/application-benchmark-protocol.json @@ -0,0 +1,234 @@ +{ + "protocol": "hack-application-benchmark", + "revision": 1, + "frozen_on": "2026-10-19", + "linear_issue": "HACK-1169", + "document": "docs/reference/application-benchmark-protocol.md", + "host": { + "id": "m3-max", + "cpu": "Apple M3 Max", + "logical_cpus": 16, + "memory_bytes": 137438953472, + "os": "macOS 27.0", + "docker_cli": "29.4.0", + "compose": "5.1.2", + "orbstack": { + "version": "2.2.3", + "build": "20963", + "cpu": 16, + "memory_mib": 71680 + } + }, + "lanes": [ + { + "id": "candidate", + "kind": "hack", + "version": "5.0.0-next.5", + "tag": "v5.0.0-next.5", + "commit": "28de2f8902d3776c3da9326e7ae6c6aa6e7f69bc", + "asset": "hack-5.0.0-next.5-darwin-arm64-native.tar.gz", + "asset_sha256": "ba1da4d349ab70069a1b0775ba671838440b5479cffd1fc2fe41c59816c3df11", + "backend": "native", + "capacity": { + "profile": "development", + "vcpus": 4, + "memory_mib": 6144, + "storage_gib": 32, + "overlay_gib": 10 + } + }, + { + "id": "compose", + "kind": "compose", + "engine": "orbstack", + "compose_version": "5.1.2", + "backend": "docker-compose" + }, + { + "id": "hack-4.1.1", + "kind": "hack", + "version": "4.1.1", + "tag": "v4.1.1", + "commit": "74bf0672f41bd3f68a4ce6cba28b9ba15467c2b2", + "asset": "hack-4.1.1-darwin-arm64.tar.gz", + "asset_sha256": "968f75fe66cd84fdf93b0c5b64ca5df02f98c74281b5a9238125b3afd2f3d72b", + "backend": "compose" + }, + { + "id": "hack-4.2.0", + "kind": "hack", + "version": "4.2.0", + "tag": "v4.2.0", + "commit": "575847486963db94623e1e7f8cd49acca286c517", + "asset": "hack-4.2.0-darwin-arm64.tar.gz", + "asset_sha256": "c62b533aaddbeaafa594cab682cf10041c9bcaea69e2fe25e33e85df49654a85", + "backend": "compose" + }, + { + "id": "hack-4.2.1", + "kind": "hack", + "version": "4.2.1", + "tag": "v4.2.1", + "commit": "5a4c51babb7ef725a30f2f3fd3677081a64bc247", + "asset": "hack-4.2.1-darwin-arm64.tar.gz", + "asset_sha256": "557583d946abcd0dd0a3fa12b2f7b81f18310d594b0699d663ebf239d0d4d729", + "backend": "compose" + } + ], + "workloads": [ + { + "id": "event-agent", + "lanes": [ + "candidate", + "compose", + "hack-4.1.1", + "hack-4.2.0", + "hack-4.2.1" + ], + "pin": "application commit and configuration digest recorded per cohort; identical across lanes", + "shape": { "services": 14, "http_health_checks": 9, "initializers": 2 }, + "correctness": [ + "all declared long-running services running and every HTTP health check healthy", + "both initializers exited 0", + "routed sign-in page returns HTTP 200", + "a per-run data marker written before restart reads back after restart" + ] + }, + { + "id": "small-fixture", + "lanes": [ + "candidate", + "compose", + "hack-4.1.1", + "hack-4.2.0", + "hack-4.2.1" + ], + "pin": "image oven/bun:1.4.2-slim with its image ID recorded per cohort, pull never", + "shape": { "services": 2, "http_health_checks": 1, "initializers": 1 }, + "correctness": [ + "initializer creates the SQLite schema in a named volume and exits 0", + "web health endpoint returns HTTP 200", + "check task inserts a per-run row and reads the expected count", + "the row reads back after restart", + "a source edit is served within the watcher acknowledgement window" + ] + } + ], + "phases": [ + "cold-startup", + "warm-startup", + "restart", + "reload", + "build-test", + "exec", + "steady-idle", + "sustained-load", + "shutdown", + "cleanup" + ], + "cache_classes": { + "budget_class": "warm", + "warm": "engine or VM running; images, dependency and build caches populated by a prior correct lifecycle of the same lane; application volumes retained", + "cold-capacity": "engine or VM stopped before the trial; images and caches retained", + "cold-cache": "engine or VM running; images present; dependency and build caches empty" + }, + "schedule": { + "design": "williams", + "rounds_per_workload": 10, + "orders": [ + ["candidate", "compose", "hack-4.2.1", "hack-4.1.1", "hack-4.2.0"], + ["compose", "hack-4.1.1", "candidate", "hack-4.2.0", "hack-4.2.1"], + ["hack-4.1.1", "hack-4.2.0", "compose", "hack-4.2.1", "candidate"], + ["hack-4.2.0", "hack-4.2.1", "hack-4.1.1", "candidate", "compose"], + ["hack-4.2.1", "candidate", "hack-4.2.0", "compose", "hack-4.1.1"], + ["hack-4.2.0", "hack-4.1.1", "hack-4.2.1", "compose", "candidate"], + ["hack-4.2.1", "hack-4.2.0", "candidate", "hack-4.1.1", "compose"], + ["candidate", "hack-4.2.1", "compose", "hack-4.2.0", "hack-4.1.1"], + ["compose", "candidate", "hack-4.1.1", "hack-4.2.1", "hack-4.2.0"], + ["hack-4.1.1", "compose", "hack-4.2.0", "candidate", "hack-4.2.1"] + ] + }, + "samples": { + "min_lifecycle_runs_per_lane": 10, + "p95_quantile": 0.95, + "p95_method": "nearest-rank", + "p95_bound_confidence": 0.95, + "p95_min_samples": 60 + }, + "budgets": { + "comparative": [ + { + "metric": "steady_physical_footprint_bytes", + "statistic": "median", + "max_ratio": 0.7 + }, + { + "metric": "cpu_seconds_per_task", + "statistic": "median", + "max_ratio": 0.75 + }, + { + "metric": "warm_startup_seconds", + "statistic": "median", + "max_ratio": 0.8 + }, + { + "metric": "warm_restart_seconds", + "statistic": "median", + "max_ratio": 0.8 + } + ], + "absolute": [ + { + "metric": "info_latency_ms", + "statistic": "p95", + "max": 100, + "inclusive": true + }, + { + "metric": "warm_admission_ms", + "statistic": "p95", + "max": 250, + "inclusive": true + }, + { + "metric": "cached_status_1000_objects_ms", + "statistic": "p95", + "max": 150, + "inclusive": true, + "workload": "small-fixture" + }, + { + "metric": "idle_control_plane_core_fraction", + "statistic": "window_mean", + "max": 0.01, + "inclusive": false, + "workload": "small-fixture" + }, + { + "metric": "watcher_ack_ms", + "statistic": "p95", + "max": 200, + "inclusive": true, + "workload": "small-fixture" + } + ] + }, + "observer": { + "control_window_seconds": 600, + "inconclusive_factor": 2 + }, + "known_limits": [ + { + "id": "storage-witness-warm-up", + "issue": "HACK-1123", + "after": "hack-dance/hack#313", + "proofs_per_warm_up": 4, + "warm_up_seconds": 17.7, + "witness_seconds": 8.7, + "witness_target_seconds": 5, + "basis": "estimate from a hosted docker-e2e phase trace, not a revision-1 cohort measurement", + "revisit": "after the first complete revision-1 cohort, or before any proof-policy change" + } + ] +} diff --git a/scripts/check-application-benchmark-protocol.py b/scripts/check-application-benchmark-protocol.py new file mode 100644 index 000000000..05ae20cd4 --- /dev/null +++ b/scripts/check-application-benchmark-protocol.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +"""Validate the frozen matched application benchmark protocol and judge retained summaries. + +This helper never runs a benchmark, starts a runtime, calls Docker or reads host state. It: + + (default) validates scripts/application-benchmark-protocol.json against the + protocol's invariants and prints a value-free verdict; + --schedule prints the frozen rotated lane order for each round; + --evaluate SUMMARY judges a retained cohort summary against the frozen budgets. + +Diagnostics are closed codes from DIAGNOSTICS; they never echo input values or paths. +A budget verdict is one of VERDICTS. Anything short of complete, sufficient, admitted +samples is inconclusive, never a pass. See docs/reference/application-benchmark-protocol.md. +""" +import argparse +import json +import math +from pathlib import Path +import re +import sys + +DEFAULT_PROTOCOL = Path(__file__).resolve().parent / "application-benchmark-protocol.json" +PROTOCOL_ID = "hack-application-benchmark" +PROTOCOL_REVISION = 1 + +COMMIT = re.compile(r"^[0-9a-f]{40}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$") +LANE_ID = re.compile(r"^[a-z0-9][a-z0-9.-]{0,31}$") + +# Program floors (Linear program acceptance, Gate C). A protocol revision may be +# stricter than these, never weaker. +MIN_ROTATED_LIFECYCLE_RUNS = 5 +REQUIRED_BASELINES = ("compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +COMPARATIVE_CEILINGS = { + "steady_physical_footprint_bytes": 0.70, + "cpu_seconds_per_task": 0.75, + "warm_startup_seconds": 0.80, + "warm_restart_seconds": 0.80, +} +# (ceiling, whether the ceiling itself passes): "p95 <= 100 ms" versus "< 1% of one core". +ABSOLUTE_CEILINGS = { + "info_latency_ms": (100, True), + "cached_status_1000_objects_ms": (150, True), + "warm_admission_ms": (250, True), + "idle_control_plane_core_fraction": (0.01, False), + "watcher_ack_ms": (200, True), +} +STATISTICS = ("median", "p95", "window_mean") + +DIAGNOSTICS = frozenset({ + "protocol_unreadable", + "protocol_not_object", + "protocol_id_mismatch", + "protocol_revision_mismatch", + "host_incomplete", + "lanes_malformed", + "lane_id_duplicate", + "candidate_lane_missing", + "baseline_lane_missing", + "artifact_commit_malformed", + "artifact_digest_malformed", + "artifact_version_malformed", + "workloads_malformed", + "workload_lane_unknown", + "cache_class_missing", + "schedule_lane_set_mismatch", + "schedule_rounds_below_minimum", + "schedule_position_unbalanced", + "schedule_carryover_unbalanced", + "p95_samples_insufficient", + "comparative_budget_missing", + "comparative_budget_weakened", + "absolute_budget_missing", + "absolute_budget_weakened", + "statistic_unknown", + "observer_policy_missing", + "known_limit_malformed", + "summary_unreadable", + "summary_not_object", + "summary_revision_mismatch", + "summary_workload_unknown", + "summary_lane_unknown", +}) + +VERDICTS = frozenset({ + "pass", + "fail", + "not_applicable", + "inconclusive_coverage", + "inconclusive_samples", + "inconclusive_observer", + "inconclusive_missing", +}) + + +def williams_orders(lanes): + """Williams design: each lane takes each position, and follows each other lane, equally. + + An even lane count needs one square (n rounds); an odd count needs the square plus + its mirror (2n rounds) to balance first-order carryover. + """ + n = len(lanes) + base, low, high = [0], 1, n - 1 + while len(base) < n: + base.append(low) + low += 1 + if len(base) < n: + base.append(high) + high -= 1 + rows = [[(index + shift) % n for index in base] for shift in range(n)] + if n % 2: + rows += [list(reversed(row)) for row in rows] + return [[lanes[index] for index in row] for row in rows] + + +def position_balanced(orders): + lanes = sorted(orders[0]) + for position in range(len(lanes)): + counts = {lane: 0 for lane in lanes} + for order in orders: + counts[order[position]] += 1 + if len(set(counts.values())) != 1: + return False + return True + + +def carryover_balanced(orders): + """Every ordered (predecessor, successor) pair of distinct lanes occurs equally often.""" + lanes = sorted(orders[0]) + pairs = {(a, b): 0 for a in lanes for b in lanes if a != b} + for order in orders: + for a, b in zip(order, order[1:]): + pairs[(a, b)] += 1 + return len(set(pairs.values())) == 1 + + +def minimum_order_statistic_samples(quantile, confidence): + """Smallest n whose sample maximum bounds the true quantile with this confidence. + + P(max of n samples >= true q-quantile) = 1 - q**n, distribution free. + """ + if not (0 < quantile < 1 and 0 < confidence < 1): + raise ValueError("quantile and confidence must be in (0, 1)") + n = math.ceil(math.log(1 - confidence) / math.log(quantile)) + while 1 - quantile ** n < confidence: + n += 1 + while n > 1 and 1 - quantile ** (n - 1) >= confidence: + n -= 1 + return n + + +def nearest_rank(samples, quantile): + ordered = sorted(samples) + return ordered[max(1, math.ceil(quantile * len(ordered))) - 1] + + +def median(samples): + ordered = sorted(samples) + middle = len(ordered) // 2 + if len(ordered) % 2: + return ordered[middle] + return (ordered[middle - 1] + ordered[middle]) / 2 + + +def _number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def validate_protocol(doc): + """Return sorted closed diagnostic codes; empty means the protocol is valid.""" + if not isinstance(doc, dict): + return ["protocol_not_object"] + found = set() + if doc.get("protocol") != PROTOCOL_ID: + found.add("protocol_id_mismatch") + if doc.get("revision") != PROTOCOL_REVISION: + found.add("protocol_revision_mismatch") + + host = doc.get("host") + host_fields = ("id", "cpu", "logical_cpus", "memory_bytes", "os", "docker_cli", "compose", "orbstack") + if not isinstance(host, dict) or any(not host.get(field) for field in host_fields): + found.add("host_incomplete") + elif not isinstance(host["orbstack"], dict) or any( + not host["orbstack"].get(field) for field in ("version", "build", "cpu", "memory_mib")): + found.add("host_incomplete") + + lanes = doc.get("lanes") + lane_ids = [] + if not isinstance(lanes, list) or not lanes or not all(isinstance(lane, dict) for lane in lanes): + found.add("lanes_malformed") + lanes = [] + for lane in lanes: + lane_id = lane.get("id") + if not isinstance(lane_id, str) or not LANE_ID.fullmatch(lane_id): + found.add("lanes_malformed") + continue + if lane_id in lane_ids: + found.add("lane_id_duplicate") + lane_ids.append(lane_id) + if lane.get("kind") == "hack": + if not isinstance(lane.get("commit"), str) or not COMMIT.fullmatch(lane["commit"]): + found.add("artifact_commit_malformed") + if not isinstance(lane.get("asset_sha256"), str) or not SHA256.fullmatch(lane["asset_sha256"]): + found.add("artifact_digest_malformed") + if not isinstance(lane.get("version"), str) or not VERSION.fullmatch(lane["version"]) \ + or lane.get("tag") != "v" + str(lane.get("version")): + found.add("artifact_version_malformed") + elif lane.get("kind") == "compose": + if not lane.get("engine") or not lane.get("compose_version"): + found.add("lanes_malformed") + else: + found.add("lanes_malformed") + if "candidate" not in lane_ids: + found.add("candidate_lane_missing") + if any(baseline not in lane_ids for baseline in REQUIRED_BASELINES): + found.add("baseline_lane_missing") + + workloads = doc.get("workloads") + if not isinstance(workloads, list) or len(workloads) < 2 or not all(isinstance(w, dict) for w in workloads): + found.add("workloads_malformed") + workloads = [] + workload_ids = {w.get("id") for w in workloads} + if not {"event-agent", "small-fixture"} <= workload_ids: + found.add("workloads_malformed") + for workload in workloads: + applicable = workload.get("lanes") + if not isinstance(applicable, list) or "candidate" not in applicable: + found.add("workloads_malformed") + elif any(lane not in lane_ids for lane in applicable): + found.add("workload_lane_unknown") + if not workload.get("correctness"): + found.add("workloads_malformed") + + cache = doc.get("cache_classes") + if not isinstance(cache, dict) or not {"warm", "cold-capacity", "cold-cache"} <= set(cache) \ + or cache.get("budget_class") != "warm": + found.add("cache_class_missing") + + schedule = doc.get("schedule") + orders = schedule.get("orders") if isinstance(schedule, dict) else None + if not isinstance(orders, list) or not orders or not all(isinstance(order, list) for order in orders): + found.add("schedule_lane_set_mismatch") + else: + if any(sorted(order) != sorted(lane_ids) or len(set(order)) != len(order) for order in orders): + found.add("schedule_lane_set_mismatch") + else: + planned = doc.get("samples", {}).get("min_lifecycle_runs_per_lane") \ + if isinstance(doc.get("samples"), dict) else None + if not isinstance(planned, int) or isinstance(planned, bool) \ + or planned < MIN_ROTATED_LIFECYCLE_RUNS or len(orders) < planned: + found.add("schedule_rounds_below_minimum") + if not position_balanced(orders): + found.add("schedule_position_unbalanced") + if not carryover_balanced(orders): + found.add("schedule_carryover_unbalanced") + + samples = doc.get("samples") + try: + required = minimum_order_statistic_samples(samples["p95_quantile"], samples["p95_bound_confidence"]) + if samples["p95_quantile"] != 0.95 or samples["p95_min_samples"] < required: + found.add("p95_samples_insufficient") + except (TypeError, KeyError, ValueError): + found.add("p95_samples_insufficient") + + budgets = doc.get("budgets") if isinstance(doc.get("budgets"), dict) else {} + comparative = {b.get("metric"): b for b in budgets.get("comparative", []) if isinstance(b, dict)} + for metric, ceiling in COMPARATIVE_CEILINGS.items(): + budget = comparative.get(metric) + if budget is None: + found.add("comparative_budget_missing") + elif not _number(budget.get("max_ratio")) or budget["max_ratio"] > ceiling: + found.add("comparative_budget_weakened") + absolute = {b.get("metric"): b for b in budgets.get("absolute", []) if isinstance(b, dict)} + for metric, (ceiling, ceiling_inclusive) in ABSOLUTE_CEILINGS.items(): + budget = absolute.get(metric) + if budget is None: + found.add("absolute_budget_missing") + elif not _number(budget.get("max")) or not isinstance(budget.get("inclusive"), bool) \ + or budget["max"] > ceiling \ + or (budget["max"] == ceiling and budget["inclusive"] and not ceiling_inclusive): + found.add("absolute_budget_weakened") + for budget in list(comparative.values()) + list(absolute.values()): + if budget.get("statistic") not in STATISTICS: + found.add("statistic_unknown") + + observer = doc.get("observer") + if not isinstance(observer, dict) or not _number(observer.get("inconclusive_factor")) \ + or observer["inconclusive_factor"] < 1 or not _number(observer.get("control_window_seconds")) \ + or observer["control_window_seconds"] < 600: + found.add("observer_policy_missing") + + for limit in doc.get("known_limits", []): + if not isinstance(limit, dict) or not all( + _number(limit.get(field)) for field in ("warm_up_seconds", "witness_seconds", "witness_target_seconds")) \ + or not limit.get("issue") or not limit.get("revisit"): + found.add("known_limit_malformed") + return sorted(found) + + +def _samples(entry): + if not isinstance(entry, dict) or not isinstance(entry.get("samples"), list) \ + or not all(_number(value) for value in entry["samples"]): + return None, None + expected = entry.get("expected") + return entry["samples"], expected if isinstance(expected, int) and not isinstance(expected, bool) else None + + +def _statistic(samples, statistic): + if statistic == "median": + return median(samples) + if statistic == "p95": + return nearest_rank(samples, 0.95) + return sum(samples) / len(samples) + + +def _sample_floor(protocol, statistic): + if statistic == "p95": + return protocol["samples"]["p95_min_samples"] + return protocol["samples"]["min_lifecycle_runs_per_lane"] + + +def _judge(protocol, entry, statistic): + """Return (statistic value or None, inconclusive verdict or None).""" + if entry is None: + return None, "inconclusive_missing" + if isinstance(entry, dict) and entry.get("not_applicable") is True: + return None, "not_applicable" + samples, expected = _samples(entry) + if samples is None or expected is None: + return None, "inconclusive_missing" + if len(samples) < expected: + return None, "inconclusive_coverage" + if len(samples) < _sample_floor(protocol, statistic): + return None, "inconclusive_samples" + return _statistic(samples, statistic), None + + +def evaluate(protocol, summary): + """Judge one workload's retained summary. Returns (diagnostics, verdict records). + + Comparative budgets are judged against each applicable baseline separately; there is no + aggregate. A candidate/baseline difference smaller than inconclusive_factor times the + observer's own cost for that metric is inconclusive rather than a pass or a fail. + """ + if not isinstance(summary, dict): + return ["summary_not_object"], [] + if summary.get("protocol_revision") != protocol["revision"]: + return ["summary_revision_mismatch"], [] + workloads = {w["id"]: w for w in protocol["workloads"]} + workload = workloads.get(summary.get("workload")) + if workload is None: + return ["summary_workload_unknown"], [] + lanes = summary.get("lanes") + if not isinstance(lanes, dict) or any(lane not in workload["lanes"] for lane in lanes): + return ["summary_lane_unknown"], [] + observer = summary.get("observer_cost") if isinstance(summary.get("observer_cost"), dict) else {} + factor = protocol["observer"]["inconclusive_factor"] + candidate = lanes.get("candidate") if isinstance(lanes.get("candidate"), dict) else {} + records = [] + + for budget in protocol["budgets"]["comparative"]: + metric, statistic = budget["metric"], budget["statistic"] + cand_value, cand_problem = _judge(protocol, candidate.get(metric), statistic) + for baseline in (lane for lane in workload["lanes"] if lane != "candidate"): + record = {"workload": workload["id"], "metric": metric, "statistic": statistic, + "baseline": baseline, "max_ratio": budget["max_ratio"]} + base_entry = lanes.get(baseline, {}).get(metric) if isinstance(lanes.get(baseline), dict) else None + base_value, base_problem = _judge(protocol, base_entry, statistic) + problem = cand_problem or base_problem + cost = observer.get(metric) + if problem is None and not (_number(cost) and cost >= 0): + problem = "inconclusive_observer" + if problem is None and base_value <= 0: + problem = "inconclusive_missing" + if problem is None: + record["ratio"] = cand_value / base_value + if abs(cand_value - base_value) < factor * cost: + problem = "inconclusive_observer" + record["verdict"] = problem or ("pass" if record["ratio"] <= budget["max_ratio"] else "fail") + records.append(record) + + for budget in protocol["budgets"]["absolute"]: + if budget.get("workload") not in (None, workload["id"]): + continue + metric, statistic = budget["metric"], budget["statistic"] + record = {"workload": workload["id"], "metric": metric, "statistic": statistic, "max": budget["max"]} + value, problem = _judge(protocol, candidate.get(metric), statistic) + if problem is None: + record["value"] = value + within = value <= budget["max"] if budget["inclusive"] else value < budget["max"] + problem = "pass" if within else "fail" + record["verdict"] = problem + records.append(record) + return [], records + + +def _load(path, unreadable): + try: + return json.loads(Path(path).read_text()), None + except (OSError, ValueError, UnicodeDecodeError): + return None, unreadable + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--protocol", default=str(DEFAULT_PROTOCOL)) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--schedule", action="store_true") + mode.add_argument("--evaluate", metavar="SUMMARY") + args = parser.parse_args(argv) + + protocol, problem = _load(args.protocol, "protocol_unreadable") + diagnostics = [problem] if problem else validate_protocol(protocol) + if diagnostics: + print(json.dumps({"status": "invalid", "diagnostics": diagnostics})) + return 2 + if args.schedule: + print(json.dumps({"protocol": PROTOCOL_ID, "revision": PROTOCOL_REVISION, + "orders": protocol["schedule"]["orders"]}, indent=2)) + return 0 + if args.evaluate: + summary, problem = _load(args.evaluate, "summary_unreadable") + diagnostics, records = ([problem], []) if problem else evaluate(protocol, summary) + if diagnostics: + print(json.dumps({"status": "invalid", "diagnostics": diagnostics})) + return 2 + failed = any(record["verdict"] != "pass" and record["verdict"] != "not_applicable" for record in records) + print(json.dumps({"status": "judged", "accepted": not failed, "verdicts": records}, indent=2)) + return 1 if failed else 0 + print(json.dumps({"status": "valid", "protocol": PROTOCOL_ID, "revision": PROTOCOL_REVISION, + "lanes": len(protocol["lanes"]), "rounds": len(protocol["schedule"]["orders"])})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/python/test_application_benchmark_protocol.py b/tests/python/test_application_benchmark_protocol.py new file mode 100644 index 000000000..67339d54b --- /dev/null +++ b/tests/python/test_application_benchmark_protocol.py @@ -0,0 +1,297 @@ +"""Offline controls for the frozen matched application benchmark protocol; no live runs.""" +import contextlib +import copy +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "scripts/check-application-benchmark-protocol.py" +SPEC = importlib.util.spec_from_file_location("application_benchmark_protocol", SOURCE) +protocol_check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(protocol_check) + +PROTOCOL = json.loads((ROOT / "scripts/application-benchmark-protocol.json").read_text()) +PROTOCOL_DOC = (ROOT / "docs/reference/application-benchmark-protocol.md").read_text() +PERFORMANCE_DOC = (ROOT / "docs/performance.md").read_text() +MARKER = "PRIVATE-MARKER-9f2c" + + +def mutated(change): + doc = copy.deepcopy(PROTOCOL) + change(doc) + return doc + + +def lane(doc, lane_id): + return next(item for item in doc["lanes"] if item["id"] == lane_id) + + +def budget(doc, kind, metric): + return next(item for item in doc["budgets"][kind] if item["metric"] == metric) + + +def entry(values, expected=None): + return {"samples": list(values), "expected": len(values) if expected is None else expected} + + +def small_summary(candidate_scale=0.5, runs=10, tail=60): + """A complete small-fixture summary where the candidate is `candidate_scale` of each baseline.""" + baselines = [item for item in PROTOCOL["workloads"][1]["lanes"] if item != "candidate"] + base = {"steady_physical_footprint_bytes": 1000.0, "cpu_seconds_per_task": 10.0, + "warm_startup_seconds": 20.0, "warm_restart_seconds": 10.0} + lanes = {item: {metric: entry([value] * runs) for metric, value in base.items()} for item in baselines} + lanes["candidate"] = {metric: entry([value * candidate_scale] * runs) for metric, value in base.items()} + lanes["candidate"].update({ + "info_latency_ms": entry([50] * tail), + "warm_admission_ms": entry([100] * tail), + "cached_status_1000_objects_ms": entry([100] * tail), + "idle_control_plane_core_fraction": entry([0.005] * runs), + "watcher_ack_ms": entry([100] * tail), + }) + return {"protocol_revision": 1, "workload": "small-fixture", + "observer_cost": {metric: 0.0 for metric in base}, "lanes": lanes} + + +def verdicts(records, metric, baseline=None): + return [record["verdict"] for record in records + if record["metric"] == metric and record.get("baseline", baseline) == baseline] + + +class FrozenProtocolTest(unittest.TestCase): + def test_frozen_protocol_is_valid(self): + self.assertEqual(protocol_check.validate_protocol(PROTOCOL), []) + + def test_frozen_schedule_is_the_williams_design_for_the_frozen_lanes(self): + lanes = [item["id"] for item in PROTOCOL["lanes"]] + self.assertEqual(PROTOCOL["schedule"]["orders"], protocol_check.williams_orders(lanes)) + self.assertEqual(PROTOCOL["schedule"]["rounds_per_workload"], len(PROTOCOL["schedule"]["orders"])) + self.assertGreaterEqual(len(PROTOCOL["schedule"]["orders"]), protocol_check.MIN_ROTATED_LIFECYCLE_RUNS) + self.assertEqual(PROTOCOL["samples"]["min_lifecycle_runs_per_lane"], len(PROTOCOL["schedule"]["orders"])) + + def test_williams_design_balances_position_and_carryover(self): + for count in range(2, 9): + orders = protocol_check.williams_orders([f"lane{index}" for index in range(count)]) + self.assertEqual(len(orders), count * (2 if count % 2 else 1)) + self.assertTrue(protocol_check.position_balanced(orders), count) + self.assertTrue(protocol_check.carryover_balanced(orders), count) + + def test_p95_sample_floor_is_the_distribution_free_bound(self): + required = protocol_check.minimum_order_statistic_samples(0.95, 0.95) + self.assertEqual(required, 59) + self.assertLess(1 - 0.95 ** (required - 1), 0.95) + self.assertGreaterEqual(1 - 0.95 ** required, 0.95) + self.assertGreaterEqual(PROTOCOL["samples"]["p95_min_samples"], required) + + def test_nearest_rank_and_median(self): + self.assertEqual(protocol_check.nearest_rank(list(range(1, 11)), 0.95), 10) + self.assertEqual(protocol_check.nearest_rank(list(range(61, 0, -1)), 0.95), 58) + self.assertEqual(protocol_check.nearest_rank(list(range(1, 61)), 0.95), 57) + self.assertEqual(protocol_check.median([3, 1, 2]), 2) + self.assertEqual(protocol_check.median([4, 1, 3, 2]), 2.5) + + def test_documents_carry_every_frozen_artifact_and_budget(self): + for item in PROTOCOL["lanes"]: + for field in ("commit", "asset", "asset_sha256", "tag"): + if field in item: + self.assertIn(item[field], PROTOCOL_DOC, (item["id"], field)) + host = PROTOCOL["host"] + for value in (host["cpu"], host["os"], host["docker_cli"], host["compose"], + host["orbstack"]["version"], host["orbstack"]["build"]): + self.assertIn(str(value), PROTOCOL_DOC) + for item in PROTOCOL["budgets"]["comparative"]: + self.assertIn(f"`{item['metric']}`", PROTOCOL_DOC) + self.assertIn(f"{item['max_ratio']:.2f}", PROTOCOL_DOC) + for item in PROTOCOL["budgets"]["absolute"]: + self.assertIn(f"`{item['metric']}`", PROTOCOL_DOC) + self.assertIn(str(PROTOCOL["samples"]["p95_min_samples"]), PROTOCOL_DOC) + limit = PROTOCOL["known_limits"][0] + for value in (limit["issue"], "#313", f"{limit['warm_up_seconds']} s", f"{limit['witness_seconds']} s", + f"{limit['witness_target_seconds']} s", "HACK-1169"): + self.assertIn(value, PERFORMANCE_DOC) + self.assertIn("application-benchmark-protocol.md", PERFORMANCE_DOC) + + +class NegativeProtocolControls(unittest.TestCase): + CASES = { + "comparative_budget_weakened": lambda d: budget(d, "comparative", "steady_physical_footprint_bytes") + .update(max_ratio=0.71), + "absolute_budget_weakened": lambda d: budget(d, "absolute", "idle_control_plane_core_fraction") + .update(inclusive=True), + "comparative_budget_missing": lambda d: d["budgets"]["comparative"].pop(), + "absolute_budget_missing": lambda d: d["budgets"]["absolute"].pop(), + "statistic_unknown": lambda d: budget(d, "absolute", "info_latency_ms").update(statistic="mean"), + "artifact_commit_malformed": lambda d: lane(d, "candidate").update(commit="28de2f89"), + "artifact_digest_malformed": lambda d: lane(d, "hack-4.2.1").update(asset_sha256="0" * 63), + "artifact_version_malformed": lambda d: lane(d, "hack-4.1.1").update(tag="v4.1.2"), + "p95_samples_insufficient": lambda d: d["samples"].update(p95_min_samples=58), + "observer_policy_missing": lambda d: d["observer"].update(inconclusive_factor=0.5), + "observer_policy_missing:window": lambda d: d["observer"].update(control_window_seconds=599), + "cache_class_missing": lambda d: d["cache_classes"].update(budget_class="cold-cache"), + "workload_lane_unknown": lambda d: d["workloads"][0]["lanes"].append("hack-3.0.0"), + "host_incomplete": lambda d: d["host"]["orbstack"].pop("build"), + "protocol_revision_mismatch": lambda d: d.update(revision=2), + "known_limit_malformed": lambda d: d["known_limits"][0].pop("revisit"), + "lane_id_duplicate": lambda d: d["lanes"].append(copy.deepcopy(d["lanes"][0])), + } + + def test_each_weakening_is_refused_with_its_closed_code(self): + for code, change in self.CASES.items(): + with self.subTest(code=code): + found = protocol_check.validate_protocol(mutated(change)) + self.assertIn(code.split(":")[0], found) + self.assertTrue(set(found) <= protocol_check.DIAGNOSTICS) + + def test_dropping_a_required_baseline_is_refused(self): + def drop(doc): + doc["lanes"] = [item for item in doc["lanes"] if item["id"] != "hack-4.2.0"] + doc["schedule"]["orders"] = protocol_check.williams_orders([item["id"] for item in doc["lanes"]]) + for workload in doc["workloads"]: + workload["lanes"].remove("hack-4.2.0") + self.assertIn("baseline_lane_missing", protocol_check.validate_protocol(mutated(drop))) + + def test_unbalanced_or_short_rotation_is_refused(self): + def swap(doc): + order = doc["schedule"]["orders"][0] + order[0], order[1] = order[1], order[0] + self.assertIn("schedule_position_unbalanced", protocol_check.validate_protocol(mutated(swap))) + repeated = mutated(lambda d: d["schedule"].update(orders=[d["schedule"]["orders"][0]] * 10)) + found = protocol_check.validate_protocol(repeated) + self.assertIn("schedule_position_unbalanced", found) + self.assertIn("schedule_carryover_unbalanced", found) + short = mutated(lambda d: d["schedule"].update(orders=d["schedule"]["orders"][:4])) + self.assertIn("schedule_rounds_below_minimum", protocol_check.validate_protocol(short)) + floor = mutated(lambda d: d["samples"].update(min_lifecycle_runs_per_lane=4)) + self.assertIn("schedule_rounds_below_minimum", protocol_check.validate_protocol(floor)) + more_than_scheduled = mutated(lambda d: d["samples"].update(min_lifecycle_runs_per_lane=11)) + self.assertIn("schedule_rounds_below_minimum", protocol_check.validate_protocol(more_than_scheduled)) + missing_lane = mutated(lambda d: d["schedule"]["orders"][3].pop()) + self.assertIn("schedule_lane_set_mismatch", protocol_check.validate_protocol(missing_lane)) + + def test_diagnostics_never_echo_input_values(self): + doc = mutated(lambda d: lane(d, "candidate").update(commit=MARKER, asset_sha256=MARKER)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / f"{MARKER}.json" + path.write_text(json.dumps(doc)) + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = protocol_check.main(["--protocol", str(path)]) + self.assertEqual(code, 2) + self.assertNotIn(MARKER, out.getvalue()) + self.assertEqual(json.loads(out.getvalue())["diagnostics"], + ["artifact_commit_malformed", "artifact_digest_malformed"]) + path.write_text("{not json " + MARKER) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(protocol_check.main(["--protocol", str(path)]), 2) + self.assertEqual(json.loads(out.getvalue()), {"status": "invalid", "diagnostics": ["protocol_unreadable"]}) + + +class BudgetEvaluationTest(unittest.TestCase): + def judge(self, summary): + diagnostics, records = protocol_check.evaluate(PROTOCOL, summary) + self.assertEqual(diagnostics, []) + self.assertTrue({record["verdict"] for record in records} <= protocol_check.VERDICTS) + return records + + def test_complete_improved_cohort_passes_every_budget_against_every_baseline(self): + records = self.judge(small_summary()) + self.assertEqual({record["verdict"] for record in records}, {"pass"}) + comparative = [record for record in records if "baseline" in record] + self.assertEqual(len(comparative), 4 * 4) + self.assertEqual(len(records) - len(comparative), 5) + + def test_boundary_ratios_follow_the_frozen_thresholds(self): + records = self.judge(small_summary(candidate_scale=0.75)) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "compose")), {"pass"}) + self.assertEqual(set(verdicts(records, "steady_physical_footprint_bytes", "compose")), {"fail"}) + self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"pass"}) + + def test_each_baseline_is_judged_separately_without_aggregation(self): + summary = small_summary() + summary["lanes"]["hack-4.2.1"]["warm_startup_seconds"] = entry([5.0] * 10) + records = self.judge(summary) + self.assertEqual(verdicts(records, "warm_startup_seconds", "hack-4.2.1"), ["fail"]) + self.assertEqual(verdicts(records, "warm_startup_seconds", "compose"), ["pass"]) + + def test_too_few_rotated_runs_or_tail_samples_are_inconclusive(self): + records = self.judge(small_summary(runs=9)) + self.assertEqual(set(verdicts(records, "warm_restart_seconds", "compose")), {"inconclusive_samples"}) + records = self.judge(small_summary(tail=59)) + self.assertEqual(verdicts(records, "info_latency_ms"), ["inconclusive_samples"]) + self.assertEqual(verdicts(records, "watcher_ack_ms"), ["inconclusive_samples"]) + + def test_missing_planned_samples_are_inconclusive_coverage(self): + summary = small_summary() + summary["lanes"]["compose"]["cpu_seconds_per_task"]["expected"] = 11 + records = self.judge(summary) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_coverage"]) + summary["lanes"]["compose"]["cpu_seconds_per_task"]["expected"] = None + records = self.judge(summary) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_missing"]) + + def test_difference_within_observer_cost_is_inconclusive(self): + summary = small_summary() + summary["observer_cost"]["cpu_seconds_per_task"] = 2.6 + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "compose")), {"inconclusive_observer"}) + summary["observer_cost"].pop("warm_startup_seconds") + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"inconclusive_observer"}) + + def test_absolute_ceilings_respect_inclusivity(self): + summary = small_summary() + summary["lanes"]["candidate"]["info_latency_ms"] = entry([100] * 60) + summary["lanes"]["candidate"]["idle_control_plane_core_fraction"] = entry([0.01] * 10) + records = self.judge(summary) + self.assertEqual(verdicts(records, "info_latency_ms"), ["pass"]) + self.assertEqual(verdicts(records, "idle_control_plane_core_fraction"), ["fail"]) + summary["lanes"]["candidate"]["info_latency_ms"] = entry([50] * 57 + [101] * 3) + self.assertEqual(verdicts(self.judge(summary), "info_latency_ms"), ["pass"]) + summary["lanes"]["candidate"]["info_latency_ms"] = entry([50] * 56 + [101] * 4) + self.assertEqual(verdicts(self.judge(summary), "info_latency_ms"), ["fail"]) + + def test_not_applicable_and_missing_lanes_are_never_passes(self): + summary = small_summary() + summary["lanes"]["hack-4.1.1"]["warm_startup_seconds"] = {"not_applicable": True} + summary["lanes"].pop("hack-4.2.0") + records = self.judge(summary) + self.assertEqual(verdicts(records, "warm_startup_seconds", "hack-4.1.1"), ["not_applicable"]) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "hack-4.2.0")), {"inconclusive_missing"}) + + def test_workload_scoped_budgets_only_apply_to_their_workload(self): + summary = small_summary() + summary["workload"] = "event-agent" + records = self.judge(summary) + metrics = {record["metric"] for record in records if "baseline" not in record} + self.assertEqual(metrics, {"info_latency_ms", "warm_admission_ms"}) + + def test_malformed_summaries_are_refused_with_closed_codes(self): + cases = { + "summary_not_object": [], + "summary_revision_mismatch": {**small_summary(), "protocol_revision": 2}, + "summary_workload_unknown": {**small_summary(), "workload": MARKER}, + "summary_lane_unknown": {**small_summary(), "lanes": {MARKER: {}}}, + } + for code, summary in cases.items(): + with self.subTest(code=code): + self.assertEqual(protocol_check.evaluate(PROTOCOL, summary), ([code], [])) + + def test_cli_exit_codes(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "summary.json" + for scale, expected in ((0.5, 0), (0.9, 1)): + path.write_text(json.dumps(small_summary(candidate_scale=scale))) + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(protocol_check.main(["--evaluate", str(path)]), expected) + self.assertEqual(json.loads(out.getvalue())["accepted"], expected == 0) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(protocol_check.main([]), 0) + self.assertEqual(protocol_check.main(["--schedule"]), 0) + + +if __name__ == "__main__": + unittest.main() From 89421c8213d4473979920cba4f9bfcc47dae929a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:15:03 -0400 Subject: [PATCH 2/4] fix(perf): refuse not-applicable candidates and frozen-protocol weakenings HACK-1169 rework of the application benchmark protocol checker. Evaluation: the candidate can never be not_applicable (exit 2, summary_candidate_not_applicable). A not_applicable record is exactly {not_applicable: true, reason} and cannot carry samples. A comparative budget whose baselines are all not_applicable gets an inconclusive_no_baseline verdict, so it is never accepted. Only warm, single-instance summaries are judged. Samples must be non-negative, expected a positive count, and flagged and failed lifecycles are accounted: any flag, unaccounted admission or failed baseline lifecycle is inconclusive_admission, and a failed candidate lifecycle fails. Every malformed summary or protocol value maps to a closed code instead of a traceback. Protocol validation now pins each budget's statistic, workload scope (only the watcher budget is limited to the small fixture, as in Gate C; idle CPU and cached status now apply to both workloads), the observer factor of 2, p95 method and confidence, rounds_per_workload, the Williams design, workload shapes, the required baselines in every workload, the Compose lane against the frozen host, the Gate C phases (adding restore) and the 1/8/32 branch-instance and 1/32 MCP-client scale points. Co-Authored-By: Claude Opus 5.5 --- .../application-benchmark-protocol.md | 97 +++- scripts/application-benchmark-protocol.json | 12 +- .../check-application-benchmark-protocol.py | 483 +++++++++++++----- .../test_application_benchmark_protocol.py | 334 +++++++++++- 4 files changed, 764 insertions(+), 162 deletions(-) diff --git a/docs/reference/application-benchmark-protocol.md b/docs/reference/application-benchmark-protocol.md index 3489df1ea..da2022937 100644 --- a/docs/reference/application-benchmark-protocol.md +++ b/docs/reference/application-benchmark-protocol.md @@ -73,8 +73,9 @@ revision. ## Workloads -Two workloads run in every lane. A cohort records each workload's pin, and every lane -in that cohort uses the same pin. +Two workloads run in every lane, and each workload is compared against every required +baseline. A cohort records each workload's pin, and every lane in that cohort uses the +same pin. **Event Agent.** The actual application configuration, not a reduced copy. The cohort records the application commit and a digest of its configuration. Its expected shape is @@ -100,8 +101,9 @@ A lifecycle counts as correct only when: - a source edit is served within the watcher acknowledgement window. A lane that cannot run a workload correctly records a failure for that lifecycle. It -is not dropped and its timings are not used. A lane is `not_applicable` only when the -lane cannot express the workload at all, and that reason is recorded. +is not dropped and its timings are not used. A baseline lane is `not_applicable` for a +metric only when it cannot express the workload at all, and that reason is recorded. +The candidate is never `not_applicable`: it is the subject of every budget. ## Cache classes @@ -119,16 +121,29 @@ Classes are never mixed in one statistic. One lifecycle is: warm startup to full readiness and correctness, the workload's check and build/test task, an exec, a source reload, a restart to full readiness and data -readback, a steady idle window, sustained application load, shutdown and cleanup -readback. Each phase is timed and reported on its own: +readback, a steady idle window, sustained application load, a retaining shutdown, a +restore (an ordinary start after that shutdown, to full readiness with the per-run data +marker or row read back), and a final shutdown with cleanup readback. Each phase is +timed and reported on its own: -`cold-startup`, `warm-startup`, `restart`, `reload`, `build-test`, `exec`, +`cold-startup`, `warm-startup`, `restart`, `restore`, `reload`, `build-test`, `exec`, `steady-idle`, `sustained-load`, `shutdown`, `cleanup`. Readiness means the workload's correctness checks pass, not that a process started or a port opened. Cleanup readback inventories owned containers, processes, sockets and allocated disk, and keeps persistent data. +## Scale points + +Each lane is also measured at 1, 8 and 32 branch instances of the small fixture, and +with 1 and 32 MCP clients connected, using the same accounting, order and sample counts. +A lane that has no MCP server records `not_applicable` with its reason. Revision 1 +judges budgets only at the single-instance matched lifecycle (`{"branch_instances": 1}`). +It sets no separate numerical budget for the other scale points: their results are +reported per lane, and every regression at any scale point is reported and blocks +acceptance under [Admission and failures](#admission-and-failures). The evaluator +refuses a summary for another scale point rather than judge it. + ## Order and repetitions Each workload runs 10 rounds. Each round runs every lane once, in the order below. This @@ -165,6 +180,11 @@ Print the same schedule with 59 samples is the smallest count that reaches 95%. - A statistic with fewer samples than planned, or fewer than these floors, is inconclusive. It is never a pass. +- A statistic is computed only from admitted samples. If any planned sample was + flagged, or its lifecycle failed, or the summary does not account for flags and + failures, the statistic is inconclusive; a failed candidate lifecycle is a failure. + Samples are never rerun or replaced to fill a gap: a summary that accounts for more + samples than were planned is refused. ## Measurement boundary @@ -228,31 +248,63 @@ document. | --- | --- | --- | --- | | `info_latency_ms` | p95 | ≤ 100 ms | both | | `warm_admission_ms` | p95 | ≤ 250 ms | both | -| `cached_status_1000_objects_ms` | p95 | ≤ 150 ms, with 1,000 cached objects | small fixture | -| `idle_control_plane_core_fraction` | mean over 600 s | < 1% of one core | small fixture | +| `cached_status_1000_objects_ms` | p95 | ≤ 150 ms, with 1,000 cached objects | both | +| `idle_control_plane_core_fraction` | mean over 600 s | < 1% of one core | both | | `watcher_ack_ms` | p95 | ≤ 200 ms | small fixture | +Only the watcher budget is limited to the small fixture, as in the program acceptance +document. A later revision may apply a budget to more workloads, but never fewer. The +checker also refuses a changed statistic, a raised ceiling, a lower observer factor, a +changed p95 method or confidence, a workload that drops a required baseline, a changed +workload shape, a missing phase, a missing scale point and a Compose lane that differs +from the frozen host engine. + These are acceptance thresholds, not measured achievements. ## Judging a summary -`--evaluate SUMMARY` reads one workload's retained summary and returns one verdict per -budget and baseline: `pass`, `fail`, `not_applicable`, `inconclusive_coverage`, -`inconclusive_samples`, `inconclusive_observer` or `inconclusive_missing`. It exits 0 -only when every verdict is `pass` or `not_applicable`. It exits 1 when any verdict -fails or is inconclusive, and 2 when the protocol or summary is invalid. Diagnostics -are closed codes and never repeat input values. +`--evaluate SUMMARY` reads one workload's retained summary for the `warm` class at the +single-instance scale point. It returns one verdict per budget and baseline: -A summary has this shape: +| Verdict | Meaning | +| --- | --- | +| `pass` | the statistic meets the budget | +| `fail` | the statistic misses the budget, or a candidate lifecycle failed | +| `not_applicable` | a baseline cannot express the workload, with a recorded reason | +| `inconclusive_coverage` | fewer admitted samples than planned | +| `inconclusive_samples` | fewer samples than the protocol's floor | +| `inconclusive_admission` | a sample was flagged, a baseline lifecycle failed, or flags and failures were not accounted | +| `inconclusive_observer` | the difference is within 2 times the observer's cost, or that cost is not reported | +| `inconclusive_missing` | the lane, metric or planned count is missing | +| `inconclusive_no_baseline` | every baseline is `not_applicable`, so there is no comparison | + +It exits 0 only when every verdict is `pass` or `not_applicable`, and a comparative +budget always has at least one applicable baseline or an `inconclusive_no_baseline` +record. It exits 1 when any verdict fails or is inconclusive. It exits 2, without +judging, when the protocol or summary is invalid: for example a summary for another +cache class or scale point, a candidate marked `not_applicable`, a `not_applicable` +record that carries samples, a negative sample, a planned count that is not a positive +integer, or a value of the wrong type. Diagnostics are closed codes and never repeat +input values. + +A summary has this shape. `samples` lists only admitted samples; `flagged` and `failed` +count the planned samples left out, so `samples` plus both counts never exceeds +`expected`: ```json { "protocol_revision": 1, "workload": "small-fixture", + "cache_class": "warm", + "scale": { "branch_instances": 1 }, "observer_cost": { "cpu_seconds_per_task": 0.02 }, "lanes": { - "candidate": { "cpu_seconds_per_task": { "samples": [1.2], "expected": 10 } }, - "compose": { "cpu_seconds_per_task": { "not_applicable": true } } + "candidate": { + "cpu_seconds_per_task": { "samples": [1.2], "expected": 10, "flagged": 0, "failed": 0 } + }, + "compose": { + "cpu_seconds_per_task": { "not_applicable": true, "reason": "no matched task" } + } } } ``` @@ -272,6 +324,9 @@ python3 -m unittest discover -s tests/python -p test_application_benchmark_proto ``` The unit tests check the frozen file, the rotation balance, the p95 sample floor and -that this page carries every frozen artifact and budget. They also prove that a -weakened budget, a bad digest, an unbalanced order, too few samples or a missing -observer policy is refused. They run offline, locally and in CI. +that this page carries every frozen artifact, budget, phase and verdict. They also +prove that each weakening listed under [Budgets](#budgets), a bad digest, an unbalanced +order, too few samples and a missing observer policy is refused; that missing, +flagged, `not_applicable`-only or cold-cache data is never accepted; and that any +malformed protocol or summary value yields a closed code rather than a traceback. They +run offline, locally and in CI. diff --git a/scripts/application-benchmark-protocol.json b/scripts/application-benchmark-protocol.json index 1823f14b2..34e1e7c83 100644 --- a/scripts/application-benchmark-protocol.json +++ b/scripts/application-benchmark-protocol.json @@ -118,6 +118,7 @@ "cold-startup", "warm-startup", "restart", + "restore", "reload", "build-test", "exec", @@ -126,6 +127,11 @@ "shutdown", "cleanup" ], + "scale_points": { + "branch_instances": [1, 8, 32], + "mcp_clients": [1, 32], + "judged": { "branch_instances": 1 } + }, "cache_classes": { "budget_class": "warm", "warm": "engine or VM running; images, dependency and build caches populated by a prior correct lifecycle of the same lane; application volumes retained", @@ -195,15 +201,13 @@ "metric": "cached_status_1000_objects_ms", "statistic": "p95", "max": 150, - "inclusive": true, - "workload": "small-fixture" + "inclusive": true }, { "metric": "idle_control_plane_core_fraction", "statistic": "window_mean", "max": 0.01, - "inclusive": false, - "workload": "small-fixture" + "inclusive": false }, { "metric": "watcher_ack_ms", diff --git a/scripts/check-application-benchmark-protocol.py b/scripts/check-application-benchmark-protocol.py index 05ae20cd4..c5d9edb2b 100644 --- a/scripts/check-application-benchmark-protocol.py +++ b/scripts/check-application-benchmark-protocol.py @@ -9,8 +9,11 @@ --evaluate SUMMARY judges a retained cohort summary against the frozen budgets. Diagnostics are closed codes from DIAGNOSTICS; they never echo input values or paths. -A budget verdict is one of VERDICTS. Anything short of complete, sufficient, admitted -samples is inconclusive, never a pass. See docs/reference/application-benchmark-protocol.md. +A budget verdict is one of VERDICTS. A statistic is judged only from complete, +sufficient, admitted samples: missing, short, flagged or unaccounted samples are +inconclusive, never a pass. The candidate is never `not_applicable`, and a comparative +budget with no applicable baseline is inconclusive. Malformed input is refused with a +closed code (exit 2), never a traceback. See docs/reference/application-benchmark-protocol.md. """ import argparse import json @@ -28,25 +31,46 @@ VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$") LANE_ID = re.compile(r"^[a-z0-9][a-z0-9.-]{0,31}$") -# Program floors (Linear program acceptance, Gate C). A protocol revision may be -# stricter than these, never weaker. +# Program floors (Linear program acceptance, Gate C) and the frozen revision-1 choices. +# A protocol revision may be stricter than these, never weaker. MIN_ROTATED_LIFECYCLE_RUNS = 5 REQUIRED_BASELINES = ("compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +# metric: (ceiling ratio, frozen statistic). A ratio at the ceiling passes. COMPARATIVE_CEILINGS = { - "steady_physical_footprint_bytes": 0.70, - "cpu_seconds_per_task": 0.75, - "warm_startup_seconds": 0.80, - "warm_restart_seconds": 0.80, + "steady_physical_footprint_bytes": (0.70, "median"), + "cpu_seconds_per_task": (0.75, "median"), + "warm_startup_seconds": (0.80, "median"), + "warm_restart_seconds": (0.80, "median"), } -# (ceiling, whether the ceiling itself passes): "p95 <= 100 ms" versus "< 1% of one core". +# metric: (ceiling, whether the ceiling itself passes, frozen statistic, frozen scope). +# "p95 <= 100 ms" versus "< 1% of one core". A scope of None means every workload. A +# scoped budget may be broadened to every workload, but no budget may be narrowed. ABSOLUTE_CEILINGS = { - "info_latency_ms": (100, True), - "cached_status_1000_objects_ms": (150, True), - "warm_admission_ms": (250, True), - "idle_control_plane_core_fraction": (0.01, False), - "watcher_ack_ms": (200, True), + "info_latency_ms": (100, True, "p95", None), + "cached_status_1000_objects_ms": (150, True, "p95", None), + "warm_admission_ms": (250, True, "p95", None), + "idle_control_plane_core_fraction": (0.01, False, "window_mean", None), + "watcher_ack_ms": (200, True, "p95", "small-fixture"), } STATISTICS = ("median", "p95", "window_mean") +P95_QUANTILE = 0.95 +P95_MIN_CONFIDENCE = 0.95 +P95_METHOD = "nearest-rank" +SCHEDULE_DESIGN = "williams" +MIN_OBSERVER_FACTOR = 2 +MIN_OBSERVER_WINDOW_SECONDS = 600 +WORKLOAD_SHAPES = { + "event-agent": {"services": 14, "http_health_checks": 9, "initializers": 2}, + "small-fixture": {"services": 2, "http_health_checks": 1, "initializers": 1}, +} +# Gate C phases. "restore" is an ordinary up after a retaining down, with data readback. +REQUIRED_PHASES = ( + "cold-startup", "warm-startup", "restart", "restore", "reload", "build-test", "exec", + "steady-idle", "sustained-load", "shutdown", "cleanup", +) +# Gate C scale points. Revision 1 judges budgets at the single-instance lifecycle only. +SCALE_POINTS = {"branch_instances": (1, 8, 32), "mcp_clients": (1, 32)} +JUDGED_SCALE = {"branch_instances": 1} DIAGNOSTICS = frozenset({ "protocol_unreadable", @@ -56,6 +80,7 @@ "host_incomplete", "lanes_malformed", "lane_id_duplicate", + "compose_lane_host_mismatch", "candidate_lane_missing", "baseline_lane_missing", "artifact_commit_malformed", @@ -63,16 +88,25 @@ "artifact_version_malformed", "workloads_malformed", "workload_lane_unknown", + "workload_baseline_missing", + "workload_shape_mismatch", + "phases_mismatch", + "scale_points_mismatch", "cache_class_missing", + "schedule_design_mismatch", "schedule_lane_set_mismatch", "schedule_rounds_below_minimum", + "schedule_rounds_mismatch", "schedule_position_unbalanced", "schedule_carryover_unbalanced", "p95_samples_insufficient", + "p95_method_mismatch", "comparative_budget_missing", "comparative_budget_weakened", "absolute_budget_missing", "absolute_budget_weakened", + "budget_statistic_mismatch", + "budget_scope_narrowed", "statistic_unknown", "observer_policy_missing", "known_limit_malformed", @@ -80,7 +114,14 @@ "summary_not_object", "summary_revision_mismatch", "summary_workload_unknown", + "summary_cache_class_mismatch", + "summary_scale_unknown", + "summary_scale_not_judged", "summary_lane_unknown", + "summary_lane_malformed", + "summary_entry_malformed", + "summary_observer_malformed", + "summary_candidate_not_applicable", }) VERDICTS = frozenset({ @@ -89,9 +130,15 @@ "not_applicable", "inconclusive_coverage", "inconclusive_samples", + "inconclusive_admission", "inconclusive_observer", "inconclusive_missing", + "inconclusive_no_baseline", }) +ACCEPTING_VERDICTS = frozenset({"pass", "not_applicable"}) + +SAMPLE_ENTRY_KEYS = frozenset({"samples", "expected", "flagged", "failed"}) +NOT_APPLICABLE_KEYS = frozenset({"not_applicable", "reason"}) def williams_orders(lanes): @@ -167,25 +214,26 @@ def _number(value): return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) -def validate_protocol(doc): - """Return sorted closed diagnostic codes; empty means the protocol is valid.""" - if not isinstance(doc, dict): - return ["protocol_not_object"] - found = set() - if doc.get("protocol") != PROTOCOL_ID: - found.add("protocol_id_mismatch") - if doc.get("revision") != PROTOCOL_REVISION: - found.add("protocol_revision_mismatch") +def _count(value): + """A non-negative JSON integer; a boolean is not a count.""" + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _strings(value): + return isinstance(value, list) and all(isinstance(item, str) for item in value) - host = doc.get("host") - host_fields = ("id", "cpu", "logical_cpus", "memory_bytes", "os", "docker_cli", "compose", "orbstack") - if not isinstance(host, dict) or any(not host.get(field) for field in host_fields): + +def _check_host(host, found): + fields = ("id", "cpu", "logical_cpus", "memory_bytes", "os", "docker_cli", "compose", "orbstack") + if not isinstance(host, dict) or any(not host.get(field) for field in fields): found.add("host_incomplete") elif not isinstance(host["orbstack"], dict) or any( not host["orbstack"].get(field) for field in ("version", "build", "cpu", "memory_mib")): found.add("host_incomplete") - lanes = doc.get("lanes") + +def _check_lanes(lanes, host, found): + """Return the well-formed lane ids, in order.""" lane_ids = [] if not isinstance(lanes, list) or not lanes or not all(isinstance(lane, dict) for lane in lanes): found.add("lanes_malformed") @@ -204,93 +252,200 @@ def validate_protocol(doc): if not isinstance(lane.get("asset_sha256"), str) or not SHA256.fullmatch(lane["asset_sha256"]): found.add("artifact_digest_malformed") if not isinstance(lane.get("version"), str) or not VERSION.fullmatch(lane["version"]) \ - or lane.get("tag") != "v" + str(lane.get("version")): + or lane.get("tag") != "v" + lane["version"]: found.add("artifact_version_malformed") elif lane.get("kind") == "compose": if not lane.get("engine") or not lane.get("compose_version"): found.add("lanes_malformed") + # The direct Compose lane is the frozen host's own engine and Compose. + elif not isinstance(host, dict) or lane["engine"] != "orbstack" \ + or lane["compose_version"] != host.get("compose"): + found.add("compose_lane_host_mismatch") else: found.add("lanes_malformed") if "candidate" not in lane_ids: found.add("candidate_lane_missing") if any(baseline not in lane_ids for baseline in REQUIRED_BASELINES): found.add("baseline_lane_missing") + return lane_ids + - workloads = doc.get("workloads") - if not isinstance(workloads, list) or len(workloads) < 2 or not all(isinstance(w, dict) for w in workloads): +def _check_workloads(workloads, lane_ids, found): + """Return the workload ids when every workload has a distinct string id; otherwise [].""" + if not isinstance(workloads, list) or not all(isinstance(w, dict) for w in workloads): found.add("workloads_malformed") - workloads = [] - workload_ids = {w.get("id") for w in workloads} - if not {"event-agent", "small-fixture"} <= workload_ids: + return [] + ids = [w.get("id") for w in workloads] + if not _strings(ids) or len(set(ids)) != len(ids): + found.add("workloads_malformed") + ids = [] + if not set(WORKLOAD_SHAPES) <= set(ids): found.add("workloads_malformed") for workload in workloads: applicable = workload.get("lanes") - if not isinstance(applicable, list) or "candidate" not in applicable: + if not _strings(applicable) or len(set(applicable)) != len(applicable) or "candidate" not in applicable: found.add("workloads_malformed") - elif any(lane not in lane_ids for lane in applicable): - found.add("workload_lane_unknown") - if not workload.get("correctness"): + else: + if any(lane not in lane_ids for lane in applicable): + found.add("workload_lane_unknown") + # Every workload is compared against every required baseline; dropping one + # from a workload would silently drop that workload's comparison. + if any(baseline not in applicable for baseline in REQUIRED_BASELINES): + found.add("workload_baseline_missing") + if not _strings(workload.get("correctness")) or not workload["correctness"]: found.add("workloads_malformed") - - cache = doc.get("cache_classes") - if not isinstance(cache, dict) or not {"warm", "cold-capacity", "cold-cache"} <= set(cache) \ - or cache.get("budget_class") != "warm": - found.add("cache_class_missing") - - schedule = doc.get("schedule") - orders = schedule.get("orders") if isinstance(schedule, dict) else None - if not isinstance(orders, list) or not orders or not all(isinstance(order, list) for order in orders): + workload_id = workload.get("id") + shape = WORKLOAD_SHAPES.get(workload_id) if isinstance(workload_id, str) else None + if shape is not None and workload.get("shape") != shape: + found.add("workload_shape_mismatch") + return ids + + +def _check_phases_and_scale(doc, found): + phases = doc.get("phases") + if not _strings(phases) or len(set(phases)) != len(phases) or not set(REQUIRED_PHASES) <= set(phases): + found.add("phases_mismatch") + scale = doc.get("scale_points") + if not isinstance(scale, dict) or scale.get("judged") != JUDGED_SCALE: + found.add("scale_points_mismatch") + return + for kind, required in SCALE_POINTS.items(): + points = scale.get(kind) + if not isinstance(points, list) or not all(_count(point) and point > 0 for point in points) \ + or not set(required) <= set(points): + found.add("scale_points_mismatch") + + +def _check_schedule(doc, lane_ids, found): + schedule = doc.get("schedule") if isinstance(doc.get("schedule"), dict) else {} + if schedule.get("design") != SCHEDULE_DESIGN: + found.add("schedule_design_mismatch") + orders = schedule.get("orders") + if not isinstance(orders, list) or not orders or not all(_strings(order) for order in orders) \ + or any(sorted(order) != sorted(lane_ids) or len(set(order)) != len(order) for order in orders): found.add("schedule_lane_set_mismatch") - else: - if any(sorted(order) != sorted(lane_ids) or len(set(order)) != len(order) for order in orders): - found.add("schedule_lane_set_mismatch") - else: - planned = doc.get("samples", {}).get("min_lifecycle_runs_per_lane") \ - if isinstance(doc.get("samples"), dict) else None - if not isinstance(planned, int) or isinstance(planned, bool) \ - or planned < MIN_ROTATED_LIFECYCLE_RUNS or len(orders) < planned: - found.add("schedule_rounds_below_minimum") - if not position_balanced(orders): - found.add("schedule_position_unbalanced") - if not carryover_balanced(orders): - found.add("schedule_carryover_unbalanced") - - samples = doc.get("samples") - try: - required = minimum_order_statistic_samples(samples["p95_quantile"], samples["p95_bound_confidence"]) - if samples["p95_quantile"] != 0.95 or samples["p95_min_samples"] < required: - found.add("p95_samples_insufficient") - except (TypeError, KeyError, ValueError): + return + rounds = schedule.get("rounds_per_workload") + if not _count(rounds) or rounds != len(orders): + found.add("schedule_rounds_mismatch") + samples = doc.get("samples") if isinstance(doc.get("samples"), dict) else {} + planned = samples.get("min_lifecycle_runs_per_lane") + if not _count(planned) or planned < MIN_ROTATED_LIFECYCLE_RUNS or len(orders) < planned: + found.add("schedule_rounds_below_minimum") + if not position_balanced(orders): + found.add("schedule_position_unbalanced") + if not carryover_balanced(orders): + found.add("schedule_carryover_unbalanced") + + +def _check_samples(doc, found): + samples = doc.get("samples") if isinstance(doc.get("samples"), dict) else {} + quantile = samples.get("p95_quantile") + confidence = samples.get("p95_bound_confidence") + floor = samples.get("p95_min_samples") + if not _number(quantile) or quantile != P95_QUANTILE \ + or not _number(confidence) or not P95_MIN_CONFIDENCE <= confidence < 1 \ + or not _count(floor) or floor < minimum_order_statistic_samples(P95_QUANTILE, confidence): found.add("p95_samples_insufficient") + if samples.get("p95_method") != P95_METHOD: + found.add("p95_method_mismatch") + + +def _budget_index(items, kind, found): + """Index a budget list by metric; malformed or duplicate entries weaken the list.""" + if not isinstance(items, list): + found.add(f"{kind}_budget_missing") + return {} + index = {} + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("metric"), str) or item["metric"] in index: + found.add(f"{kind}_budget_weakened") + continue + index[item["metric"]] = item + return index + +def _check_budgets(doc, workload_ids, found): budgets = doc.get("budgets") if isinstance(doc.get("budgets"), dict) else {} - comparative = {b.get("metric"): b for b in budgets.get("comparative", []) if isinstance(b, dict)} - for metric, ceiling in COMPARATIVE_CEILINGS.items(): + comparative = _budget_index(budgets.get("comparative"), "comparative", found) + absolute = _budget_index(budgets.get("absolute"), "absolute", found) + # Structural checks for every listed budget, frozen or additional: the evaluator + # judges each listed budget, so each must be judgeable. + for budget in list(comparative.values()) + list(absolute.values()): + if budget.get("statistic") not in STATISTICS: + found.add("statistic_unknown") + for budget in comparative.values(): + if not _number(budget.get("max_ratio")): + found.add("comparative_budget_weakened") + for budget in absolute.values(): + if not _number(budget.get("max")) or not isinstance(budget.get("inclusive"), bool): + found.add("absolute_budget_weakened") + scope = budget.get("workload") + if scope is not None and scope not in workload_ids: + found.add("budget_scope_narrowed") + + for metric, (ceiling, statistic) in COMPARATIVE_CEILINGS.items(): budget = comparative.get(metric) if budget is None: found.add("comparative_budget_missing") - elif not _number(budget.get("max_ratio")) or budget["max_ratio"] > ceiling: + continue + if _number(budget.get("max_ratio")) and budget["max_ratio"] > ceiling: found.add("comparative_budget_weakened") - absolute = {b.get("metric"): b for b in budgets.get("absolute", []) if isinstance(b, dict)} - for metric, (ceiling, ceiling_inclusive) in ABSOLUTE_CEILINGS.items(): + if budget.get("statistic") in STATISTICS and budget["statistic"] != statistic: + found.add("budget_statistic_mismatch") + for metric, (ceiling, ceiling_inclusive, statistic, scope) in ABSOLUTE_CEILINGS.items(): budget = absolute.get(metric) if budget is None: found.add("absolute_budget_missing") - elif not _number(budget.get("max")) or not isinstance(budget.get("inclusive"), bool) \ - or budget["max"] > ceiling \ - or (budget["max"] == ceiling and budget["inclusive"] and not ceiling_inclusive): + continue + if _number(budget.get("max")) and isinstance(budget.get("inclusive"), bool) and ( + budget["max"] > ceiling + or (budget["max"] == ceiling and budget["inclusive"] and not ceiling_inclusive)): found.add("absolute_budget_weakened") - for budget in list(comparative.values()) + list(absolute.values()): - if budget.get("statistic") not in STATISTICS: - found.add("statistic_unknown") + if budget.get("statistic") in STATISTICS and budget["statistic"] != statistic: + found.add("budget_statistic_mismatch") + if budget.get("workload") not in (None, scope): + found.add("budget_scope_narrowed") + + +def validate_protocol(doc): + """Return sorted closed diagnostic codes; empty means the protocol is valid. + + Never raises on any JSON value: every malformed node maps to a closed code. + """ + if not isinstance(doc, dict): + return ["protocol_not_object"] + found = set() + if doc.get("protocol") != PROTOCOL_ID: + found.add("protocol_id_mismatch") + if not _count(doc.get("revision")) or doc["revision"] != PROTOCOL_REVISION: + found.add("protocol_revision_mismatch") + _check_host(doc.get("host"), found) + lane_ids = _check_lanes(doc.get("lanes"), doc.get("host"), found) + workload_ids = _check_workloads(doc.get("workloads"), lane_ids, found) + _check_phases_and_scale(doc, found) + + cache = doc.get("cache_classes") + if not isinstance(cache, dict) or not {"warm", "cold-capacity", "cold-cache"} <= set(cache) \ + or cache.get("budget_class") != "warm": + found.add("cache_class_missing") + + _check_schedule(doc, lane_ids, found) + _check_samples(doc, found) + _check_budgets(doc, workload_ids, found) observer = doc.get("observer") if not isinstance(observer, dict) or not _number(observer.get("inconclusive_factor")) \ - or observer["inconclusive_factor"] < 1 or not _number(observer.get("control_window_seconds")) \ - or observer["control_window_seconds"] < 600: + or observer["inconclusive_factor"] < MIN_OBSERVER_FACTOR \ + or not _number(observer.get("control_window_seconds")) \ + or observer["control_window_seconds"] < MIN_OBSERVER_WINDOW_SECONDS: found.add("observer_policy_missing") - for limit in doc.get("known_limits", []): + limits = doc.get("known_limits", []) + if not isinstance(limits, list): + found.add("known_limit_malformed") + limits = [] + for limit in limits: if not isinstance(limit, dict) or not all( _number(limit.get(field)) for field in ("warm_up_seconds", "witness_seconds", "witness_target_seconds")) \ or not limit.get("issue") or not limit.get("revisit"): @@ -298,19 +453,99 @@ def validate_protocol(doc): return sorted(found) -def _samples(entry): - if not isinstance(entry, dict) or not isinstance(entry.get("samples"), list) \ - or not all(_number(value) for value in entry["samples"]): - return None, None +def _entry_malformed(entry): + """True when a budget-metric entry is neither a sample record nor a not-applicable record. + + A sample record is {"samples": [non-negative numbers], "expected": positive int, + "flagged": count, "failed": count}; samples hold only admitted samples, and flagged + or failed lifecycles are counted, not listed. A not-applicable record is exactly + {"not_applicable": true, "reason": non-empty string}: it can never carry samples. + """ + if not isinstance(entry, dict): + return True + if "not_applicable" in entry: + return set(entry) != NOT_APPLICABLE_KEYS or entry["not_applicable"] is not True \ + or not isinstance(entry["reason"], str) or not entry["reason"].strip() + if not set(entry) <= SAMPLE_ENTRY_KEYS: + return True + samples = entry.get("samples") + if not isinstance(samples, list) or not all(_number(value) and value >= 0 for value in samples): + return True + if any(field in entry and not _count(entry[field]) for field in ("flagged", "failed")): + return True expected = entry.get("expected") - return entry["samples"], expected if isinstance(expected, int) and not isinstance(expected, bool) else None + if expected is None: + return False + if not _count(expected) or expected == 0: + return True + # More accounted samples than planned means unplanned reruns, which could select results. + return len(samples) + entry.get("flagged", 0) + entry.get("failed", 0) > expected + + +def _scale_known(protocol, scale): + if not isinstance(scale, dict) or len(scale) != 1: + return False + ((kind, count),) = scale.items() + points = protocol["scale_points"].get(kind) if kind in SCALE_POINTS else None + return isinstance(points, list) and _count(count) and count in points + + +def _budget_metrics(protocol): + return {budget["metric"] for kind in ("comparative", "absolute") for budget in protocol["budgets"][kind]} + + +def validate_summary(protocol, summary): + """Return (sorted closed diagnostics, workload); a summary with diagnostics is never judged. + + `protocol` must already be valid. Never raises on any JSON value. + """ + if not isinstance(summary, dict): + return ["summary_not_object"], None + if not _count(summary.get("protocol_revision")) or summary["protocol_revision"] != protocol["revision"]: + return ["summary_revision_mismatch"], None + workload_id = summary.get("workload") + workloads = {w["id"]: w for w in protocol["workloads"]} + workload = workloads.get(workload_id) if isinstance(workload_id, str) else None + if workload is None: + return ["summary_workload_unknown"], None + found = set() + # Budgets are frozen for the warm class only; other classes are reported separately. + if summary.get("cache_class") != protocol["cache_classes"]["budget_class"]: + found.add("summary_cache_class_mismatch") + if not _scale_known(protocol, summary.get("scale")): + found.add("summary_scale_unknown") + elif summary["scale"] != protocol["scale_points"]["judged"]: + found.add("summary_scale_not_judged") + + observer = summary.get("observer_cost", {}) + if not isinstance(observer, dict) or not all(_number(cost) and cost >= 0 for cost in observer.values()): + found.add("summary_observer_malformed") + + lanes = summary.get("lanes") + if not isinstance(lanes, dict): + found.add("summary_lane_malformed") + return sorted(found), workload + metrics = _budget_metrics(protocol) + for lane_id, lane in lanes.items(): + if lane_id not in workload["lanes"]: + found.add("summary_lane_unknown") + if not isinstance(lane, dict): + found.add("summary_lane_malformed") + continue + for metric in metrics & set(lane): + if _entry_malformed(lane[metric]): + found.add("summary_entry_malformed") + elif lane_id == "candidate" and "not_applicable" in lane[metric]: + # The candidate is the subject of every budget: it cannot opt out of one. + found.add("summary_candidate_not_applicable") + return sorted(found), workload def _statistic(samples, statistic): if statistic == "median": return median(samples) if statistic == "p95": - return nearest_rank(samples, 0.95) + return nearest_rank(samples, P95_QUANTILE) return sum(samples) / len(samples) @@ -320,15 +555,25 @@ def _sample_floor(protocol, statistic): return protocol["samples"]["min_lifecycle_runs_per_lane"] -def _judge(protocol, entry, statistic): - """Return (statistic value or None, inconclusive verdict or None).""" +def _judge(protocol, entry, statistic, candidate): + """Return (statistic value or None, verdict or None) for a validated entry. + + A None verdict means the value is judgeable. A failed candidate lifecycle is a + failure; a flagged sample, an unaccounted admission or a failed baseline lifecycle + leaves the statistic inconclusive. + """ if entry is None: return None, "inconclusive_missing" - if isinstance(entry, dict) and entry.get("not_applicable") is True: + if "not_applicable" in entry: return None, "not_applicable" - samples, expected = _samples(entry) - if samples is None or expected is None: + if candidate and entry.get("failed"): + return None, "fail" + expected = entry.get("expected") + if expected is None: return None, "inconclusive_missing" + if not _count(entry.get("flagged")) or not _count(entry.get("failed")) or entry["flagged"] or entry["failed"]: + return None, "inconclusive_admission" + samples = entry["samples"] if len(samples) < expected: return None, "inconclusive_coverage" if len(samples) < _sample_floor(protocol, statistic): @@ -341,35 +586,36 @@ def evaluate(protocol, summary): Comparative budgets are judged against each applicable baseline separately; there is no aggregate. A candidate/baseline difference smaller than inconclusive_factor times the - observer's own cost for that metric is inconclusive rather than a pass or a fail. + observer's own cost for that metric is inconclusive rather than a pass or a fail. A + comparative budget whose baselines are all not_applicable gets one extra + inconclusive_no_baseline record, so it can never be accepted without a comparison. """ - if not isinstance(summary, dict): - return ["summary_not_object"], [] - if summary.get("protocol_revision") != protocol["revision"]: - return ["summary_revision_mismatch"], [] - workloads = {w["id"]: w for w in protocol["workloads"]} - workload = workloads.get(summary.get("workload")) - if workload is None: - return ["summary_workload_unknown"], [] - lanes = summary.get("lanes") - if not isinstance(lanes, dict) or any(lane not in workload["lanes"] for lane in lanes): - return ["summary_lane_unknown"], [] - observer = summary.get("observer_cost") if isinstance(summary.get("observer_cost"), dict) else {} + diagnostics, workload = validate_summary(protocol, summary) + if diagnostics: + return diagnostics, [] + lanes = summary["lanes"] + observer = summary.get("observer_cost", {}) factor = protocol["observer"]["inconclusive_factor"] - candidate = lanes.get("candidate") if isinstance(lanes.get("candidate"), dict) else {} + candidate = lanes.get("candidate", {}) records = [] for budget in protocol["budgets"]["comparative"]: metric, statistic = budget["metric"], budget["statistic"] - cand_value, cand_problem = _judge(protocol, candidate.get(metric), statistic) + cand_value, cand_problem = _judge(protocol, candidate.get(metric), statistic, candidate=True) + applicable = 0 for baseline in (lane for lane in workload["lanes"] if lane != "candidate"): record = {"workload": workload["id"], "metric": metric, "statistic": statistic, "baseline": baseline, "max_ratio": budget["max_ratio"]} - base_entry = lanes.get(baseline, {}).get(metric) if isinstance(lanes.get(baseline), dict) else None - base_value, base_problem = _judge(protocol, base_entry, statistic) + base_value, base_problem = _judge(protocol, lanes.get(baseline, {}).get(metric), statistic, + candidate=False) + if base_problem == "not_applicable": + record["verdict"] = "not_applicable" + records.append(record) + continue + applicable += 1 problem = cand_problem or base_problem cost = observer.get(metric) - if problem is None and not (_number(cost) and cost >= 0): + if problem is None and cost is None: problem = "inconclusive_observer" if problem is None and base_value <= 0: problem = "inconclusive_missing" @@ -379,13 +625,17 @@ def evaluate(protocol, summary): problem = "inconclusive_observer" record["verdict"] = problem or ("pass" if record["ratio"] <= budget["max_ratio"] else "fail") records.append(record) + if applicable == 0: + records.append({"workload": workload["id"], "metric": metric, "statistic": statistic, + "max_ratio": budget["max_ratio"], "applicable_baselines": 0, + "verdict": "fail" if cand_problem == "fail" else "inconclusive_no_baseline"}) for budget in protocol["budgets"]["absolute"]: if budget.get("workload") not in (None, workload["id"]): continue metric, statistic = budget["metric"], budget["statistic"] record = {"workload": workload["id"], "metric": metric, "statistic": statistic, "max": budget["max"]} - value, problem = _judge(protocol, candidate.get(metric), statistic) + value, problem = _judge(protocol, candidate.get(metric), statistic, candidate=True) if problem is None: record["value"] = value within = value <= budget["max"] if budget["inclusive"] else value < budget["max"] @@ -395,10 +645,15 @@ def evaluate(protocol, summary): return [], records +def accepted(records): + """A summary is accepted only when it has verdicts and every one is pass or not_applicable.""" + return bool(records) and all(record["verdict"] in ACCEPTING_VERDICTS for record in records) + + def _load(path, unreadable): try: return json.loads(Path(path).read_text()), None - except (OSError, ValueError, UnicodeDecodeError): + except (OSError, ValueError, UnicodeDecodeError, RecursionError): return None, unreadable @@ -425,9 +680,9 @@ def main(argv=None): if diagnostics: print(json.dumps({"status": "invalid", "diagnostics": diagnostics})) return 2 - failed = any(record["verdict"] != "pass" and record["verdict"] != "not_applicable" for record in records) - print(json.dumps({"status": "judged", "accepted": not failed, "verdicts": records}, indent=2)) - return 1 if failed else 0 + ok = accepted(records) + print(json.dumps({"status": "judged", "accepted": ok, "verdicts": records}, indent=2)) + return 0 if ok else 1 print(json.dumps({"status": "valid", "protocol": PROTOCOL_ID, "revision": PROTOCOL_REVISION, "lanes": len(protocol["lanes"]), "rounds": len(protocol["schedule"]["orders"])})) return 0 diff --git a/tests/python/test_application_benchmark_protocol.py b/tests/python/test_application_benchmark_protocol.py index 67339d54b..038204583 100644 --- a/tests/python/test_application_benchmark_protocol.py +++ b/tests/python/test_application_benchmark_protocol.py @@ -18,10 +18,17 @@ PROTOCOL_DOC = (ROOT / "docs/reference/application-benchmark-protocol.md").read_text() PERFORMANCE_DOC = (ROOT / "docs/performance.md").read_text() MARKER = "PRIVATE-MARKER-9f2c" +BASELINES = ("compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +COMPARATIVE = ("steady_physical_footprint_bytes", "cpu_seconds_per_task", "warm_startup_seconds", + "warm_restart_seconds") +ABSOLUTE = ("info_latency_ms", "warm_admission_ms", "cached_status_1000_objects_ms", + "idle_control_plane_core_fraction", "watcher_ack_ms") +# Every JSON value type, including the non-standard numbers Python's json module accepts. +FUZZ_VALUES = (None, True, 0, -1, 1.5, float("nan"), float("inf"), "x", [], ["x"], [["x"]], {}, {"x": []}) -def mutated(change): - doc = copy.deepcopy(PROTOCOL) +def mutated(change, doc=None): + doc = copy.deepcopy(PROTOCOL if doc is None else doc) change(doc) return doc @@ -30,20 +37,28 @@ def lane(doc, lane_id): return next(item for item in doc["lanes"] if item["id"] == lane_id) +def workload(doc, workload_id): + return next(item for item in doc["workloads"] if item["id"] == workload_id) + + def budget(doc, kind, metric): return next(item for item in doc["budgets"][kind] if item["metric"] == metric) -def entry(values, expected=None): - return {"samples": list(values), "expected": len(values) if expected is None else expected} +def entry(values, expected=None, flagged=0, failed=0): + return {"samples": list(values), "expected": len(values) if expected is None else expected, + "flagged": flagged, "failed": failed} + + +def not_applicable(): + return {"not_applicable": True, "reason": "lane cannot express this workload"} def small_summary(candidate_scale=0.5, runs=10, tail=60): """A complete small-fixture summary where the candidate is `candidate_scale` of each baseline.""" - baselines = [item for item in PROTOCOL["workloads"][1]["lanes"] if item != "candidate"] base = {"steady_physical_footprint_bytes": 1000.0, "cpu_seconds_per_task": 10.0, "warm_startup_seconds": 20.0, "warm_restart_seconds": 10.0} - lanes = {item: {metric: entry([value] * runs) for metric, value in base.items()} for item in baselines} + lanes = {item: {metric: entry([value] * runs) for metric, value in base.items()} for item in BASELINES} lanes["candidate"] = {metric: entry([value * candidate_scale] * runs) for metric, value in base.items()} lanes["candidate"].update({ "info_latency_ms": entry([50] * tail), @@ -52,7 +67,8 @@ def small_summary(candidate_scale=0.5, runs=10, tail=60): "idle_control_plane_core_fraction": entry([0.005] * runs), "watcher_ack_ms": entry([100] * tail), }) - return {"protocol_revision": 1, "workload": "small-fixture", + return {"protocol_revision": 1, "workload": "small-fixture", "cache_class": "warm", + "scale": {"branch_instances": 1}, "observer_cost": {metric: 0.0 for metric in base}, "lanes": lanes} @@ -61,6 +77,48 @@ def verdicts(records, metric, baseline=None): if record["metric"] == metric and record.get("baseline", baseline) == baseline] +def run_cli(argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = protocol_check.main(argv) + return code, json.loads(out.getvalue()) + + +def evaluate_file(summary): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "summary.json" + path.write_text(json.dumps(summary)) + return run_cli(["--evaluate", str(path)]) + + +def node_paths(node, prefix=(), list_limit=None): + """Yield the path of every node below `node`; `list_limit` caps indices per list.""" + if isinstance(node, dict): + children = node.items() + elif isinstance(node, list): + children = list(enumerate(node))[:list_limit] + else: + return + for key, child in children: + yield prefix + (key,) + yield from node_paths(child, prefix + (key,), list_limit) + + +def replace_at(doc, path, value): + doc = copy.deepcopy(doc) + parent = doc + for key in path[:-1]: + parent = parent[key] + if value is DELETE: + del parent[path[-1]] + else: + parent[path[-1]] = copy.deepcopy(value) + return doc + + +DELETE = object() + + class FrozenProtocolTest(unittest.TestCase): def test_frozen_protocol_is_valid(self): self.assertEqual(protocol_check.validate_protocol(PROTOCOL), []) @@ -93,6 +151,16 @@ def test_nearest_rank_and_median(self): self.assertEqual(protocol_check.median([3, 1, 2]), 2) self.assertEqual(protocol_check.median([4, 1, 3, 2]), 2.5) + def test_frozen_gate_c_coverage(self): + self.assertIn("restore", PROTOCOL["phases"]) + self.assertEqual(PROTOCOL["scale_points"]["branch_instances"], [1, 8, 32]) + self.assertEqual(PROTOCOL["scale_points"]["mcp_clients"], [1, 32]) + scopes = {item["metric"]: item.get("workload") for item in PROTOCOL["budgets"]["absolute"]} + # Gate C limits only the watcher budget to the small fixture. + self.assertEqual(scopes, {metric: None for metric in ABSOLUTE} | {"watcher_ack_ms": "small-fixture"}) + for item in PROTOCOL["workloads"]: + self.assertEqual(sorted(item["lanes"]), sorted(("candidate",) + BASELINES)) + def test_documents_carry_every_frozen_artifact_and_budget(self): for item in PROTOCOL["lanes"]: for field in ("commit", "asset", "asset_sha256", "tag"): @@ -107,6 +175,12 @@ def test_documents_carry_every_frozen_artifact_and_budget(self): self.assertIn(f"{item['max_ratio']:.2f}", PROTOCOL_DOC) for item in PROTOCOL["budgets"]["absolute"]: self.assertIn(f"`{item['metric']}`", PROTOCOL_DOC) + for phase in PROTOCOL["phases"]: + self.assertIn(f"`{phase}`", PROTOCOL_DOC) + self.assertIn("1, 8 and 32 branch instances", PROTOCOL_DOC) + self.assertIn("1 and 32 MCP clients", PROTOCOL_DOC) + for code in sorted(protocol_check.VERDICTS): + self.assertIn(f"`{code}`", PROTOCOL_DOC) self.assertIn(str(PROTOCOL["samples"]["p95_min_samples"]), PROTOCOL_DOC) limit = PROTOCOL["known_limits"][0] for value in (limit["issue"], "#313", f"{limit['warm_up_seconds']} s", f"{limit['witness_seconds']} s", @@ -115,27 +189,71 @@ def test_documents_carry_every_frozen_artifact_and_budget(self): self.assertIn("application-benchmark-protocol.md", PERFORMANCE_DOC) +def _set_scope(metric, scope): + def change(doc): + budget(doc, "absolute", metric)["workload"] = scope + return change + + class NegativeProtocolControls(unittest.TestCase): CASES = { "comparative_budget_weakened": lambda d: budget(d, "comparative", "steady_physical_footprint_bytes") .update(max_ratio=0.71), + # A weaker duplicate listed before the frozen entry must not hide behind it. + "comparative_budget_weakened:duplicate": lambda d: d["budgets"]["comparative"].insert( + 0, {"metric": "cpu_seconds_per_task", "statistic": "median", "max_ratio": 0.9}), "absolute_budget_weakened": lambda d: budget(d, "absolute", "idle_control_plane_core_fraction") .update(inclusive=True), + "absolute_budget_weakened:ceiling": lambda d: budget(d, "absolute", "info_latency_ms").update(max=1000), + "absolute_budget_weakened:ceiling-exclusive": lambda d: budget(d, "absolute", "watcher_ack_ms") + .update(max=201, inclusive=False), "comparative_budget_missing": lambda d: d["budgets"]["comparative"].pop(), "absolute_budget_missing": lambda d: d["budgets"]["absolute"].pop(), "statistic_unknown": lambda d: budget(d, "absolute", "info_latency_ms").update(statistic="mean"), + "budget_statistic_mismatch:absolute": lambda d: budget(d, "absolute", "info_latency_ms") + .update(statistic="median"), + "budget_statistic_mismatch:comparative": lambda d: budget(d, "comparative", "cpu_seconds_per_task") + .update(statistic="window_mean"), + "budget_statistic_mismatch:idle": lambda d: budget(d, "absolute", "idle_control_plane_core_fraction") + .update(statistic="median"), + "budget_scope_narrowed:watcher": _set_scope("watcher_ack_ms", "event-agent"), + "budget_scope_narrowed:info": _set_scope("info_latency_ms", "small-fixture"), + "budget_scope_narrowed:idle": _set_scope("idle_control_plane_core_fraction", "small-fixture"), + "budget_scope_narrowed:unknown": _set_scope("watcher_ack_ms", "nowhere"), + "budget_scope_narrowed:additional": lambda d: d["budgets"]["absolute"].append( + {"metric": "exec_ms", "statistic": "p95", "max": 50, "inclusive": True, "workload": "nowhere"}), + "workload_baseline_missing": lambda d: workload(d, "event-agent")["lanes"].remove("compose"), + "workload_shape_mismatch": lambda d: workload(d, "event-agent")["shape"].update(services=13), + "phases_mismatch": lambda d: d["phases"].remove("restore"), + "scale_points_mismatch:instances": lambda d: d["scale_points"].update(branch_instances=[1, 8]), + "scale_points_mismatch:clients": lambda d: d["scale_points"].update(mcp_clients=[1]), + "scale_points_mismatch:judged": lambda d: d["scale_points"].update(judged={"branch_instances": 8}), "artifact_commit_malformed": lambda d: lane(d, "candidate").update(commit="28de2f89"), "artifact_digest_malformed": lambda d: lane(d, "hack-4.2.1").update(asset_sha256="0" * 63), "artifact_version_malformed": lambda d: lane(d, "hack-4.1.1").update(tag="v4.1.2"), "p95_samples_insufficient": lambda d: d["samples"].update(p95_min_samples=58), + # 0.90 needs only 29 samples, so 60 still clears the bound: only the quantile check catches it. + "p95_samples_insufficient:quantile": lambda d: d["samples"].update(p95_quantile=0.9), + # 0.90 confidence needs 45 samples: only the confidence floor catches it. + "p95_samples_insufficient:confidence": lambda d: d["samples"].update( + p95_bound_confidence=0.9, p95_min_samples=45), + "p95_method_mismatch": lambda d: d["samples"].update(p95_method="linear-interpolation"), + "schedule_rounds_mismatch": lambda d: d["schedule"].update(rounds_per_workload=9), + "schedule_design_mismatch": lambda d: d["schedule"].update(design="latin-square"), "observer_policy_missing": lambda d: d["observer"].update(inconclusive_factor=0.5), + "observer_policy_missing:factor": lambda d: d["observer"].update(inconclusive_factor=1), "observer_policy_missing:window": lambda d: d["observer"].update(control_window_seconds=599), "cache_class_missing": lambda d: d["cache_classes"].update(budget_class="cold-cache"), "workload_lane_unknown": lambda d: d["workloads"][0]["lanes"].append("hack-3.0.0"), + "workloads_malformed:id": lambda d: d["workloads"][0].update(id=["event-agent"]), "host_incomplete": lambda d: d["host"]["orbstack"].pop("build"), "protocol_revision_mismatch": lambda d: d.update(revision=2), + "protocol_revision_mismatch:bool": lambda d: d.update(revision=True), "known_limit_malformed": lambda d: d["known_limits"][0].pop("revisit"), + "known_limit_malformed:number": lambda d: d.update(known_limits=7), "lane_id_duplicate": lambda d: d["lanes"].append(copy.deepcopy(d["lanes"][0])), + "compose_lane_host_mismatch": lambda d: lane(d, "compose").update(compose_version="2.0.0"), + "compose_lane_host_mismatch:engine": lambda d: lane(d, "compose").update(engine="docker-desktop"), } def test_each_weakening_is_refused_with_its_closed_code(self): @@ -145,12 +263,23 @@ def test_each_weakening_is_refused_with_its_closed_code(self): self.assertIn(code.split(":")[0], found) self.assertTrue(set(found) <= protocol_check.DIAGNOSTICS) + def test_stricter_revisions_stay_valid(self): + stricter = mutated(lambda d: ( + budget(d, "absolute", "watcher_ack_ms").pop("workload"), + budget(d, "absolute", "info_latency_ms").update(max=90), + budget(d, "comparative", "cpu_seconds_per_task").update(max_ratio=0.6), + d["observer"].update(inconclusive_factor=3), + d["scale_points"].update(branch_instances=[1, 8, 16, 32]), + )) + self.assertEqual(protocol_check.validate_protocol(stricter), []) + def test_dropping_a_required_baseline_is_refused(self): def drop(doc): doc["lanes"] = [item for item in doc["lanes"] if item["id"] != "hack-4.2.0"] doc["schedule"]["orders"] = protocol_check.williams_orders([item["id"] for item in doc["lanes"]]) - for workload in doc["workloads"]: - workload["lanes"].remove("hack-4.2.0") + doc["schedule"]["rounds_per_workload"] = len(doc["schedule"]["orders"]) + for item in doc["workloads"]: + item["lanes"].remove("hack-4.2.0") self.assertIn("baseline_lane_missing", protocol_check.validate_protocol(mutated(drop))) def test_unbalanced_or_short_rotation_is_refused(self): @@ -171,6 +300,18 @@ def swap(doc): missing_lane = mutated(lambda d: d["schedule"]["orders"][3].pop()) self.assertIn("schedule_lane_set_mismatch", protocol_check.validate_protocol(missing_lane)) + def test_any_malformed_node_maps_to_closed_codes_without_raising(self): + for path in node_paths(PROTOCOL): + for value in FUZZ_VALUES + (DELETE,): + with self.subTest(path=path, value=repr(value)): + found = protocol_check.validate_protocol(replace_at(PROTOCOL, path, value)) + self.assertTrue(set(found) <= protocol_check.DIAGNOSTICS, found) + for value in FUZZ_VALUES: + found = protocol_check.validate_protocol(value) + if not isinstance(value, dict): + self.assertEqual(found, ["protocol_not_object"]) + self.assertTrue(found and set(found) <= protocol_check.DIAGNOSTICS) + def test_diagnostics_never_echo_input_values(self): doc = mutated(lambda d: lane(d, "candidate").update(commit=MARKER, asset_sha256=MARKER)) with tempfile.TemporaryDirectory() as directory: @@ -197,12 +338,20 @@ def judge(self, summary): self.assertTrue({record["verdict"] for record in records} <= protocol_check.VERDICTS) return records + def refused(self, summary): + diagnostics, records = protocol_check.evaluate(PROTOCOL, summary) + self.assertEqual(records, []) + self.assertTrue(diagnostics and set(diagnostics) <= protocol_check.DIAGNOSTICS, diagnostics) + return diagnostics + def test_complete_improved_cohort_passes_every_budget_against_every_baseline(self): records = self.judge(small_summary()) self.assertEqual({record["verdict"] for record in records}, {"pass"}) comparative = [record for record in records if "baseline" in record] self.assertEqual(len(comparative), 4 * 4) self.assertEqual(len(records) - len(comparative), 5) + self.assertTrue(protocol_check.accepted(records)) + self.assertFalse(protocol_check.accepted([])) def test_boundary_ratios_follow_the_frozen_thresholds(self): records = self.judge(small_summary(candidate_scale=0.75)) @@ -233,6 +382,28 @@ def test_missing_planned_samples_are_inconclusive_coverage(self): records = self.judge(summary) self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_missing"]) + def test_flagged_failed_or_unaccounted_samples_are_never_passes(self): + summary = small_summary() + summary["lanes"]["compose"]["cpu_seconds_per_task"] = entry([10.0] * 10, expected=11, flagged=1) + summary["lanes"]["hack-4.1.1"]["cpu_seconds_per_task"] = entry([10.0] * 10, expected=11, failed=1) + summary["lanes"]["hack-4.2.0"]["cpu_seconds_per_task"].pop("flagged") + summary["lanes"]["candidate"]["info_latency_ms"].pop("failed") + records = self.judge(summary) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "hack-4.1.1"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "hack-4.2.0"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "info_latency_ms"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "hack-4.2.1"), ["pass"]) + self.assertFalse(protocol_check.accepted(records)) + + def test_failed_candidate_lifecycle_is_a_failure(self): + summary = small_summary() + summary["lanes"]["candidate"]["warm_startup_seconds"] = entry([10.0] * 10, expected=11, failed=1) + summary["lanes"]["candidate"]["watcher_ack_ms"] = entry([100] * 60, expected=61, failed=1) + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"fail"}) + self.assertEqual(verdicts(records, "watcher_ack_ms"), ["fail"]) + def test_difference_within_observer_cost_is_inconclusive(self): summary = small_summary() summary["observer_cost"]["cpu_seconds_per_task"] = 2.6 @@ -242,6 +413,15 @@ def test_difference_within_observer_cost_is_inconclusive(self): records = self.judge(summary) self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"inconclusive_observer"}) + def test_frozen_observer_factor_is_two(self): + # A 5.0 difference against a 2.6 cost is inside 2 x cost; a factor of 1 would pass it. + summary = small_summary() + summary["observer_cost"]["cpu_seconds_per_task"] = 2.6 + weak = mutated(lambda d: d["observer"].update(inconclusive_factor=1)) + _, records = protocol_check.evaluate(weak, summary) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "compose")), {"pass"}) + self.assertIn("observer_policy_missing", protocol_check.validate_protocol(weak)) + def test_absolute_ceilings_respect_inclusivity(self): summary = small_summary() summary["lanes"]["candidate"]["info_latency_ms"] = entry([100] * 60) @@ -254,20 +434,131 @@ def test_absolute_ceilings_respect_inclusivity(self): summary["lanes"]["candidate"]["info_latency_ms"] = entry([50] * 56 + [101] * 4) self.assertEqual(verdicts(self.judge(summary), "info_latency_ms"), ["fail"]) - def test_not_applicable_and_missing_lanes_are_never_passes(self): + def test_a_not_applicable_baseline_with_a_reason_is_skipped_and_missing_lanes_are_inconclusive(self): summary = small_summary() - summary["lanes"]["hack-4.1.1"]["warm_startup_seconds"] = {"not_applicable": True} - summary["lanes"].pop("hack-4.2.0") + summary["lanes"]["hack-4.1.1"]["warm_startup_seconds"] = not_applicable() records = self.judge(summary) self.assertEqual(verdicts(records, "warm_startup_seconds", "hack-4.1.1"), ["not_applicable"]) + self.assertEqual(verdicts(records, "warm_startup_seconds", "compose"), ["pass"]) + self.assertTrue(protocol_check.accepted(records)) + summary["lanes"].pop("hack-4.2.0") + records = self.judge(summary) self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "hack-4.2.0")), {"inconclusive_missing"}) + self.assertFalse(protocol_check.accepted(records)) + + def test_no_applicable_baseline_is_inconclusive_never_accepted(self): + summary = small_summary() + for baseline in BASELINES: + summary["lanes"][baseline] = {metric: not_applicable() for metric in COMPARATIVE} + records = self.judge(summary) + for metric in COMPARATIVE: + self.assertEqual(verdicts(records, metric), ["inconclusive_no_baseline"]) + self.assertFalse(protocol_check.accepted(records)) + code, report = evaluate_file(summary) + self.assertEqual((code, report["accepted"]), (1, False)) + + def test_the_candidate_can_never_be_not_applicable(self): + summary = small_summary() + summary["lanes"]["candidate"] = {metric: not_applicable() for metric in COMPARATIVE + ABSOLUTE} + self.assertEqual(self.refused(summary), ["summary_candidate_not_applicable"]) + self.assertEqual(evaluate_file(summary), + (2, {"status": "invalid", "diagnostics": ["summary_candidate_not_applicable"]})) + for metric in COMPARATIVE + ABSOLUTE: + with self.subTest(metric=metric): + one = small_summary() + one["lanes"]["candidate"][metric] = not_applicable() + self.assertEqual(self.refused(one), ["summary_candidate_not_applicable"]) + + def test_not_applicable_never_overrides_samples(self): + for lane_id in ("candidate", "compose"): + with self.subTest(lane=lane_id): + summary = small_summary(candidate_scale=5) + summary["lanes"][lane_id]["warm_startup_seconds"]["not_applicable"] = True + self.assertEqual(self.refused(summary), ["summary_entry_malformed"]) + for bad in ({"not_applicable": True}, {"not_applicable": True, "reason": " "}, + {"not_applicable": False, "reason": "x"}, {"not_applicable": 1, "reason": "x"}): + with self.subTest(entry=bad): + summary = small_summary() + summary["lanes"]["compose"]["warm_startup_seconds"] = bad + self.assertEqual(self.refused(summary), ["summary_entry_malformed"]) + + def test_only_warm_single_instance_summaries_are_judged(self): + cases = { + "summary_cache_class_mismatch": [("cache_class", "cold-cache"), ("cache_class", None)], + "summary_scale_unknown": [("scale", None), ("scale", {"branch_instances": 2}), + ("scale", {"branch_instances": True}), + ("scale", {"branch_instances": 1, "mcp_clients": 1}), ("scale", [1])], + "summary_scale_not_judged": [("scale", {"branch_instances": 32}), ("scale", {"mcp_clients": 1})], + } + for code, changes in cases.items(): + for key, value in changes: + with self.subTest(code=code, value=value): + self.assertEqual(self.refused({**small_summary(), key: value}), [code]) + summary = small_summary() + summary.pop("cache_class") + self.assertEqual(self.refused(summary), ["summary_cache_class_mismatch"]) + self.assertEqual(evaluate_file({**small_summary(), "cache_class": "cold-cache"})[0], 2) + + def test_malformed_values_are_refused_with_closed_codes(self): + def candidate(metric, value): + def change(summary): + summary["lanes"]["candidate"][metric] = value + return change + + def baseline(metric, field, value): + def change(summary): + summary["lanes"]["compose"][metric][field] = value + return change + + cases = [ + ("summary_entry_malformed", candidate("info_latency_ms", entry([-5] * 60))), + ("summary_entry_malformed", candidate("warm_startup_seconds", entry([-1.0] * 10))), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", 0)), + ("summary_entry_malformed", candidate("info_latency_ms", entry([], expected=0))), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", -10)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", True)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", 10.0)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", 9)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "flagged", -1)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "failed", "0")), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "samples", [1, "2"])), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "samples", [float("nan")] * 10)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "median", 1.0)), + ("summary_entry_malformed", candidate("watcher_ack_ms", [100] * 60)), + ("summary_lane_malformed", lambda s: s["lanes"].update(compose=[MARKER])), + ("summary_lane_malformed", lambda s: s.update(lanes=[MARKER])), + ("summary_observer_malformed", lambda s: s["observer_cost"].update(cpu_seconds_per_task=-1)), + ("summary_observer_malformed", lambda s: s.update(observer_cost=[0])), + ("summary_workload_unknown", lambda s: s.update(workload=["x"])), + ("summary_workload_unknown", lambda s: s.update(workload={"id": "small-fixture"})), + ("summary_revision_mismatch", lambda s: s.update(protocol_revision=True)), + ] + for code, change in cases: + summary = small_summary() + change(summary) + with self.subTest(code=code, summary=summary.get("workload")): + self.assertEqual(self.refused(summary), [code]) + + def test_list_valued_workload_exits_invalid_not_with_a_traceback(self): + self.assertEqual(evaluate_file({**small_summary(), "workload": ["x"]}), + (2, {"status": "invalid", "diagnostics": ["summary_workload_unknown"]})) + + def test_any_malformed_summary_node_maps_to_closed_codes_without_raising(self): + summary = small_summary() + for path in node_paths(summary, list_limit=1): + for value in FUZZ_VALUES + (DELETE,): + with self.subTest(path=path, value=repr(value)): + diagnostics, records = protocol_check.evaluate(PROTOCOL, replace_at(summary, path, value)) + self.assertTrue(set(diagnostics) <= protocol_check.DIAGNOSTICS, diagnostics) + self.assertTrue({record["verdict"] for record in records} <= protocol_check.VERDICTS) + self.assertTrue(bool(diagnostics) != bool(records)) def test_workload_scoped_budgets_only_apply_to_their_workload(self): summary = small_summary() summary["workload"] = "event-agent" records = self.judge(summary) metrics = {record["metric"] for record in records if "baseline" not in record} - self.assertEqual(metrics, {"info_latency_ms", "warm_admission_ms"}) + self.assertEqual(metrics, set(ABSOLUTE) - {"watcher_ack_ms"}) def test_malformed_summaries_are_refused_with_closed_codes(self): cases = { @@ -281,16 +572,13 @@ def test_malformed_summaries_are_refused_with_closed_codes(self): self.assertEqual(protocol_check.evaluate(PROTOCOL, summary), ([code], [])) def test_cli_exit_codes(self): - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "summary.json" - for scale, expected in ((0.5, 0), (0.9, 1)): - path.write_text(json.dumps(small_summary(candidate_scale=scale))) - with contextlib.redirect_stdout(io.StringIO()) as out: - self.assertEqual(protocol_check.main(["--evaluate", str(path)]), expected) - self.assertEqual(json.loads(out.getvalue())["accepted"], expected == 0) - with contextlib.redirect_stdout(io.StringIO()): - self.assertEqual(protocol_check.main([]), 0) - self.assertEqual(protocol_check.main(["--schedule"]), 0) + for scale, expected in ((0.5, 0), (0.9, 1)): + code, report = evaluate_file(small_summary(candidate_scale=scale)) + self.assertEqual(code, expected) + self.assertEqual(report["accepted"], expected == 0) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(protocol_check.main([]), 0) + self.assertEqual(protocol_check.main(["--schedule"]), 0) if __name__ == "__main__": From ca810049a03b8f5280040ffd5ee97343d50d2e66 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 14:26:06 -0400 Subject: [PATCH 3/4] fix(perf): pin frozen benchmark identities and refuse oversized numbers The protocol validator checked only the format of artifact commits and digests, so a different well-formed candidate commit, a removed asset, a recast lane kind or backend, or a changed candidate capacity still validated. Pin the revision-1 lane artifacts, lane kinds and backends, candidate capacity, host values, workload pins and correctness checks exactly, with closed refusal codes. Bound every protocol and summary number to the interoperable JSON range so an oversized integer is refused as malformed (exit 2) instead of raising OverflowError, and keep an overflowing ratio a failure without printing a non-standard JSON constant. Document why revision 1 freezes the native development profile although the candidate labels it experimental-development-not-benchmark-qualified: graph allocation refuses every other profile and native up always requests it. Co-Authored-By: Claude Opus 5.5 --- .../application-benchmark-protocol.md | 38 ++++- .../check-application-benchmark-protocol.py | 136 +++++++++++++++++- .../test_application_benchmark_protocol.py | 113 ++++++++++++++- 3 files changed, 273 insertions(+), 14 deletions(-) diff --git a/docs/reference/application-benchmark-protocol.md b/docs/reference/application-benchmark-protocol.md index da2022937..9f53d6c8e 100644 --- a/docs/reference/application-benchmark-protocol.md +++ b/docs/reference/application-benchmark-protocol.md @@ -71,6 +71,25 @@ of the whole runtime, not the configured limit. A lower configured limit is not footprint result. Any capacity-matched control is a separate experiment with its own revision. +**Why `development`.** The candidate's own source describes the `development` profile +as an experimental allocation that cannot supply benchmark qualification, and its +lifecycle records carry the label `experimental-development-not-benchmark-qualified`. +That label belongs to the earlier WU02 provider comparison, whose frozen envelope is the +`research` profile (2 vCPUs, 2048 MiB, 4 GiB storage). It does not apply to this +protocol, for two reasons: + +- Graph allocation refuses every profile except `development` with `graph_profile`, + and both workloads here are service graphs. The `research` profile cannot run either + workload. +- A native `hack up` or `restart` always requests `development`. It is the capacity a + user gets by default, which is what this table freezes. + +So revision 1 freezes `development` on purpose. A candidate record that carries that +label is expected and recorded, not a reason to refuse the cohort. This protocol does +not produce WU02 research-envelope results, and WU02 results are not revision-1 +results. Changing the candidate's label or comment is separate runtime work; it cannot +change the frozen `28de2f89` artifact. + ## Workloads Two workloads run in every lane, and each workload is compared against every required @@ -259,6 +278,15 @@ changed p95 method or confidence, a workload that drops a required baseline, a c workload shape, a missing phase, a missing scale point and a Compose lane that differs from the frozen host engine. +The identities on this page are exact, not floors. The checker pins each lane's tag, +version, commit, asset name and digest, each lane's kind and backend, the candidate's +capacity, the host and engine values, and each workload's pin. A different or missing +value is refused (`artifact_identity_mismatch`, `lane_runtime_mismatch`, +`candidate_capacity_mismatch`, `host_mismatch`, `workload_pin_mismatch`), and so is a +workload that drops a frozen correctness check (`workload_correctness_weakened`). A +revision may add correctness checks. Every number in the protocol or a summary must be +within ±(2⁵³ − 1), the interoperable JSON range; a larger value is refused as malformed. + These are acceptance thresholds, not measured achievements. ## Judging a summary @@ -325,8 +353,10 @@ python3 -m unittest discover -s tests/python -p test_application_benchmark_proto The unit tests check the frozen file, the rotation balance, the p95 sample floor and that this page carries every frozen artifact, budget, phase and verdict. They also -prove that each weakening listed under [Budgets](#budgets), a bad digest, an unbalanced -order, too few samples and a missing observer policy is refused; that missing, -flagged, `not_applicable`-only or cold-cache data is never accepted; and that any -malformed protocol or summary value yields a closed code rather than a traceback. They +prove that each weakening listed under [Budgets](#budgets), a bad digest, a changed +artifact, host, capacity or workload pin, an unbalanced order, too few samples and a +missing observer policy is refused; that missing, flagged, `not_applicable`-only or +cold-cache data is never accepted; and that any malformed protocol or summary value, +including an integer too large for a float, yields a closed code rather than a +traceback. They run offline, locally and in CI. diff --git a/scripts/check-application-benchmark-protocol.py b/scripts/check-application-benchmark-protocol.py index c5d9edb2b..5eff046ad 100644 --- a/scripts/check-application-benchmark-protocol.py +++ b/scripts/check-application-benchmark-protocol.py @@ -8,6 +8,9 @@ --schedule prints the frozen rotated lane order for each round; --evaluate SUMMARY judges a retained cohort summary against the frozen budgets. +Revision-1 identities (artifacts, host, lane kinds and backends, candidate capacity, +workload pins) are exact; budgets and sample policy are floors a revision may tighten. +Every number must lie within MAX_JSON_NUMBER. Diagnostics are closed codes from DIAGNOSTICS; they never echo input values or paths. A budget verdict is one of VERDICTS. A statistic is judged only from complete, sufficient, admitted samples: missing, short, flagged or unaccounted samples are @@ -72,12 +75,86 @@ SCALE_POINTS = {"branch_instances": (1, 8, 32), "mcp_clients": (1, 32)} JUDGED_SCALE = {"branch_instances": 1} +# Revision-1 identities. These are exact, not floors: a different artifact, host, lane +# kind or backend, candidate capacity or workload pin is a different experiment and +# needs a new revision (and new constants here), never an edit to revision 1. +FROZEN_HOST = { + "id": "m3-max", + "cpu": "Apple M3 Max", + "logical_cpus": 16, + "memory_bytes": 137438953472, + "os": "macOS 27.0", + "docker_cli": "29.4.0", + "compose": "5.1.2", + "orbstack": {"version": "2.2.3", "build": "20963", "cpu": 16, "memory_mib": 71680}, +} +# Fields that identify the exact released artifact a lane installs. +ARTIFACT_FIELDS = ("version", "tag", "commit", "asset", "asset_sha256") +# Fields that identify how a lane runs. +LANE_RUNTIME_FIELDS = ("kind", "backend", "engine", "compose_version") +FROZEN_LANES = { + "candidate": { + "kind": "hack", "backend": "native", "version": "5.0.0-next.5", "tag": "v5.0.0-next.5", + "commit": "28de2f8902d3776c3da9326e7ae6c6aa6e7f69bc", + "asset": "hack-5.0.0-next.5-darwin-arm64-native.tar.gz", + "asset_sha256": "ba1da4d349ab70069a1b0775ba671838440b5479cffd1fc2fe41c59816c3df11", + }, + "compose": {"kind": "compose", "backend": "docker-compose", "engine": "orbstack", "compose_version": "5.1.2"}, + "hack-4.1.1": { + "kind": "hack", "backend": "compose", "version": "4.1.1", "tag": "v4.1.1", + "commit": "74bf0672f41bd3f68a4ce6cba28b9ba15467c2b2", + "asset": "hack-4.1.1-darwin-arm64.tar.gz", + "asset_sha256": "968f75fe66cd84fdf93b0c5b64ca5df02f98c74281b5a9238125b3afd2f3d72b", + }, + "hack-4.2.0": { + "kind": "hack", "backend": "compose", "version": "4.2.0", "tag": "v4.2.0", + "commit": "575847486963db94623e1e7f8cd49acca286c517", + "asset": "hack-4.2.0-darwin-arm64.tar.gz", + "asset_sha256": "c62b533aaddbeaafa594cab682cf10041c9bcaea69e2fe25e33e85df49654a85", + }, + "hack-4.2.1": { + "kind": "hack", "backend": "compose", "version": "4.2.1", "tag": "v4.2.1", + "commit": "5a4c51babb7ef725a30f2f3fd3677081a64bc247", + "asset": "hack-4.2.1-darwin-arm64.tar.gz", + "asset_sha256": "557583d946abcd0dd0a3fa12b2f7b81f18310d594b0699d663ebf239d0d4d729", + }, +} +# The native `development` profile: graph execution, and so every workload here, +# refuses any other profile (see the protocol page, "Capacities"). +FROZEN_CANDIDATE_CAPACITY = {"profile": "development", "vcpus": 4, "memory_mib": 6144, "storage_gib": 32, + "overlay_gib": 10} +FROZEN_WORKLOAD_PINS = { + "event-agent": "application commit and configuration digest recorded per cohort; identical across lanes", + "small-fixture": "image oven/bun:1.4.2-slim with its image ID recorded per cohort, pull never", +} +# Frozen correctness checks per workload. A revision may add checks, never drop one. +FROZEN_CORRECTNESS = { + "event-agent": ( + "all declared long-running services running and every HTTP health check healthy", + "both initializers exited 0", + "routed sign-in page returns HTTP 200", + "a per-run data marker written before restart reads back after restart", + ), + "small-fixture": ( + "initializer creates the SQLite schema in a named volume and exits 0", + "web health endpoint returns HTTP 200", + "check task inserts a per-run row and reads the expected count", + "the row reads back after restart", + "a source edit is served within the watcher acknowledgement window", + ), +} +# Largest JSON number magnitude accepted anywhere: the I-JSON (RFC 7493) interoperable +# integer range. Bounding every number keeps each statistic and ratio finite and keeps +# Python from converting an unbounded JSON integer to a float. +MAX_JSON_NUMBER = 2 ** 53 - 1 + DIAGNOSTICS = frozenset({ "protocol_unreadable", "protocol_not_object", "protocol_id_mismatch", "protocol_revision_mismatch", "host_incomplete", + "host_mismatch", "lanes_malformed", "lane_id_duplicate", "compose_lane_host_mismatch", @@ -86,10 +163,15 @@ "artifact_commit_malformed", "artifact_digest_malformed", "artifact_version_malformed", + "artifact_identity_mismatch", + "lane_runtime_mismatch", + "candidate_capacity_mismatch", "workloads_malformed", "workload_lane_unknown", "workload_baseline_missing", "workload_shape_mismatch", + "workload_pin_mismatch", + "workload_correctness_weakened", "phases_mismatch", "scale_points_mismatch", "cache_class_missing", @@ -211,7 +293,18 @@ def median(samples): def _number(value): - return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + """A finite JSON number within MAX_JSON_NUMBER; a boolean is not a number. + + The range comparison never converts an int to a float, so an oversized integer is + refused rather than raising OverflowError, and NaN and infinities compare false. + """ + return isinstance(value, (int, float)) and not isinstance(value, bool) \ + and -MAX_JSON_NUMBER <= value <= MAX_JSON_NUMBER + + +def _same(value, frozen): + """Exact JSON equality: unlike ==, 4.0 is not 4 and true is not 1.""" + return json.dumps(value, sort_keys=True) == json.dumps(frozen, sort_keys=True) def _count(value): @@ -230,6 +323,29 @@ def _check_host(host, found): elif not isinstance(host["orbstack"], dict) or any( not host["orbstack"].get(field) for field in ("version", "build", "cpu", "memory_mib")): found.add("host_incomplete") + # Native stateful results are host-specific: any other host or engine is a new revision. + if not _same(host, FROZEN_HOST): + found.add("host_mismatch") + + +def _check_lane_identity(lane_id, lane, found): + """Refuse a frozen lane whose artifact, runtime or capacity differs from revision 1. + + A missing field counts as different, so removing an identity cannot weaken it. + """ + frozen = FROZEN_LANES.get(lane_id) + if frozen is None: + return + for field in ARTIFACT_FIELDS + LANE_RUNTIME_FIELDS: + if (field in frozen or field in lane) and not _same(lane.get(field), frozen.get(field)): + found.add("artifact_identity_mismatch" if field in ARTIFACT_FIELDS else "lane_runtime_mismatch") + if lane_id == "candidate": + if not _same(lane.get("capacity"), FROZEN_CANDIDATE_CAPACITY): + found.add("candidate_capacity_mismatch") + elif "capacity" in lane: + # Baselines run with the shared engine's default capacity; a per-lane capacity + # would make them a different experiment. + found.add("lane_runtime_mismatch") def _check_lanes(lanes, host, found): @@ -263,6 +379,7 @@ def _check_lanes(lanes, host, found): found.add("compose_lane_host_mismatch") else: found.add("lanes_malformed") + _check_lane_identity(lane_id, lane, found) if "candidate" not in lane_ids: found.add("candidate_lane_missing") if any(baseline not in lane_ids for baseline in REQUIRED_BASELINES): @@ -296,8 +413,14 @@ def _check_workloads(workloads, lane_ids, found): found.add("workloads_malformed") workload_id = workload.get("id") shape = WORKLOAD_SHAPES.get(workload_id) if isinstance(workload_id, str) else None - if shape is not None and workload.get("shape") != shape: + if shape is not None and not _same(workload.get("shape"), shape): found.add("workload_shape_mismatch") + if isinstance(workload_id, str) and workload_id in FROZEN_WORKLOAD_PINS: + if not _same(workload.get("pin"), FROZEN_WORKLOAD_PINS[workload_id]): + found.add("workload_pin_mismatch") + checks = workload.get("correctness") + if not _strings(checks) or not set(FROZEN_CORRECTNESS[workload_id]) <= set(checks): + found.add("workload_correctness_weakened") return ids @@ -619,11 +742,16 @@ def evaluate(protocol, summary): problem = "inconclusive_observer" if problem is None and base_value <= 0: problem = "inconclusive_missing" + ratio = None if problem is None: - record["ratio"] = cand_value / base_value + ratio = cand_value / base_value + # A tiny positive baseline can overflow the ratio to infinity: that is a + # failure, but only a finite ratio is printed, so the report stays strict JSON. + if math.isfinite(ratio): + record["ratio"] = ratio if abs(cand_value - base_value) < factor * cost: problem = "inconclusive_observer" - record["verdict"] = problem or ("pass" if record["ratio"] <= budget["max_ratio"] else "fail") + record["verdict"] = problem or ("pass" if ratio <= budget["max_ratio"] else "fail") records.append(record) if applicable == 0: records.append({"workload": workload["id"], "metric": metric, "statistic": statistic, diff --git a/tests/python/test_application_benchmark_protocol.py b/tests/python/test_application_benchmark_protocol.py index 038204583..06350adce 100644 --- a/tests/python/test_application_benchmark_protocol.py +++ b/tests/python/test_application_benchmark_protocol.py @@ -23,8 +23,14 @@ "warm_restart_seconds") ABSOLUTE = ("info_latency_ms", "warm_admission_ms", "cached_status_1000_objects_ms", "idle_control_plane_core_fraction", "watcher_ack_ms") -# Every JSON value type, including the non-standard numbers Python's json module accepts. -FUZZ_VALUES = (None, True, 0, -1, 1.5, float("nan"), float("inf"), "x", [], ["x"], [["x"]], {}, {"x": []}) +# Every JSON value type, including the non-standard numbers Python's json module accepts +# and integers too large for a float, which Python's json module also parses. +HUGE = 10 ** 400 +FUZZ_VALUES = (None, True, 0, -1, 1.5, float("nan"), float("inf"), float("-inf"), HUGE, -HUGE, 2 ** 53, + "x", [], ["x"], [["x"]], {}, {"x": []}) +# Another well-formed 40-hex commit and 64-hex digest: only the identity pin refuses them. +OTHER_COMMIT = "0123456789abcdef0123456789abcdef01234567" +OTHER_DIGEST = "0123456789abcdef" * 4 def mutated(change, doc=None): @@ -161,11 +167,32 @@ def test_frozen_gate_c_coverage(self): for item in PROTOCOL["workloads"]: self.assertEqual(sorted(item["lanes"]), sorted(("candidate",) + BASELINES)) + def test_frozen_file_matches_the_pinned_identities(self): + for lane_id, frozen in protocol_check.FROZEN_LANES.items(): + item = lane(PROTOCOL, lane_id) + for field, value in frozen.items(): + self.assertEqual(item[field], value, (lane_id, field)) + self.assertEqual(lane(PROTOCOL, "candidate")["capacity"], protocol_check.FROZEN_CANDIDATE_CAPACITY) + self.assertEqual(PROTOCOL["host"], protocol_check.FROZEN_HOST) + for workload_id, pin in protocol_check.FROZEN_WORKLOAD_PINS.items(): + self.assertEqual(workload(PROTOCOL, workload_id)["pin"], pin) + self.assertEqual(workload(PROTOCOL, workload_id)["correctness"], + list(protocol_check.FROZEN_CORRECTNESS[workload_id])) + def test_documents_carry_every_frozen_artifact_and_budget(self): - for item in PROTOCOL["lanes"]: + # Driven by the validator's pins, not the JSON, so a removed field is still required. + for lane_id, frozen in protocol_check.FROZEN_LANES.items(): + if frozen["kind"] != "hack": + continue for field in ("commit", "asset", "asset_sha256", "tag"): - if field in item: - self.assertIn(item[field], PROTOCOL_DOC, (item["id"], field)) + self.assertIn(f"`{frozen[field]}`", PROTOCOL_DOC, (lane_id, field)) + capacity = protocol_check.FROZEN_CANDIDATE_CAPACITY + self.assertIn(f"native `{capacity['profile']}` profile: {capacity['vcpus']} vCPUs, " + f"{capacity['memory_mib']} MiB guest memory, {capacity['storage_gib']} GiB storage, " + f"{capacity['overlay_gib']} GiB overlay", PROTOCOL_DOC) + # The candidate's own source labels this profile non-qualifying; the page must say why it is used. + for phrase in ("experimental-development-not-benchmark-qualified", "graph_profile", "`research`"): + self.assertIn(phrase, PROTOCOL_DOC) host = PROTOCOL["host"] for value in (host["cpu"], host["os"], host["docker_cli"], host["compose"], host["orbstack"]["version"], host["orbstack"]["build"]): @@ -189,6 +216,12 @@ def test_documents_carry_every_frozen_artifact_and_budget(self): self.assertIn("application-benchmark-protocol.md", PERFORMANCE_DOC) +def _recast_as_compose(item): + for field in ("version", "commit", "asset_sha256", "tag", "asset", "capacity"): + item.pop(field, None) + item.update(kind="compose", engine="orbstack", compose_version="5.1.2") + + def _set_scope(metric, scope): def change(doc): budget(doc, "absolute", metric)["workload"] = scope @@ -254,6 +287,32 @@ class NegativeProtocolControls(unittest.TestCase): "lane_id_duplicate": lambda d: d["lanes"].append(copy.deepcopy(d["lanes"][0])), "compose_lane_host_mismatch": lambda d: lane(d, "compose").update(compose_version="2.0.0"), "compose_lane_host_mismatch:engine": lambda d: lane(d, "compose").update(engine="docker-desktop"), + # Well-formed but different identities: only the revision-1 pins refuse these. + "artifact_identity_mismatch:candidate-commit": lambda d: lane(d, "candidate").update(commit=OTHER_COMMIT), + "artifact_identity_mismatch:baseline-commit": lambda d: lane(d, "hack-4.2.0").update(commit=OTHER_COMMIT), + "artifact_identity_mismatch:digest": lambda d: lane(d, "hack-4.1.1").update(asset_sha256=OTHER_DIGEST), + "artifact_identity_mismatch:version": lambda d: lane(d, "candidate").update( + version="5.0.0-next.6", tag="v5.0.0-next.6"), + "artifact_identity_mismatch:asset": lambda d: lane(d, "hack-4.2.1").update( + asset="hack-4.2.1-darwin-x64.tar.gz"), + "artifact_identity_mismatch:asset-removed": lambda d: lane(d, "candidate").pop("asset"), + # A hack lane recast as a Compose lane on the frozen engine, with its artifact removed. + "lane_runtime_mismatch:candidate-kind": lambda d: _recast_as_compose(lane(d, "candidate")), + "lane_runtime_mismatch:baseline-kind": lambda d: _recast_as_compose(lane(d, "hack-4.1.1")), + "lane_runtime_mismatch:backend": lambda d: lane(d, "candidate").update(backend="compose"), + "lane_runtime_mismatch:backend-removed": lambda d: lane(d, "candidate").pop("backend"), + "lane_runtime_mismatch:compose-backend": lambda d: lane(d, "compose").update(backend="hack"), + "lane_runtime_mismatch:baseline-capacity": lambda d: lane(d, "compose").update(capacity={"cpus": 4}), + "candidate_capacity_mismatch": lambda d: lane(d, "candidate")["capacity"].update(memory_mib=12288), + "candidate_capacity_mismatch:profile": lambda d: lane(d, "candidate")["capacity"].update( + profile="research"), + "candidate_capacity_mismatch:removed": lambda d: lane(d, "candidate").pop("capacity"), + "candidate_capacity_mismatch:float": lambda d: lane(d, "candidate")["capacity"].update(vcpus=4.0), + "host_mismatch:os": lambda d: d["host"].update(os="macOS 26.5.1"), + "host_mismatch:cpu": lambda d: d["host"].update(cpu="Apple M5 Max", logical_cpus=18), + "host_mismatch:orbstack": lambda d: d["host"]["orbstack"].update(memory_mib=32768), + "workload_pin_mismatch": lambda d: workload(d, "small-fixture").update(pin="image oven/bun:latest"), + "workload_correctness_weakened": lambda d: workload(d, "event-agent")["correctness"].pop(), } def test_each_weakening_is_refused_with_its_closed_code(self): @@ -265,6 +324,7 @@ def test_each_weakening_is_refused_with_its_closed_code(self): def test_stricter_revisions_stay_valid(self): stricter = mutated(lambda d: ( + workload(d, "small-fixture")["correctness"].append("cleanup leaves no owned container"), budget(d, "absolute", "watcher_ack_ms").pop("workload"), budget(d, "absolute", "info_latency_ms").update(max=90), budget(d, "comparative", "cpu_seconds_per_task").update(max_ratio=0.6), @@ -323,7 +383,8 @@ def test_diagnostics_never_echo_input_values(self): self.assertEqual(code, 2) self.assertNotIn(MARKER, out.getvalue()) self.assertEqual(json.loads(out.getvalue())["diagnostics"], - ["artifact_commit_malformed", "artifact_digest_malformed"]) + ["artifact_commit_malformed", "artifact_digest_malformed", + "artifact_identity_mismatch"]) path.write_text("{not json " + MARKER) out = io.StringIO() with contextlib.redirect_stdout(out): @@ -539,6 +600,46 @@ def change(summary): with self.subTest(code=code, summary=summary.get("workload")): self.assertEqual(self.refused(summary), [code]) + def test_oversized_integers_are_refused_not_a_traceback(self): + """Python parses 10**400 exactly; converting it to a float would raise OverflowError.""" + cases = { + "candidate": lambda s: s["lanes"]["candidate"].update(info_latency_ms=entry([HUGE] * 60)), + "baseline": lambda s: s["lanes"]["compose"].update(cpu_seconds_per_task=entry([HUGE] * 10)), + "beyond-interoperable": lambda s: s["lanes"]["compose"].update( + cpu_seconds_per_task=entry([2 ** 53] * 10)), + } + for name, change in cases.items(): + with self.subTest(case=name): + summary = small_summary() + change(summary) + self.assertEqual(evaluate_file(summary), + (2, {"status": "invalid", "diagnostics": ["summary_entry_malformed"]})) + summary = small_summary() + summary["observer_cost"]["cpu_seconds_per_task"] = HUGE + self.assertEqual(evaluate_file(summary), + (2, {"status": "invalid", "diagnostics": ["summary_observer_malformed"]})) + protocol = mutated(lambda d: budget(d, "comparative", "cpu_seconds_per_task").update(max_ratio=HUGE)) + self.assertIn("comparative_budget_weakened", protocol_check.validate_protocol(protocol)) + + def test_an_overflowing_ratio_fails_and_the_report_stays_strict_json(self): + summary = small_summary() + summary["lanes"]["compose"]["warm_startup_seconds"] = entry([5e-324] * 10) + records = self.judge(summary) + [record] = [item for item in records + if item["metric"] == "warm_startup_seconds" and item.get("baseline") == "compose"] + self.assertEqual(record["verdict"], "fail") + self.assertNotIn("ratio", record) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "summary.json" + path.write_text(json.dumps(summary)) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(protocol_check.main(["--evaluate", str(path)]), 1) + + def refuse_constant(name): + raise AssertionError(f"non-standard JSON constant {name}") + json.loads(out.getvalue(), parse_constant=refuse_constant) + def test_list_valued_workload_exits_invalid_not_with_a_traceback(self): self.assertEqual(evaluate_file({**small_summary(), "workload": ["x"]}), (2, {"status": "invalid", "diagnostics": ["summary_workload_unknown"]})) From b5c2e9f839069f876165d0b78631e2ec003a6318 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Tue, 20 Oct 2026 09:19:28 -0400 Subject: [PATCH 4/4] fix(perf): pin the frozen rotated order and sample-count floors The validator still accepted a different balanced Williams order, so reordering the lane list and regenerating the schedule validated while the published order table and --schedule disagreed. It also judged sample counts against the program floors (5 runs, 59 tail samples) rather than the frozen revision-1 values (10 and 60), so a protocol edit could lower them without refusal. Pin the revision-1 order exactly (schedule_order_mismatch) and treat the frozen sample counts as floors. Check that the published order table matches the frozen order round for round, and that the page states both frozen sample counts. Co-Authored-By: Claude Opus 5.5 --- .../application-benchmark-protocol.md | 28 ++++++++++--------- .../check-application-benchmark-protocol.py | 21 ++++++++++++-- .../test_application_benchmark_protocol.py | 28 ++++++++++++++++--- 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/docs/reference/application-benchmark-protocol.md b/docs/reference/application-benchmark-protocol.md index 9f53d6c8e..98bafa25e 100644 --- a/docs/reference/application-benchmark-protocol.md +++ b/docs/reference/application-benchmark-protocol.md @@ -280,11 +280,14 @@ from the frozen host engine. The identities on this page are exact, not floors. The checker pins each lane's tag, version, commit, asset name and digest, each lane's kind and backend, the candidate's -capacity, the host and engine values, and each workload's pin. A different or missing -value is refused (`artifact_identity_mismatch`, `lane_runtime_mismatch`, -`candidate_capacity_mismatch`, `host_mismatch`, `workload_pin_mismatch`), and so is a -workload that drops a frozen correctness check (`workload_correctness_weakened`). A -revision may add correctness checks. Every number in the protocol or a summary must be +capacity, the host and engine values, each workload's pin and the rotated order above. +A different or missing value is refused (`artifact_identity_mismatch`, +`lane_runtime_mismatch`, `candidate_capacity_mismatch`, `host_mismatch`, +`workload_pin_mismatch`, `schedule_order_mismatch`), and so is a workload that drops a +frozen correctness check (`workload_correctness_weakened`). A revision may add +correctness checks. The sample counts are floors at their frozen values: a protocol +that plans fewer than 10 lifecycle runs or 60 tail samples per lane is refused, even +where the program floor or the p95 bound would allow fewer. Every number in the protocol or a summary must be within ±(2⁵³ − 1), the interoperable JSON range; a larger value is refused as malformed. These are acceptance thresholds, not measured achievements. @@ -352,11 +355,10 @@ python3 -m unittest discover -s tests/python -p test_application_benchmark_proto ``` The unit tests check the frozen file, the rotation balance, the p95 sample floor and -that this page carries every frozen artifact, budget, phase and verdict. They also -prove that each weakening listed under [Budgets](#budgets), a bad digest, a changed -artifact, host, capacity or workload pin, an unbalanced order, too few samples and a -missing observer policy is refused; that missing, flagged, `not_applicable`-only or -cold-cache data is never accepted; and that any malformed protocol or summary value, -including an integer too large for a float, yields a closed code rather than a -traceback. They -run offline, locally and in CI. +that this page carries every frozen artifact, budget, phase, round and verdict. They +also prove that each weakening listed under [Budgets](#budgets), a bad digest, a +changed artifact, host, capacity, workload pin or rotated order, an unbalanced order, +too few samples and a missing observer policy is refused; that missing, flagged, +`not_applicable`-only or cold-cache data is never accepted; and that any malformed +protocol or summary value, including an integer too large for a float, yields a closed +code rather than a traceback. They run offline, locally and in CI. diff --git a/scripts/check-application-benchmark-protocol.py b/scripts/check-application-benchmark-protocol.py index 5eff046ad..bc20e4336 100644 --- a/scripts/check-application-benchmark-protocol.py +++ b/scripts/check-application-benchmark-protocol.py @@ -9,7 +9,8 @@ --evaluate SUMMARY judges a retained cohort summary against the frozen budgets. Revision-1 identities (artifacts, host, lane kinds and backends, candidate capacity, -workload pins) are exact; budgets and sample policy are floors a revision may tighten. +workload pins, rotated lane order) are exact; budgets and sample counts are floors at +their frozen values, which a revision may tighten. Every number must lie within MAX_JSON_NUMBER. Diagnostics are closed codes from DIAGNOSTICS; they never echo input values or paths. A budget verdict is one of VERDICTS. A statistic is judged only from complete, @@ -37,6 +38,9 @@ # Program floors (Linear program acceptance, Gate C) and the frozen revision-1 choices. # A protocol revision may be stricter than these, never weaker. MIN_ROTATED_LIFECYCLE_RUNS = 5 +# Frozen revision-1 sample counts. These are floors: a revision may plan more, never fewer. +FROZEN_LIFECYCLE_RUNS = 10 +FROZEN_P95_MIN_SAMPLES = 60 REQUIRED_BASELINES = ("compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") # metric: (ceiling ratio, frozen statistic). A ratio at the ceiling passes. COMPARATIVE_CEILINGS = { @@ -181,6 +185,7 @@ "schedule_rounds_mismatch", "schedule_position_unbalanced", "schedule_carryover_unbalanced", + "schedule_order_mismatch", "p95_samples_insufficient", "p95_method_mismatch", "comparative_budget_missing", @@ -243,6 +248,12 @@ def williams_orders(lanes): return [[lanes[index] for index in row] for row in rows] +# The frozen revision-1 rotation: the Williams square for this lane sequence. Another +# balanced order is a different schedule, so it needs a new revision. +FROZEN_LANE_SEQUENCE = ("candidate", "compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +FROZEN_ORDERS = williams_orders(list(FROZEN_LANE_SEQUENCE)) + + def position_balanced(orders): lanes = sorted(orders[0]) for position in range(len(lanes)): @@ -453,12 +464,15 @@ def _check_schedule(doc, lane_ids, found): found.add("schedule_rounds_mismatch") samples = doc.get("samples") if isinstance(doc.get("samples"), dict) else {} planned = samples.get("min_lifecycle_runs_per_lane") - if not _count(planned) or planned < MIN_ROTATED_LIFECYCLE_RUNS or len(orders) < planned: + if not _count(planned) or planned < max(MIN_ROTATED_LIFECYCLE_RUNS, FROZEN_LIFECYCLE_RUNS) \ + or len(orders) < planned: found.add("schedule_rounds_below_minimum") if not position_balanced(orders): found.add("schedule_position_unbalanced") if not carryover_balanced(orders): found.add("schedule_carryover_unbalanced") + if orders != FROZEN_ORDERS: + found.add("schedule_order_mismatch") def _check_samples(doc, found): @@ -468,7 +482,8 @@ def _check_samples(doc, found): floor = samples.get("p95_min_samples") if not _number(quantile) or quantile != P95_QUANTILE \ or not _number(confidence) or not P95_MIN_CONFIDENCE <= confidence < 1 \ - or not _count(floor) or floor < minimum_order_statistic_samples(P95_QUANTILE, confidence): + or not _count(floor) or floor < FROZEN_P95_MIN_SAMPLES \ + or floor < minimum_order_statistic_samples(P95_QUANTILE, confidence): found.add("p95_samples_insufficient") if samples.get("p95_method") != P95_METHOD: found.add("p95_method_mismatch") diff --git a/tests/python/test_application_benchmark_protocol.py b/tests/python/test_application_benchmark_protocol.py index 06350adce..4ac0b0af9 100644 --- a/tests/python/test_application_benchmark_protocol.py +++ b/tests/python/test_application_benchmark_protocol.py @@ -5,6 +5,7 @@ import io import json from pathlib import Path +import re import tempfile import unittest @@ -132,6 +133,9 @@ def test_frozen_protocol_is_valid(self): def test_frozen_schedule_is_the_williams_design_for_the_frozen_lanes(self): lanes = [item["id"] for item in PROTOCOL["lanes"]] self.assertEqual(PROTOCOL["schedule"]["orders"], protocol_check.williams_orders(lanes)) + self.assertEqual(PROTOCOL["schedule"]["orders"], protocol_check.FROZEN_ORDERS) + self.assertEqual(PROTOCOL["samples"]["min_lifecycle_runs_per_lane"], protocol_check.FROZEN_LIFECYCLE_RUNS) + self.assertEqual(PROTOCOL["samples"]["p95_min_samples"], protocol_check.FROZEN_P95_MIN_SAMPLES) self.assertEqual(PROTOCOL["schedule"]["rounds_per_workload"], len(PROTOCOL["schedule"]["orders"])) self.assertGreaterEqual(len(PROTOCOL["schedule"]["orders"]), protocol_check.MIN_ROTATED_LIFECYCLE_RUNS) self.assertEqual(PROTOCOL["samples"]["min_lifecycle_runs_per_lane"], len(PROTOCOL["schedule"]["orders"])) @@ -208,7 +212,13 @@ def test_documents_carry_every_frozen_artifact_and_budget(self): self.assertIn("1 and 32 MCP clients", PROTOCOL_DOC) for code in sorted(protocol_check.VERDICTS): self.assertIn(f"`{code}`", PROTOCOL_DOC) - self.assertIn(str(PROTOCOL["samples"]["p95_min_samples"]), PROTOCOL_DOC) + self.assertIn(f"at least {protocol_check.FROZEN_P95_MIN_SAMPLES} samples per lane", PROTOCOL_DOC) + self.assertIn(f"at least {protocol_check.FROZEN_LIFECYCLE_RUNS} lifecycles per lane", PROTOCOL_DOC) + # The published order table is the frozen rotation, round for round. + rows = re.findall(r"^\| (\d+) \| ([a-z0-9., -]+) \|$", PROTOCOL_DOC, re.MULTILINE) + self.assertEqual([int(number) for number, _ in rows], + list(range(1, len(protocol_check.FROZEN_ORDERS) + 1))) + self.assertEqual([order.split(", ") for _, order in rows], protocol_check.FROZEN_ORDERS) limit = PROTOCOL["known_limits"][0] for value in (limit["issue"], "#313", f"{limit['warm_up_seconds']} s", f"{limit['witness_seconds']} s", f"{limit['witness_target_seconds']} s", "HACK-1169"): @@ -222,6 +232,11 @@ def _recast_as_compose(item): item.update(kind="compose", engine="orbstack", compose_version="5.1.2") +def _reorder_schedule(doc): + doc["lanes"][1], doc["lanes"][2] = doc["lanes"][2], doc["lanes"][1] + doc["schedule"]["orders"] = protocol_check.williams_orders([item["id"] for item in doc["lanes"]]) + + def _set_scope(metric, scope): def change(doc): budget(doc, "absolute", metric)["workload"] = scope @@ -265,14 +280,19 @@ class NegativeProtocolControls(unittest.TestCase): "artifact_digest_malformed": lambda d: lane(d, "hack-4.2.1").update(asset_sha256="0" * 63), "artifact_version_malformed": lambda d: lane(d, "hack-4.1.1").update(tag="v4.1.2"), "p95_samples_insufficient": lambda d: d["samples"].update(p95_min_samples=58), + # 59 clears the distribution-free bound; only the frozen revision-1 floor of 60 refuses it. + "p95_samples_insufficient:frozen-floor": lambda d: d["samples"].update(p95_min_samples=59), # 0.90 needs only 29 samples, so 60 still clears the bound: only the quantile check catches it. "p95_samples_insufficient:quantile": lambda d: d["samples"].update(p95_quantile=0.9), - # 0.90 confidence needs 45 samples: only the confidence floor catches it. - "p95_samples_insufficient:confidence": lambda d: d["samples"].update( - p95_bound_confidence=0.9, p95_min_samples=45), + # 0.90 confidence needs only 45 samples, so 60 still clears it: only the confidence floor catches it. + "p95_samples_insufficient:confidence": lambda d: d["samples"].update(p95_bound_confidence=0.9), "p95_method_mismatch": lambda d: d["samples"].update(p95_method="linear-interpolation"), "schedule_rounds_mismatch": lambda d: d["schedule"].update(rounds_per_workload=9), "schedule_design_mismatch": lambda d: d["schedule"].update(design="latin-square"), + # Above the program floor of 5 but below the frozen 10 runs per lane. + "schedule_rounds_below_minimum:frozen-floor": lambda d: d["samples"].update(min_lifecycle_runs_per_lane=5), + # Another balanced Williams square, from a reordered lane list: only the order pin refuses it. + "schedule_order_mismatch": _reorder_schedule, "observer_policy_missing": lambda d: d["observer"].update(inconclusive_factor=0.5), "observer_policy_missing:factor": lambda d: d["observer"].update(inconclusive_factor=1), "observer_policy_missing:window": lambda d: d["observer"].update(control_window_seconds=599),