diff --git a/changelog.d/graph-amendments-11-13.added.md b/changelog.d/graph-amendments-11-13.added.md new file mode 100644 index 000000000..95adf2fa4 --- /dev/null +++ b/changelog.d/graph-amendments-11-13.added.md @@ -0,0 +1 @@ +Graph interface amendments 11–14 (declared entrants, partitioned mass, declared tolerance, entrant-person strata) and their executor semantics: charter properties B6, B7, C5, and D6 go green; `compile_graph` refuses any later owner of the mass-partition column; readers see a column's producer tolerance through structural carriers; the whole `Capabilities` projection joins node identity and is compared on cache hits; the acceptance ratchet compares property identities against the baseline rather than per-file counts. Amendment 17 adds `NumericScope` and `KernelContext.numerics`, the per-coordinate numeric class, bound, and platform scope a gate holds an input to. The run-manifest key hashes every node receipt less its run-level fields (`hit`, `wall_time`, and a release node's decision-derived `outcome`) plus the derived tier, so cold and warm runs of one computation share a key; signed decisions stay outside the key and are authenticated at certified load. Amendment 15: `compile_graph` refuses any entity, column, or row-mask name containing `.` (a previously valid dotted declaration now fails at compile time), and CREATE frames and `expand_cells` are held to the same rule. Amendment 16: `Numeric.PLATFORM_BITWISE` is added and `fit.qrf@1` is reclassified from `tolerance_bound` to `platform_bitwise` with no tolerance; a platform fingerprint (architecture, OS, Python minor) is folded into every platform-bitwise node key, so a shared store partitions by platform and every `fit.qrf@1` node key moves; parity pins record the platform and the node key. Amendment 18: receipts are deterministic facts of the computation, because the run-manifest key (schema 2) hashes them; a schema-1 manifest loads as legacy in full with `hit` forced to false and is refused by `load_certified`. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index b202fe632..5365a086a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "271c23eca777b63bbfecc5b0bdbfffee634e13f79dd37273172f5fadda4afefb", - "protocol_sha256": "870f9b43766d4c11f6dc1538bf97e360c6a85dd0d4a4a68c149595dfc0902f0a" + "map_sha256": "6b4902d9a640dd459942e588ae2cc7fb937f1c42ad814365c7c1616c98ce1b68", + "protocol_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911" }, "failures": [], "observed": { - "map_sha256": "271c23eca777b63bbfecc5b0bdbfffee634e13f79dd37273172f5fadda4afefb", - "protocol_sha256": "870f9b43766d4c11f6dc1538bf97e360c6a85dd0d4a4a68c149595dfc0902f0a" + "map_sha256": "6b4902d9a640dd459942e588ae2cc7fb937f1c42ad814365c7c1616c98ce1b68", + "protocol_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "870f9b43766d4c11f6dc1538bf97e360c6a85dd0d4a4a68c149595dfc0902f0a", + "implementation_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "870f9b43766d4c11f6dc1538bf97e360c6a85dd0d4a4a68c149595dfc0902f0a", + "implementation_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "835b4d61de3b153b13c536e371dca6a16fad6d41e003d070f9804c7c251f3590" + "spec_sha256": "a6bc79878eb6f64637b9f3eceeea6cc2b050c0e5b8f9aca446179258940c44f2" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "835b4d61de3b153b13c536e371dca6a16fad6d41e003d070f9804c7c251f3590" + "spec_sha256": "a6bc79878eb6f64637b9f3eceeea6cc2b050c0e5b8f9aca446179258940c44f2" }, "status": "pass" } diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index e5d422fad..4f602d4a7 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -45,6 +45,8 @@ its owner. | B3 | **Storage-preserving patch.** Patching owned positions preserves the incumbent column's dtype (nullable `boolean` stays nullable `boolean`; float bits including negative zero survive) and leaves every non-owned position byte-identical. This is the WIC guard, made structural. | leg 2 §3.3 | same | | B4 | **Inputs are immutable.** A kernel receives read-only views; an in-place write raises inside the kernel and the node fails. | leg 1 finding 5 | same | | B5 | **Null means absence.** A node declares each owned cell as *produced* or *absent*. A kernel writing a non-null value into an absent-declared cell is rejected. | `DESIGN.md:128-134` | same | +| B6 | **Entrants are declared.** An `EXPAND` node with `entrants=True` may return rows with null lineage; the executor requires the kernel to materialize every carried column for such a row (dtype-checked), records them as entrants rather than copies in the lineage receipt, and refuses null lineage on a node without the declaration. `entrants=True` with `mass='conserve'` is a compile error. An entrant needs a design anchor: it is admitted only while the weighted entity still carries `design` weights, and its admitted weight becomes that anchor; after any reweight the executor refuses entrant rows on that entity, because a derived weight has no design origin to record (ruled 2026-09-03, #847 gate round 3). | Dynamics: immigrant cohorts (microcosm-dynamics#412, #218) | Max's session; amendment 11 | +| B7 | **Entrant persons carry their stratum.** An entrant row on the person entity takes its stratum from `KernelResult.strata` (indexed by its new id); an entrant person absent from it, a label for a copied or incumbent person, or a label for an unknown id rejects the node. Entrant persons join incumbent or entrant groups through the materialized membership columns, and the mass ledger counts them from the node that admits them. | Dynamics: immigrant cohorts are persons (microcosm-dynamics#412, #218) | Max's session; amendment 14 | ## C. Seeds and factorization @@ -54,16 +56,18 @@ its owner. | C2 | **Removal invariance.** Removing a node that nothing depends on, or adding a new leaf node, changes no other node's key or output. This is the `0347a009` replay: five targets removed, zero survivors re-modeled. | F5 | same | | C3 | **Declared predecessors only.** A chained target's predictors are exactly its declared predecessors. The executor hands a kernel only its declared slices, so an undeclared read is impossible rather than merely detected. | F5, leg 3 §legibility | same | | C4 | **Seed from identity.** A node's RNG seed is a pure function of its node key. Two nodes with identical declarations, inputs, and kernels in different graphs draw identical values. No positional RNG consumption exists anywhere in the shard (static check). | F4, `docs/spec-engine.md:254-282` | same | +| C5 | **Tolerance is declared.** A kernel claiming `tolerance_bound` numerics without a `Tolerance` is refused at registration, and a bitwise kernel may not carry one. The tolerance is recorded in every receipt, and a kernel reading a cell sees the declared tolerance of the node that produced it in `KernelContext.tolerances`: a structural version (`FILTER`, `EXPAND`, `REWEIGHT`) carries a column's tolerance through unchanged, so a bitwise carrier neither tightens nor erases a producer's bound, and a rewrite sees the incumbent producer's. Where more than one node wrote rows of a column in a version — a producer plus an `EXPAND` kernel that materialized entrant rows, or a claimant that took them over — the reader sees the loosest declared tolerance among those writers (componentwise maximum of `rtol`, `atol`, `ulps`; a bitwise writer contributes none), and a claimant's `KernelContext.tolerances` includes the coordinates it claims; a gate comparing against anything else says so in its evidence. | H2 (arm64/x86 one-ulp weights); microcosm-dynamics#412 | Max's session; amendment 13 | ## D. Weights and mass | Id | Property | Closes | Owner | |---|---|---|---| | D1 | **Weight transitions are typed nodes.** `design → importance → calibrated` are the only legal transitions; the executor rejects a regression and rejects a transition declared on inherited (non-explicit) weights. | F9 (leg 1 finding 1) | María / Max's session | -| D2 | **Mass ledger.** Every population-changing node (select, concat, clone, reweight) emits a mass record with before/after totals and per-stratum mass. Under `conserve`, a stratum losing mass fails the node. `select` cannot drop mass silently. | F9 | same | +| D2 | **Mass ledger.** Every population-changing node (select, concat, clone, reweight) emits a mass record with before/after totals and per-stratum mass. Mass is weighted person mass per stratum, within each declared partition (amendment 12). Under `conserve`, a stratum losing mass fails the node. An expansion that conserves its weight entity's mass while changing group composition changes person mass and must say so: `declared`, with a receipt stating the invariant it does hold (ruled 2026-09-02 on #844). `select` cannot drop mass silently. | F9 | same | | D3 | **Cap anchored to design.** A calibration node's `max_weight_ratio` is asserted against the declared anchor across composed stages; a selection-then-refit chain that ships a record above `R × design` fails. | F9 (#493) | same | | D4 | **Filters are binary.** A target filter containing NaN or a non-binary value is rejected at compile. | F9 | same | | D5 | **Uncertainty travels.** A target's declared standard error reaches the calibration kernel's inputs; a kernel that ignores a declared `se` must say so in its capability record. | scoreboard row 5 (leg 1 finding 7) | same | +| D6 | **Mass is partitioned.** With `Graph.mass_partition` set, the ledger reports per stratum within each partition value, `conserve` holds within each partition, and a node that moves mass between partitions under `conserve` fails. Every `CREATE` node declares the partition column with a partition dtype, and no later node may own it (write or rewrite), or compilation fails: a partition value is fixed when the row is created, because a reassignment with the total unchanged is invisible to every mass policy. A row contributes mass only to the partitions it exists in. | Dynamics: person-period residency (microcosm-dynamics#412) | Max's session; amendment 12 | ## E. Store and resume @@ -110,7 +114,7 @@ hand-drawn, so they stay true as the code moves. | Id | Property | Closes | Owner | |---|---|---|---| -| H1 | **Kernel parity.** Each wrapped legacy kernel (QRF fit and draw via `microcosm-fit`, calibrate via `microcosm-calibrate`, simulate via a `RulesEngine`) produces byte-identical output to the direct call on a pinned fixture and seed. | #378 step 3 | Max's session | +| H1 | **Kernel parity.** Each wrapped legacy kernel (QRF fit and draw via `microcosm-fit`, calibrate via `microcosm-calibrate`, simulate via a `RulesEngine`) produces byte-identical output to the direct call on a pinned fixture and seed. A platform-bitwise kernel (amendment 16) is byte-identical within a platform: its fixture carries one pin per platform CI runs on plus the authoring platform, bytes are asserted on each of those, and on any other platform the node key must differ from every pinned key (identity partitioning). | #378 step 3 | Max's session | | H2 | **UK spine parity.** The UK 26-stage spine expressed as a graph reproduces the current spine's `uk_frame_content_identity` on the fixture. Both sides run all 26 transforms from the fixture's raw tables in the test's own process, the graph through a CREATE kernel bound to the root transform, and nothing is pinned: the root transform's household weights differ by one ulp between machines (2026-09-02: two of 135 households on x86 versus this Mac), which the LCFS raking and every weight split then inherit. Stage order is derived from declared `consumes`; the hand-maintained `_STAGE_NAMES` tuple is deleted. | F12, F8 | María | | H3 | **US post-transfer parity.** The stacked spine's derive → seed → simulate subgraph reproduces a pinned fixture output. | #378 step 3 | Max's session, later | @@ -201,6 +205,96 @@ Amendments so far (each re-locked): (python storage); a population entering the graph with `object` strings is normalized at `CREATE`. Parity fixtures compare identities after the same normalization on the legacy side, and say so. +11. **Entrants are declared.** `Node.entrants=True` (EXPAND only) lets a + kernel add rows that copy no base row: their lineage is null, the + kernel materializes every carried column for them, the executor + records them as entrants, and the node's mass policy cannot be + `conserve`. An entrant is admitted only while the weighted entity + carries `design` weights (its admitted weight is its design anchor); + after a reweight the executor refuses entrant rows on that entity, + since a derived weight has no design origin to record. Raised by the + dynamics program (immigrant cohorts through the scheduled-entries seam, + microcosm-dynamics#412 / #218); Max ruled go 2026-09-02; adopted + 2026-09-02; the design-anchor rule was stated 2026-09-03. +12. **Mass is partitioned.** `Graph.mass_partition = (entity, column)` + partitions mass accounting (per stratum within each partition value; + `conserve` per partition). Every `CREATE` node declares the column + with a dtype in `PARTITION_DTYPES`, and `compile_graph` refuses any + later owner of it, rewrite or not: a partition value is fixed when + the row is created (review finding, 2026-09-02). The field is + normative: the executor folds it into every structural node's key, so + structural keys move once when a graph adopts it. Raised by the + dynamics program for person-period residency; adopted 2026-09-02. +13. **Tolerance is declared.** `Capabilities.tolerance: Tolerance | None` + (`rtol`, `atol`, `ulps`) is required for `tolerance_bound` kernels and + forbidden for bitwise ones; `KernelContext.tolerances` hands each + reader the declared tolerance of every input cell's producer, resolved + through structural carriers to the node that wrote the values (a + rewrite sees the incumbent producer's; where several nodes wrote rows + of one column — entrant materialization, claims — the loosest declared + tolerance among them). The whole `Capabilities` + projection, tolerance included, is part of a node's identity and is + compared on every cache hit. Raised by the H2 parity finding (root + weights differ by one ulp between arm64 and x86) and the dynamics + review; adopted 2026-09-02. + +14. **Entrant persons carry their stratum.** `KernelResult.strata` (EXPAND + kernels on an `entrants=True` node only) names the stratum of every + entrant person by its new id; the executor requires exactly the entrant + persons there. Raised by the implementation of amendment 11, which + found the frozen result had no channel for a new person's mandatory + stratum and left person entrants fail-closed; adopted 2026-09-02. + +15. **Names are dot-free.** `compile`-time declarations refuse an entity, + column, or row-mask name containing `.`: receipts, keys, and gate + evidence spell a coordinate `entity.column`, and a dot inside either + part would let two coordinates collide. `EntitySchema` itself still + permits dots; the graph does not. Raised by the #847 gate review; + adopted 2026-09-03. + +16. **Platform-bitwise numerics.** `Numeric.PLATFORM_BITWISE`: identical + bytes on one platform (architecture and locked dependencies), with no + bound on cross-platform movement; a tolerance is forbidden on it as on + `bitwise`. Adopted when measuring `fit.qrf@1` showed that a one-ulp + difference in the forest flips which donor a quantile draw lands on + (45 of 6,000 cells moved by up to 7% between arm64 and x86_64 while the + rest agreed to one ulp; `docs/graph-qrf-cross-platform.md`), so no + per-cell `Tolerance` is true of it. The node key of a platform-bitwise + kernel carries a platform fingerprint (architecture, OS, Python minor), + so a shared store never serves another platform's output, and parity + pins record the platform: H1 asserts bytes on the pinned platform and + identity partitioning elsewhere; a cross-platform gate on such a kernel + says so in its evidence. Raised by the #847 gate review; adopted + 2026-09-03. + +17. **Numeric scope per input coordinate.** `KernelContext.numerics` maps + each declared input coordinate to a `NumericScope`: the loosest + `Numeric` class among its writers (`bitwise` < `platform_bitwise` < + `tolerance_bound`), the loosest declared `Tolerance` among the + bounded writers, and the platform fingerprint the contract holds on. + A platform-bitwise writer never disappears into a bound: it sets + `platform`, so a sole platform-bitwise writer reaches a gate as + `platform_bitwise` rather than as `None`, and a bounded writer mixed + with one yields a bound that holds on that platform only. + `KernelContext.tolerances` stays as the projection of `numerics`. A + gate that compares across platforms must consult the scope and refuse + or evidence a platform-scoped input (amendment 16). Raised by the + #847 gate review (round 3); adopted 2026-09-03. + +18. **Receipts are deterministic.** `KernelResult.receipt` is never hashed + into a node key, but the run manifest's key hashes every node receipt + less the executor's run-level fields (`hit`, `wall_time`, and a release + node's decision-derived `outcome`), so a receipt must be a deterministic + function of the computation: gate outcomes and evidence belong there; + timings, host names, and iteration diagnostics that vary between runs of + one computation do not. Manifest schema 2 records this identity; a + schema-1 manifest loads as legacy in full (its receipts unauthenticated, + `hit` forced to false) and `load_certified` refuses it. Raised by the + #847 gate review; adopted 2026-09-03. + +Adding a normative field with a default changes the canonical projection +of every node that carries it, so node keys moved with amendments 11 and +13's sibling field `entrants`; no released artifact pins a graph key yet. ## Ownership diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index 994c05b9d..cabec2d32 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ -b6fbf50e0253e6f04a2684fcac295fb27a4608f6a272bedb86c43f087fd36b8e decl.py -8ac8328c086bfbea0e36032e9c7fa9cb1ed62f63a1036bca4eb528c11b9b1890 kernel.py +635fef92c599c298e7f19ca0badfa85aa040bf8e81eafed59f37c48db1fcff06 decl.py +eaf07da2eded1b1895aa0c59f603eb93744ed928df65aa9e65aa633762833949 kernel.py diff --git a/docs/graph-qrf-cross-platform.md b/docs/graph-qrf-cross-platform.md new file mode 100644 index 000000000..c676772c3 --- /dev/null +++ b/docs/graph-qrf-cross-platform.md @@ -0,0 +1,56 @@ +# fit.qrf@1 across platforms: a measurement + +Date: 2026-09-03. Method: `tools/graph_qrf_platform_probe.py`, run natively on +arm64 (Python 3.14) and under Rosetta from an x86_64 environment synced to the +same `uv.lock` (torch stubbed; it is not on the kernel's path). Twenty cases: +seeds 0–4 in four regimes (positive-only, mixed-sign, near-ties, zero-inflated), +600 donors and 300 recipients each, unweighted fits, one draw per recipient. + +## Result + +| statistic | value | +|---|---| +| cells compared | 6000 | +| cells that differ | 45 | +| largest absolute movement | 2.322e+00 | +| largest relative movement | 6.825e-02 | +| largest int64-view distance (not a ulps count once a donor flips) | 450359958233450 | + +Cases with any difference (cases not listed were bit-identical): + +| case | max abs | max rel | int64-view distance | differing | +|---|---|---|---|---| +| positive/0 | 3.907e-02 | 3.194e-02 | 175940752005003 | 3/300 | +| positive/1 | 7.406e-02 | 6.825e-02 | 333524120092336 | 2/300 | +| positive/2 | 2.220e-16 | 1.777e-16 | 1 | 2/300 | +| positive/4 | 1.110e-16 | 1.659e-16 | 1 | 1/300 | +| mixed_sign/0 | 2.220e-16 | 1.954e-16 | 1 | 1/300 | +| mixed_sign/1 | 1.901e-02 | 7.927e-03 | 42798880769607 | 2/300 | +| mixed_sign/3 | 4.441e-16 | 1.399e-16 | 1 | 1/300 | +| mixed_sign/4 | 2.220e-16 | 1.747e-16 | 1 | 1/300 | +| near_ties/0 | 1.000e-01 | 6.667e-02 | 450359958233450 | 8/300 | +| near_ties/1 | 1.000e-09 | 6.342e-10 | 4503600 | 3/300 | +| near_ties/2 | 1.399e-02 | 8.227e-03 | 62989703563009 | 6/300 | +| near_ties/3 | 1.000e-09 | 1.377e-09 | 9919248 | 6/300 | +| near_ties/4 | 7.095e-03 | 4.225e-03 | 31951854077424 | 5/300 | +| zero_inflated/0 | 1.819e-12 | 1.598e-16 | 1 | 1/300 | +| zero_inflated/2 | 1.819e-12 | 1.533e-16 | 1 | 2/300 | +| zero_inflated/3 | 2.322e+00 | 5.400e-04 | 2553496694405 | 1/300 | + +## Reading + +Most differing cells move by one ulp: ordinary floating-point reassociation +between the two architectures. A few cells move by up to 7% relative: a +one-ulp difference inside the forest flips which donor a quantile draw lands +on, and the drawn value jumps to a different donor's value. That is not a +rounding error a per-cell `Tolerance` could bound; it is a discrete outcome +that depends on the platform. + +## Consequence (amendment 16) + +`fit.qrf@1` declares `Numeric.PLATFORM_BITWISE`: identical bytes on one +platform (H1 parity holds on the platform that produced the pins), no bound +across platforms, and no `Tolerance`. Gates that compare its output across +platforms must say so in their evidence. Earlier drafts declared +`Tolerance(ulps=1)` and then `Tolerance(rtol=1e-6)`; both were unmeasured +and both are false, as the table shows. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 6e214e980..1c38f8357 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "271c23eca777b63bbfecc5b0bdbfffee634e13f79dd37273172f5fadda4afefb", - "seed_protocol": "870f9b43766d4c11f6dc1538bf97e360c6a85dd0d4a4a68c149595dfc0902f0a", + "seed_map": "6b4902d9a640dd459942e588ae2cc7fb937f1c42ad814365c7c1616c98ce1b68", + "seed_protocol": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 5aef5be01..3298e907d 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "659b6baf5ebbd71fb7786ec4c4d49df565b2bddabeb868a9385ed226c56880f9" +AM_SPEC_SHA256 = "b128d14f8e6351d745a16ef4537d5c1b8d71d11b9d96cb8a65f1a4fb953f13ed" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "7062e38f4d623553fb0604380a8dac0edacb6261c155b6e31fc38ef7c0f1c57c", + "8c6018eaf518b239a5625f8df8bf783ce4723a03704d100f3dbf4c2bb7765772", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "815158a6318bfce7c85b5454f0b62f964bc8f0105cf8b9721d60e32c289fc5f4", + "d732d3aa599a5c911f1900419142ddff6a1771b6b95c38e06b4c0e513619ead6", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index a77c90f6e..a7effd506 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "9f5b372796f2638378125d97ef5150be4c1f4cba9147b44973e4cd6a5f52f10a" + "7a0fbb0a2a16aff28a9ac6e205e6679b013c580171db87b39597a0b14a97e423" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index f4ccd1471..a9049e406 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2499,7 +2499,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "835b4d61de3b153b13c536e371dca6a16fad6d41e003d070f9804c7c251f3590", + "spec_sha256": "a6bc79878eb6f64637b9f3eceeea6cc2b050c0e5b8f9aca446179258940c44f2", }, } diff --git a/packages/microcosm-fit/src/microcosm/fit/kernels.py b/packages/microcosm-fit/src/microcosm/fit/kernels.py index c39c9a83e..8ec494009 100644 --- a/packages/microcosm-fit/src/microcosm/fit/kernels.py +++ b/packages/microcosm-fit/src/microcosm/fit/kernels.py @@ -48,6 +48,15 @@ ) """Distributions whose versions form part of ``fit.qrf@1``'s identity.""" +#: ``fit.qrf@1`` is bitwise on one platform and NOT tolerance-bound across +#: platforms. Measured 2026-09-03 (``tools/graph_qrf_platform_probe.py``, +#: ``docs/graph-qrf-cross-platform.md``): over 20 seed x regime cases and +#: 6,000 drawn cells, native arm64 and x86_64-under-Rosetta agreed to one ulp +#: on all but 45 cells, and those 45 moved by up to 7% relative (0.074 absolute) +#: because a one-ulp difference in the forest flips which donor a quantile +#: draw lands on. No finite per-cell Tolerance is true of that, so the kernel +#: declares ``Numeric.PLATFORM_BITWISE`` and no tolerance (amendment 16). + QRF_EXECUTOR_SEED_HIGH = 2**31 - 1 """Exclusive upper bound for the one seed drawn from ``KernelContext.rng``.""" @@ -111,7 +120,7 @@ def __init__(self, seed_source: SeedSource = SeedSource.EXECUTOR) -> None: ) self.capabilities = Capabilities( determinism=Determinism.SEEDED, - numeric=Numeric.TOLERANCE_BOUND, + numeric=Numeric.PLATFORM_BITWISE, seed_source=seed_source, dependencies=FIT_QRF_DEPENDENCIES, ) diff --git a/packages/microcosm-fit/src/microcosm/fit/qrf.py b/packages/microcosm-fit/src/microcosm/fit/qrf.py index 6d76bffe0..a39c57d2a 100644 --- a/packages/microcosm-fit/src/microcosm/fit/qrf.py +++ b/packages/microcosm-fit/src/microcosm/fit/qrf.py @@ -47,6 +47,7 @@ from __future__ import annotations +import copy import hashlib import json import os @@ -264,6 +265,9 @@ def _interp_rows( return values_lo + weight * (values_hi - values_lo) +_SERIALIZED_FOREST_N_JOBS = 1 + + @dataclass(frozen=True) class _Forest: """A fitted quantile forest plus the feature columns it was fit on.""" @@ -271,6 +275,33 @@ class _Forest: model: RandomForestQuantileRegressor columns: tuple[str, ...] + def __getstate__(self) -> dict[str, object]: + """Return a worker-count-neutral pickle payload. + + ``RandomForestQuantileRegressor.n_jobs`` controls only runtime + parallelism. A shallow model copy lets serialization pin that field + without mutating the fitted object that will perform the first draw. + """ + + model = copy.copy(self.model) + model.n_jobs = _SERIALIZED_FOREST_N_JOBS + return {"model": model, "columns": self.columns} + + def __setstate__(self, state: Mapping[str, object]) -> None: + """Restore the current runtime worker setting after trusted loading.""" + + model = state["model"] + if not isinstance(model, RandomForestQuantileRegressor): + raise TypeError("Serialized QRF forest model has an invalid type.") + columns = state["columns"] + if not isinstance(columns, tuple) or any( + not isinstance(column, str) for column in columns + ): + raise TypeError("Serialized QRF forest columns are invalid.") + model.n_jobs = _fit_n_jobs() + object.__setattr__(self, "model", model) + object.__setattr__(self, "columns", columns) + def draw(self, frame: pd.DataFrame, quantiles: np.ndarray) -> np.ndarray: """Draw one value per row at that row's quantile. diff --git a/packages/microcosm-fit/tests/test_kernels.py b/packages/microcosm-fit/tests/test_kernels.py index 356705a7b..dba6fc9e5 100644 --- a/packages/microcosm-fit/tests/test_kernels.py +++ b/packages/microcosm-fit/tests/test_kernels.py @@ -230,7 +230,7 @@ def test_capabilities_protocol_and_wrapped_source_hash() -> None: assert QRF_PARAM_KERNEL.ref == QRF_EXECUTOR_KERNEL.ref == "fit.qrf@1" assert QRF_PARAM_KERNEL.capabilities == Capabilities( determinism=Determinism.SEEDED, - numeric=Numeric.TOLERANCE_BOUND, + numeric=Numeric.PLATFORM_BITWISE, seed_source=SeedSource.PARAM, dependencies=FIT_QRF_DEPENDENCIES, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index 696158e73..aaa781484 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -13,6 +13,7 @@ DTYPES, GATE_OUTCOMES, MASS_POLICIES, + PARTITION_DTYPES, ROWS_ALL, WEIGHT_KINDS, CompiledGraph, @@ -45,15 +46,20 @@ KernelResult, KernelRole, Numeric, + NumericScope, SeedSource, + Tolerance, source_hash, ) +from .keys import platform_fingerprint __all__ = [ + "platform_fingerprint", "DESCRIPTIVE_FIELDS", "DTYPES", "GATE_OUTCOMES", "MASS_POLICIES", + "PARTITION_DTYPES", "ROWS_ALL", "WEIGHT_KINDS", "Capabilities", @@ -80,11 +86,14 @@ "Ownership", "Param", "Population", + "PopulationView", "PopulationError", "ResumePolicy", "RunManifest", "SOURCE_CODECS", "SeedSource", + "NumericScope", + "Tolerance", "Slice", "SourceCodec", "SourceCodecRegistry", @@ -133,7 +142,7 @@ def _check_frame_version() -> None: ) from .executor import NodeRejected, run_graph # noqa: E402 from .explain import explain_html # noqa: E402 -from .manifest import Decision, NodeReceipt, RunManifest # noqa: E402 +from .manifest import Decision, NodeReceipt, PopulationView, RunManifest # noqa: E402 from .population import MassRecord, Population, PopulationError # noqa: E402 from .serialize import graph_from_json, graph_to_json # noqa: E402 from .store import ( # noqa: E402 diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index b7460730b..b25e4f621 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -24,6 +24,17 @@ known at compile time: a mask that is not ``bool`` or ``boolean`` is a compile error (charter D4). Nulls inside a nullable mask are a run-time rejection by the executor. +- An ``EXPAND`` node copies rows: every new row names the base row it + copies, so lineage is total. A node that declares ``entrants=True`` may + also add rows that copy nothing (births not patterned on a parent, + immigrant cohorts); the kernel materializes every carried column for + such a row, the executor records them as entrants rather than copies, + and the node cannot claim to conserve mass (amendment 11). +- Mass is accounted as weighted person mass per stratum. A graph may name + a partition column (:attr:`Graph.mass_partition`, e.g. a period on a + person-period population); the ledger then reports per stratum within + each partition and ``conserve`` holds within each partition, so a row + contributes mass only to the partitions it exists in (amendment 12). This file is a frozen interface (see ``docs/graph-acceptance.md``). """ @@ -42,6 +53,7 @@ "GATE_OUTCOMES", "MASK_DTYPES", "MASS_POLICIES", + "PARTITION_DTYPES", "ROWS_ALL", "WEIGHT_KINDS", "CompiledGraph", @@ -82,6 +94,9 @@ #: Mass policies a weight transition or structural node may declare. MASS_POLICIES = frozenset({"conserve", "free", "declared"}) +#: The dtypes a mass-partition column may have. +PARTITION_DTYPES = frozenset({"int32", "int64", "string"}) + #: The closed set of gate outcomes (charter F4). ``unreached`` is also the #: outcome of a release whose required human decisions are absent. GATE_OUTCOMES = ("pass", "fail", "evidence_absent", "not_applicable", "unreached") @@ -130,6 +145,17 @@ def _nonempty(label: str, value: str) -> None: raise GraphError(f"{label} must be a non-empty string, got {value!r}.") +def _name(label: str, value: str) -> None: + """An entity or column name: non-empty and free of ``.``. + + Receipts, keys, and evidence spell a coordinate ``entity.column``; a dot + inside either part would make two coordinates collide (amendment 15). + """ + _nonempty(label, value) + if "." in value: + raise GraphError(f"{label} may not contain '.', got {value!r}.") + + @dataclass(frozen=True) class SourceRef: """A named external input, identified by content at run time. @@ -165,14 +191,14 @@ class Slice: rows: str = ROWS_ALL def __post_init__(self) -> None: - _nonempty("Slice.entity", self.entity) + _name("Slice.entity", self.entity) if not self.columns: raise GraphError(f"Slice on {self.entity!r} declares no columns.") if len(set(self.columns)) != len(self.columns): raise GraphError(f"Slice on {self.entity!r} repeats a column.") for column in self.columns: - _nonempty("Slice.columns[]", column) - _nonempty("Slice.rows", self.rows) + _name("Slice.columns[]", column) + _name("Slice.rows", self.rows) @dataclass(frozen=True) @@ -202,14 +228,14 @@ class Owned: rewrite: bool = False def __post_init__(self) -> None: - _nonempty("Owned.entity", self.entity) - _nonempty("Owned.column", self.column) + _name("Owned.entity", self.entity) + _name("Owned.column", self.column) if self.dtype not in DTYPES: raise GraphError( f"Owned {self.entity}.{self.column}: dtype {self.dtype!r} is not " f"one of {sorted(DTYPES)}." ) - _nonempty("Owned.rows", self.rows) + _name("Owned.rows", self.rows) if not isinstance(self.ownership, Ownership): raise GraphError("Owned.ownership must be an Ownership value.") @@ -233,7 +259,7 @@ class WeightTransition: mass: str = "conserve" def __post_init__(self) -> None: - _nonempty("WeightTransition.entity", self.entity) + _name("WeightTransition.entity", self.entity) if self.to_kind not in WEIGHT_KINDS: raise GraphError( f"WeightTransition.to_kind {self.to_kind!r} is not one of " @@ -269,6 +295,11 @@ class Node: sources: Names of :class:`SourceRef` entries this node reads. weights: A declared weight-kind transition, if any. mass: Mass policy for structural nodes that change rows or weights. + entrants: ``EXPAND`` nodes only: the kernel may add rows that copy + no base row. Such a row has null lineage, the kernel supplies + every carried column for it, and the executor records it as an + entrant. Entrants add mass, so the node's mass policy cannot be + ``conserve``. description: Descriptive; never hashed. citation: Descriptive; never hashed. """ @@ -286,6 +317,7 @@ class Node: mass: str = "conserve" description: str = "" citation: str = "" + entrants: bool = False def __post_init__(self) -> None: _nonempty("Node.id", self.id) @@ -333,6 +365,17 @@ def __post_init__(self) -> None: ) elif self.base is not None: raise GraphError(f"Node {self.id!r}: only structural nodes have a base.") + if not isinstance(self.entrants, bool): + raise GraphError(f"Node {self.id!r}: entrants must be a boolean.") + if self.entrants and self.structural is not StructuralDelta.EXPAND: + raise GraphError( + f"Node {self.id!r}: only an EXPAND node may admit entrants." + ) + if self.entrants and self.mass == "conserve": + raise GraphError( + f"Node {self.id!r}: entrants add mass, so an entrant-admitting " + "node cannot declare mass='conserve'." + ) if self.weights is not None and self.structural is not StructuralDelta.REWEIGHT: raise GraphError( f"Node {self.id!r}: a weight transition changes the population " @@ -386,11 +429,18 @@ class Graph: into node keys (a node's identity is its computation). sources: External inputs by name. nodes: Every node. Declaration order carries no meaning. + mass_partition: ``(entity, column)`` of a column that partitions + mass accounting, or ``None``. When set, every ``CREATE`` node + declares the column with a dtype in :data:`PARTITION_DTYPES`, + the executor's ledger reports per stratum within each partition + value, and ``conserve`` holds within each partition. Normative: + it enters the key of every structural node. """ country: str sources: tuple[SourceRef, ...] nodes: tuple[Node, ...] + mass_partition: tuple[str, str] | None = None def __post_init__(self) -> None: _nonempty("Graph.country", self.country) @@ -398,6 +448,21 @@ def __post_init__(self) -> None: raise GraphError("Graph repeats a source name.") if len({n.id for n in self.nodes}) != len(self.nodes): raise GraphError("Graph repeats a node id.") + if self.mass_partition is not None: + if ( + not isinstance(self.mass_partition, tuple) + or len(self.mass_partition) != 2 + or not all(isinstance(part, str) for part in self.mass_partition) + ): + raise GraphError( + "Graph.mass_partition must be an (entity, column) pair of strings." + ) + _name("Graph.mass_partition entity", self.mass_partition[0]) + _name("Graph.mass_partition column", self.mass_partition[1]) + + def normative(self) -> dict[str, object]: + """The graph-level facts that enter every structural node's key.""" + return {"mass_partition": self.mass_partition} def node(self, node_id: str) -> Node: for node in self.nodes: @@ -436,8 +501,9 @@ def compile_graph(graph: Graph) -> CompiledGraph: GraphError: A cell with two owners or none (ownership is total and exclusive), an unknown source or population, a structural node whose base is not structural, a row mask whose declared dtype is - not boolean, a cycle, or a graph with several structural nodes - and a node that omits ``population``. + not boolean, a mass-partition column that a ``CREATE`` node does + not declare with a partition dtype, a cycle, or a graph with + several structural nodes and a node that omits ``population``. """ by_id = {node.id: node for node in graph.nodes} @@ -494,6 +560,37 @@ def compile_graph(graph: Graph) -> CompiledGraph: owners[key] = node.id dtypes[key] = owned.dtype + if graph.mass_partition is not None: + entity, column = graph.mass_partition + for node in structural: + if node.structural is not StructuralDelta.CREATE: + continue + dtype = dtypes.get((node.id, entity, column)) + if dtype is None: + raise GraphError( + f"Graph.mass_partition names {entity}.{column}, which CREATE " + f"node {node.id!r} does not declare; partitions must exist " + "from the first version." + ) + if dtype not in PARTITION_DTYPES: + raise GraphError( + f"Graph.mass_partition {entity}.{column} is declared {dtype!r}; " + f"a partition column must be one of {sorted(PARTITION_DTYPES)}." + ) + # A partition value is fixed when a row is created. Any later owner of + # the column, rewrite or not, could move mass between partitions with + # the total unchanged, which no mass policy can see; refuse it here. + for (version, owner_entity, owner_column), owner_id in sorted(owners.items()): + if (owner_entity, owner_column) != (entity, column): + continue + if by_id[owner_id].structural is not StructuralDelta.CREATE: + raise GraphError( + f"Node {owner_id!r} owns mass partition {entity}.{column} in " + f"version {version!r}; a partition value is fixed by the CREATE " + "node that admits the row, and no later node may write or " + "rewrite it." + ) + def declared_dtype(version: str, entity: str, column: str) -> str | None: """The owner-declared dtype of a column as visible in ``version``.""" while True: diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 6bb035966..689c1cf9a 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -16,6 +16,7 @@ from microcosm.frame import Frame, WeightKind, Weights +from . import keys as graph_keys from .canonical import canonical_json, sha256_domain from .codecs import SOURCE_CODECS, SourceCodecRegistry from .decl import ( @@ -34,8 +35,12 @@ KernelRegistry, KernelResult, KernelRole, + Numeric, + NumericScope, + Tolerance, ) from .keys import ( + _capabilities_projection, artifact_key, frame_key, node_key, @@ -46,7 +51,11 @@ from .manifest import Decision, NodeReceipt, RunManifest from .population import ( Population, + _expand_cells, + entrant_strata_receipt, expand_lineage_receipt, + expand_writes_receipt, + mass_record_receipt, patch, restore_cached_expand, weight_cap_receipt, @@ -66,6 +75,7 @@ NodeRejected = NodeRejectedError _CERTIFYING_GATE_OUTCOMES = frozenset({"pass", "not_applicable"}) +_EXPAND_WRITE_CLASSES = ("entrant", "copied-rewrite", "new-column") def _now() -> str: @@ -80,18 +90,6 @@ def _opaque_artifact_key(key: str, name: str) -> str: return sha256_domain("node-artifact", canonical_json((key, name))) -def _capabilities_payload(capabilities: Capabilities) -> dict[str, object]: - return { - "determinism": capabilities.determinism.value, - "numeric": capabilities.numeric.value, - "seed_source": capabilities.seed_source.value, - "structural": capabilities.structural.value, - "role": capabilities.role.value, - "consumes_se": capabilities.consumes_se, - "dependencies": list(capabilities.dependencies), - } - - def _normal_json_mapping(value: Mapping[str, object], label: str) -> dict[str, object]: """Validate and detach a descriptive mapping through canonical JSON.""" @@ -203,7 +201,9 @@ def _decision_names(decisions: tuple[Decision, ...]) -> frozenset[str]: return frozenset(names) -def _release_outcome(node: Node, tier: str, decisions: tuple[Decision, ...]) -> str: +def _required_decision_names(node: Node) -> tuple[str, ...]: + """Validate and return a release node's normative decision requirements.""" + required = node.params.get("requires_decisions", ()) if not isinstance(required, tuple) or any( not isinstance(name, str) or not name for name in required @@ -216,6 +216,11 @@ def _release_outcome(node: Node, tier: str, decisions: tuple[Decision, ...]) -> raise NodeRejected( f"Release node {node.id!r} repeats a required decision name." ) + return required + + +def _release_outcome(node: Node, tier: str, decisions: tuple[Decision, ...]) -> str: + required = _required_decision_names(node) if not set(required) <= _decision_names(decisions): return "unreached" return "pass" if tier == "certified" else "fail" @@ -394,12 +399,68 @@ def _structural_columns(frame: Frame, entity: str) -> list[str]: return columns +def _materialized_expand_coordinates(node: Node) -> frozenset[tuple[str, str]]: + """Return and validate the carried EXPAND cells an ordinary node claims.""" + + raw_materialized = node.params.get("materialized_expand_outputs", ()) + if not isinstance(raw_materialized, tuple) or any( + not isinstance(value, str) or "." not in value for value in raw_materialized + ): + raise NodeRejected( + f"Node {node.id!r} params['materialized_expand_outputs'] must be a " + "tuple of 'entity.column' strings." + ) + materialized: set[tuple[str, str]] = set() + owned_by_coordinate = { + (output.entity, output.column): output for output in node.outputs + } + for value in raw_materialized: + entity, column = value.split(".", 1) + coordinate = (entity, column) + output = owned_by_coordinate.get(coordinate) + if output is None or output.rewrite: + raise NodeRejected( + f"Node {node.id!r} materialized EXPAND output {value!r} must be " + "one of its non-rewrite owned cells." + ) + materialized.add(coordinate) + if len(materialized) != len(raw_materialized): + raise NodeRejected(f"Node {node.id!r} repeats a materialized EXPAND output.") + return frozenset(materialized) + + +def _expand_rewrite_coordinates( + compiled: CompiledGraph, node: Node +) -> frozenset[tuple[str, str]]: + """Return overlays an EXPAND's full-cell same-version claimant rewrites.""" + + if node.structural is not StructuralDelta.EXPAND: + return frozenset() + overlay_coordinates = _expand_writer_coordinates(node) + rewrites: set[tuple[str, str]] = set() + for (version, entity, column), owner_id in compiled.owners.items(): + coordinate = (entity, column) + if version != node.id or coordinate not in overlay_coordinates: + continue + owner = compiled.graph.node(owner_id) + output = next( + candidate + for candidate in owner.outputs + if (candidate.entity, candidate.column) == coordinate + ) + if output.rewrite and output.rows == ROWS_ALL: + rewrites.add(coordinate) + return frozenset(rewrites) + + def _project_context( node: Node, population: Population | None, *, key: str, sources: Mapping[str, Path], + tolerances: Mapping[tuple[str, str], Tolerance | None], + numerics: Mapping[tuple[str, str], NumericScope], ) -> KernelContext: if population is None: return KernelContext( @@ -410,6 +471,8 @@ def _project_context( params=node.params, rng=np.random.default_rng(seed(key)), sources=MappingProxyType({name: sources[name] for name in node.sources}), + tolerances=tolerances, + numerics=numerics, ) frame = population.frame @@ -417,35 +480,15 @@ def _project_context( for slice_ in node.inputs: slices.setdefault(slice_.entity, []).append(slice_) - raw_materialized = node.params.get("materialized_expand_outputs", ()) - if not isinstance(raw_materialized, tuple) or any( - not isinstance(value, str) or "." not in value for value in raw_materialized - ): - raise NodeRejected( - f"Node {node.id!r} params['materialized_expand_outputs'] must be a " - "tuple of 'entity.column' strings." - ) - materialized: set[tuple[str, str]] = set() - owned_by_coordinate = { - (output.entity, output.column): output for output in node.outputs - } - for value in raw_materialized: - entity, column = value.split(".", 1) + materialized = _materialized_expand_coordinates(node) + for entity, column in materialized: coordinate = (entity, column) - output = owned_by_coordinate.get(coordinate) - if output is None or output.rewrite: - raise NodeRejected( - f"Node {node.id!r} materialized EXPAND output {value!r} must be " - "one of its non-rewrite owned cells." - ) if population.owners.get(coordinate) != population.version: raise NodeRejected( - f"Node {node.id!r} materialized EXPAND output {value!r} was not " + f"Node {node.id!r} materialized EXPAND output " + f"{entity}.{column!s} was not " f"installed by population version {population.version!r}." ) - materialized.add(coordinate) - if len(materialized) != len(raw_materialized): - raise NodeRejected(f"Node {node.id!r} repeats a materialized EXPAND output.") tables: dict[str, pd.DataFrame] = {} entity_masks: dict[str, np.ndarray] = {} @@ -519,9 +562,319 @@ def _project_context( params=node.params, rng=np.random.default_rng(seed(key)), sources=MappingProxyType({name: sources[name] for name in node.sources}), + tolerances=tolerances, + numerics=numerics, + ) + + +_NUMERIC_RANK = { + Numeric.BITWISE: 0, + Numeric.PLATFORM_BITWISE: 1, + Numeric.TOLERANCE_BOUND: 2, +} + + +def _input_numerics( + compiled: CompiledGraph, + node_id: str, + kernels: KernelRegistry, + *, + writers: Mapping[tuple[str, str], tuple[str, ...]] | None = None, +) -> Mapping[tuple[str, str], NumericScope]: + """Resolve each read coordinate to its loosest writer numeric scope.""" + + writer_map = _input_writers(compiled, node_id) if writers is None else writers + resolved: dict[tuple[str, str], NumericScope] = {} + for coordinate, writer_ids in writer_map.items(): + capabilities = tuple( + kernels.get(compiled.graph.node(writer_id).kernel).capabilities + for writer_id in writer_ids + ) + numeric = max( + (capability.numeric for capability in capabilities), + key=_NUMERIC_RANK.__getitem__, + default=Numeric.BITWISE, + ) + bounds = tuple( + capability.tolerance + for capability in capabilities + if capability.numeric is Numeric.TOLERANCE_BOUND + ) + tolerance = ( + None + if not bounds + else Tolerance( + rtol=max(bound.rtol for bound in bounds if bound is not None), + atol=max(bound.atol for bound in bounds if bound is not None), + ulps=max(bound.ulps for bound in bounds if bound is not None), + ) + ) + platform = ( + graph_keys.platform_fingerprint() + if any( + capability.numeric is Numeric.PLATFORM_BITWISE + for capability in capabilities + ) + else None + ) + resolved[coordinate] = NumericScope( + numeric=numeric, + tolerance=tolerance, + platform=platform, + ) + return MappingProxyType(resolved) + + +def _input_tolerances( + compiled: CompiledGraph, + node_id: str, + kernels: KernelRegistry, + *, + writers: Mapping[tuple[str, str], tuple[str, ...]] | None = None, + numerics: Mapping[tuple[str, str], NumericScope] | None = None, +) -> Mapping[tuple[str, str], Tolerance | None]: + """Project each input numeric scope to its legacy tolerance value.""" + + scopes = ( + _input_numerics(compiled, node_id, kernels, writers=writers) + if numerics is None + else numerics + ) + return MappingProxyType( + {coordinate: scope.tolerance for coordinate, scope in scopes.items()} ) +def _input_writers( + compiled: CompiledGraph, + node_id: str, + *, + receipts: Mapping[str, NodeReceipt] | None = None, +) -> Mapping[tuple[str, str], tuple[str, ...]]: + """Return causal writer lists for explicit, rewrite, and claim reads.""" + + node = compiled.graph.node(node_id) + if node.structural is StructuralDelta.CREATE: + return MappingProxyType({}) + input_version = ( + compiled.versions[node_id] + if node.structural is StructuralDelta.NONE + else node.base + ) + assert input_version is not None + coordinates = { + (owned.entity, owned.column) for owned in node.outputs if owned.rewrite + } + coordinates.update(_materialized_expand_coordinates(node)) + coordinates.update( + (slice_.entity, column) for slice_ in node.inputs for column in slice_.columns + ) + writers: dict[tuple[str, str], tuple[str, ...]] = {} + for coordinate in sorted(coordinates): + entity, column = coordinate + writers[coordinate] = _writers_of( + compiled, + input_version, + entity, + column, + exclude_node=node.id, + receipts=receipts, + ) + return MappingProxyType(writers) + + +def _expand_writer_coordinates(node: Node) -> frozenset[tuple[str, str]]: + """Coordinates an EXPAND declares it may materialize.""" + + if node.structural is not StructuralDelta.EXPAND: + return frozenset() + return frozenset((entity, column) for entity, column, _dtype in _expand_cells(node)) + + +def _expand_declared_payload(node: Node) -> list[str]: + """Canonical receipt spellings of every declared EXPAND coordinate.""" + + return [ + f"{entity}.{column}" + for entity, column in sorted(_expand_writer_coordinates(node)) + ] + + +def _parse_expand_declared(node: Node, raw: object) -> frozenset[tuple[str, str]]: + """Validate the executor-authored EXPAND declaration attestation.""" + + expected = tuple(_expand_declared_payload(node)) + if not isinstance(raw, list | tuple) or tuple(raw) != expected: + raise ValueError( + f"EXPAND node {node.id!r} expand_declared must exactly equal its " + f"canonical declaration {expected!r}." + ) + return _expand_writer_coordinates(node) + + +def _parse_expand_writes( + node: Node, raw: object +) -> Mapping[tuple[str, str], tuple[str, ...]]: + """Validate an executor-authored EXPAND coordinate/row-class record.""" + + if not isinstance(raw, Mapping): + raise ValueError(f"EXPAND node {node.id!r} expand_writes must be a mapping.") + declared = _expand_writer_coordinates(node) + parsed: dict[tuple[str, str], tuple[str, ...]] = {} + for spelling, raw_classes in raw.items(): + if not isinstance(spelling, str) or spelling.count(".") != 1: + raise ValueError( + f"EXPAND node {node.id!r} expand_writes coordinate {spelling!r} " + "must be an 'entity.column' string." + ) + entity, column = spelling.split(".") + coordinate = (entity, column) + if coordinate not in declared: + raise ValueError( + f"EXPAND node {node.id!r} expand_writes names undeclared " + f"coordinate {spelling!r}." + ) + if not isinstance(raw_classes, list | tuple) or not raw_classes: + raise ValueError( + f"EXPAND node {node.id!r} expand_writes {spelling!r} must name " + "at least one row class." + ) + classes = tuple(raw_classes) + if any(not isinstance(value, str) for value in classes): + raise ValueError( + f"EXPAND node {node.id!r} expand_writes {spelling!r} row classes " + "must be strings." + ) + canonical = tuple(value for value in _EXPAND_WRITE_CLASSES if value in classes) + if classes != canonical: + raise ValueError( + f"EXPAND node {node.id!r} expand_writes {spelling!r} row classes " + f"must be unique and ordered as {_EXPAND_WRITE_CLASSES!r}." + ) + parsed[coordinate] = classes + return MappingProxyType(parsed) + + +def _validate_materialized_expand_outputs( + compiled: CompiledGraph, + node: Node, + population: Population | None, + receipts: Mapping[str, NodeReceipt], +) -> None: + """Bind the no-Slice materialization bridge to its immediate EXPAND.""" + + if "materialized_expand_outputs" not in node.params: + return + materialized = _materialized_expand_coordinates(node) + if population is None: + raise NodeRejected( + f"Node {node.id!r} uses materialized_expand_outputs without an " + "incumbent population; an immediate EXPAND population is required." + ) + holder = compiled.graph.node(population.version) + if holder.structural is not StructuralDelta.EXPAND: + raise NodeRejected( + f"Node {node.id!r} uses materialized_expand_outputs on population " + f"version {population.version!r}, whose holder is {holder.structural.name}; " + "an immediate EXPAND population is required." + ) + holder_receipt = receipts.get(holder.id) + if holder_receipt is None: # compiled population ancestry should prevent this + raise NodeRejected( + f"Node {node.id!r} cannot validate materialized_expand_outputs: " + f"EXPAND population version {holder.id!r} has no runtime receipt." + ) + try: + expand_declared = _parse_expand_declared( + holder, holder_receipt.receipt.get("expand_declared") + ) + except ValueError as error: # executor-authored receipts cannot be malformed + raise NodeRejected( + f"Node {node.id!r} cannot validate materialized_expand_outputs for " + f"EXPAND population version {holder.id!r}: {error}" + ) from error + for entity, column in sorted(materialized): + if (entity, column) not in expand_declared: + raise NodeRejected( + f"Node {node.id!r} names materialized EXPAND output " + f"{entity}.{column}, but EXPAND population version {holder.id!r} " + "did not declare that coordinate." + ) + + +def _writers_of( + compiled: CompiledGraph, + version: str, + entity: str, + column: str, + *, + exclude_node: str | None = None, + receipts: Mapping[str, NodeReceipt] | None = None, +) -> tuple[str, ...]: + """All nodes that wrote rows of ``entity.column`` as seen from ``version``. + + The result is in causal order: the originating producer, EXPAND + materializers, rewrites, and materialization claimants. Structural nodes + that only carry the coordinate do not appear. + """ + + coordinate = (entity, column) + newest_first: list[str] = [] + + def add(writer_id: str) -> None: + if writer_id != exclude_node and writer_id not in newest_first: + newest_first.append(writer_id) + + while True: + holder = compiled.graph.node(version) + owner_id = compiled.owners.get((version, entity, column)) + if owner_id is not None: + owner = compiled.graph.node(owner_id) + output = next( + owned + for owned in owner.outputs + if (owned.entity, owned.column) == coordinate + ) + inherited = ( + output.rewrite + or output.rows != ROWS_ALL + or coordinate in _materialized_expand_coordinates(owner) + ) + add(owner_id) + if not inherited: + break + + if _expand_wrote_rows(holder, coordinate, receipts): + add(holder.id) + if holder.structural is StructuralDelta.CREATE or holder.base is None: + break + version = holder.base + return tuple(reversed(newest_first)) + + +def _expand_wrote_rows( + node: Node, + coordinate: tuple[str, str], + receipts: Mapping[str, NodeReceipt] | None, +) -> bool: + """Whether this EXPAND actually wrote any row of a coordinate.""" + + if coordinate not in _expand_writer_coordinates(node): + return False + if receipts is None: + # Static preflight has no runtime receipt with which to refine the + # declaration. Exact writer ids are checked during execution. + return True + node_receipt = receipts.get(node.id) + if node_receipt is None: + return False + try: + writes = _parse_expand_writes(node, node_receipt.receipt.get("expand_writes")) + except ValueError as error: # executor-authored receipts cannot be malformed + raise NodeRejected(str(error)) from error + return coordinate in writes + + def _validate_series( node: Node, owned: Owned, @@ -591,6 +944,18 @@ def _validate_create(node: Node, frame: Frame) -> None: raise NodeRejected( f"Node {node.id!r} returned an invalid Frame: {error}" ) from error + # Amendment 15: every name the graph will spell as entity.column is dot-free. + dotted_names = sorted( + name + for entity in frame.entities + for name in (entity, *map(str, frame.table(entity).columns)) + if "." in name + ) + if dotted_names: + raise NodeRejected( + f"CREATE node {node.id!r} returned a Frame with dotted names " + f"{dotted_names[:5]}; entity and column names may not contain '.'." + ) expected_columns = {(owned.entity, owned.column) for owned in node.outputs} actual_columns = { (entity, str(column)) @@ -645,6 +1010,15 @@ def _validate_result( raise NodeRejected(f"Node {node.id!r} result.artifacts is not a mapping.") if not isinstance(result.receipt, Mapping): raise NodeRejected(f"Node {node.id!r} result.receipt is not a mapping.") + if result.strata is not None and not isinstance(result.strata, pd.Series): + raise NodeRejected(f"Node {node.id!r} result.strata is not a Series.") + if result.strata is not None and ( + cache_hit or node.structural is not StructuralDelta.EXPAND or not node.entrants + ): + raise NodeRejected( + f"Node {node.id!r} returned entrant strata outside a fresh " + "entrants=True EXPAND." + ) if kernel_capabilities.structural is not node.structural: raise NodeRejected( f"Node {node.id!r} declares structural={node.structural.value!r}, but " @@ -763,10 +1137,18 @@ def _validate_result( assert result.expand is not None try: receipt["expand"] = expand_lineage_receipt(result.expand) + assert population is not None + strata_receipt = entrant_strata_receipt( + population.frame, node, result.expand, result.strata + ) except (TypeError, ValueError) as error: raise NodeRejected( - f"EXPAND node {node.id!r} returned malformed lineage: {error}" + f"EXPAND node {node.id!r} returned malformed lineage or " + f"entrant strata: {error}" ) from error + receipt.pop("entrant_strata", None) + if strata_receipt is not None: + receipt["entrant_strata"] = strata_receipt if kernel_capabilities.role is KernelRole.GATE: outcome = receipt.get("outcome") if outcome not in GATE_OUTCOMES: @@ -777,6 +1159,86 @@ def _validate_result( return receipt, artifacts +def _validate_entrant_materialization_contract( + compiled: CompiledGraph, + node: Node, + population: Population | None, + receipt: Mapping[str, object], +) -> None: + """Require every entrant's carried data cells to have downstream claims.""" + + if not node.entrants or population is None: + return + raw_expand = receipt.get("expand") + if not isinstance(raw_expand, Mapping): + return # the ordinary EXPAND validation reports the malformed receipt + entrant_entities: set[str] = set() + for entity, entries in raw_expand.items(): + if not isinstance(entity, str) or not isinstance(entries, list): + continue + if any( + isinstance(entry, list) and len(entry) == 2 and entry[1] is None + for entry in entries + ): + entrant_entities.add(entity) + + frame = population.frame + for entity in sorted(entrant_entities): + if entity not in frame.entities: + continue # lineage validation supplies the node-naming rejection + structural = set(_structural_columns(frame, entity)) + if ( + compiled.graph.mass_partition is not None + and compiled.graph.mass_partition[0] == entity + ): + structural.add(compiled.graph.mass_partition[1]) + for column in frame.table(entity).columns: + column = str(column) + if column in structural: + continue + coordinate = (entity, column) + claimant_id = compiled.owners.get((node.id, entity, column)) + if claimant_id is None: + raise NodeRejected( + f"EXPAND node {node.id!r} entrant cell {entity}.{column} " + "has no materialized_expand_outputs ownership claim." + ) + claimant = compiled.graph.node(claimant_id) + claimed = claimant.params.get("materialized_expand_outputs", ()) + spelling = f"{entity}.{column}" + output = next( + ( + owned + for owned in claimant.outputs + if (owned.entity, owned.column) == coordinate + ), + None, + ) + if ( + not isinstance(claimed, tuple) + or spelling not in claimed + or output is None + or output.rewrite + ): + raise NodeRejected( + f"EXPAND node {node.id!r} entrant cell {spelling} is not " + f"declared through node {claimant_id!r}'s " + "materialized_expand_outputs." + ) + if output.rows != ROWS_ALL: + raise NodeRejected( + f"EXPAND node {node.id!r} entrant cell {spelling} is claimed " + f"through masked rows {output.rows!r}; materialization bridge " + "claims must use rows='all'." + ) + carried_dtype = _dtype_token(frame.table(entity)[column]) + if output.dtype != carried_dtype: + raise NodeRejected( + f"EXPAND node {node.id!r} entrant cell {spelling} is claimed " + f"as {output.dtype!r}; its carried dtype is {carried_dtype!r}." + ) + + def _create_population(node: Node, frame: Frame) -> Population: # Entity ids and membership columns are structural Frame columns rather # than declaration-owned data cells, but Population ownership is total @@ -816,7 +1278,21 @@ def _apply_result( population: Population | None, *, cache_hit: bool = False, + mass_partition: tuple[str, str] | None = None, + rewrite_coordinates: frozenset[tuple[str, str]] = frozenset(), ) -> Population: + if ( + mass_partition is not None + and node.structural is StructuralDelta.NONE + and any( + (owned.entity, owned.column) == mass_partition for owned in node.outputs + ) + ): + entity, column = mass_partition + raise NodeRejected( + f"Node {node.id!r} cannot own mass partition {entity}.{column}; " + "partition values are fixed by the structural population." + ) if node.structural is StructuralDelta.CREATE: assert result.frame is not None return _create_population(node, result.frame) @@ -827,7 +1303,13 @@ def _apply_result( and result.frame is not None ): try: - return restore_cached_expand(population, node, result) + return restore_cached_expand( + population, + node, + result, + mass_partition=mass_partition, + rewrite_coordinates=rewrite_coordinates, + ) except (TypeError, ValueError) as error: raise NodeRejected( f"Node {node.id!r} cached EXPAND rejected: {error}" @@ -860,7 +1342,13 @@ def _apply_result( receipt=result.receipt, ) try: - return patch(population, node, result) + return patch( + population, + node, + result, + mass_partition=mass_partition, + rewrite_coordinates=rewrite_coordinates, + ) except NodeRejected: raise except (TypeError, ValueError) as error: @@ -979,7 +1467,7 @@ def _write_node( "node_key": key, "kernel_ref": node.kernel, "kernel_impl_hash": kernel_impl_hash, - "capabilities": _capabilities_payload(capabilities), + "capabilities": _capabilities_projection(capabilities), "receipt": dict(receipt), "columns": column_entries, "frame_key": stored_frame_key, @@ -996,7 +1484,12 @@ def _write_node( def _require_record_shape( - raw: object, node: Node, *, key: str, kernel_impl_hash: str + raw: object, + node: Node, + *, + key: str, + kernel_impl_hash: str, + capabilities: Capabilities, ) -> dict[str, object]: if not isinstance(raw, dict): raise StoreCorrupt(f"Cached receipt for node {node.id!r} is not an object.") @@ -1035,6 +1528,40 @@ def _require_record_shape( f"Cached receipt identity for node {node.id!r} is {actual!r}, " f"not {expected!r}." ) + expected_capabilities = _capabilities_projection(capabilities) + stored_capabilities = raw["capabilities"] + if ( + isinstance(stored_capabilities, Mapping) + and "tolerance" not in stored_capabilities + ): + raise StoreMiss( + f"Cached receipt for node {node.id!r} has legacy_capabilities: " + "the schema-v1 contract omits tolerance." + ) + if raw["capabilities"] != expected_capabilities: + raise StoreMiss( + f"Cached receipt capabilities for node {node.id!r} disagree with " + "the registered kernel contract." + ) + if node.structural is StructuralDelta.EXPAND: + raw_receipt = raw["receipt"] + if not isinstance(raw_receipt, Mapping): + raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") + if "expand_writes" not in raw_receipt: + raise StoreMiss( + f"Cached EXPAND node {node.id!r} predates expand_writes provenance." + ) + if "expand_declared" not in raw_receipt: + raise StoreMiss( + f"Cached EXPAND node {node.id!r} predates expand_declared provenance." + ) + try: + _parse_expand_declared(node, raw_receipt["expand_declared"]) + _parse_expand_writes(node, raw_receipt["expand_writes"]) + except ValueError as error: + raise StoreCorrupt( + f"Cached EXPAND node {node.id!r} has malformed EXPAND provenance." + ) from error return raw @@ -1053,9 +1580,75 @@ def _load_record( *, key: str, kernel_impl_hash: str, + capabilities: Capabilities, ) -> dict[str, object]: raw = store.load_json(_cache_record_key(key)) - return _require_record_shape(raw, node, key=key, kernel_impl_hash=kernel_impl_hash) + return _require_record_shape( + raw, + node, + key=key, + kernel_impl_hash=kernel_impl_hash, + capabilities=capabilities, + ) + + +def _tolerance_writer_payload( + writers: Mapping[tuple[str, str], tuple[str, ...]], +) -> dict[str, list[str]]: + return { + f"{entity}.{column}": list(writer_ids) + for (entity, column), writer_ids in writers.items() + } + + +def _require_tolerance_writer_receipt( + node: Node, + record: Mapping[str, object], + writers: Mapping[tuple[str, str], tuple[str, ...]], + *, + exact: bool, +) -> None: + """Reject cache receipts predating or disagreeing with writer provenance.""" + + expected = _tolerance_writer_payload(writers) + if not expected: + return + raw_receipt = record.get("receipt") + if not isinstance(raw_receipt, Mapping): + raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") + raw_capabilities = raw_receipt.get("capabilities") + actual = ( + raw_capabilities.get("tolerance_writers") + if isinstance(raw_capabilities, Mapping) + else None + ) + expected_coordinates = set(expected) + matches = ( + actual == expected + if exact + else (isinstance(actual, Mapping) and set(actual) == expected_coordinates) + ) + if not matches: + raise StoreMiss( + f"Cached node {node.id!r} has stale tolerance_writers provenance." + ) + + evidence = raw_receipt.get("evidence") + if isinstance(evidence, Mapping) and "tolerance" in evidence: + evidence_writers = evidence.get("tolerance_writers") + evidence_matches = ( + evidence_writers == expected + if exact + else ( + isinstance(evidence_writers, Mapping) + and set(evidence_writers) == expected_coordinates + ) + ) + if not evidence_matches: + raise StoreMiss( + f"Cached node {node.id!r} has stale evidence " + "tolerance_writers provenance." + ) def _preflight_record(store: ContentStore, record: Mapping[str, object]) -> None: @@ -1252,6 +1845,7 @@ def _all_node_keys( keys, implementation, source_keys, + kernel_capabilities=kernel.capabilities, ) return keys, implementations @@ -1261,6 +1855,7 @@ def _preflight_require( store: ContentStore, keys: Mapping[str, str], implementations: Mapping[str, str], + kernels: KernelRegistry, ) -> None: missing: list[str] = [] for node_id in compiled.order: @@ -1271,6 +1866,13 @@ def _preflight_require( node, key=keys[node_id], kernel_impl_hash=implementations[node_id], + capabilities=kernels.get(node.kernel).capabilities, + ) + _require_tolerance_writer_receipt( + node, + record, + _input_writers(compiled, node_id), + exact=False, ) _preflight_record(store, record) except StoreMiss: @@ -1282,6 +1884,21 @@ def _preflight_require( ) +def _preflight_expand_declarations(compiled: CompiledGraph) -> None: + """Reject malformed runtime EXPAND conventions before keys or cache I/O.""" + + for node_id in compiled.order: + node = compiled.graph.node(node_id) + if node.structural is not StructuralDelta.EXPAND: + continue + try: + _expand_writer_coordinates(node) + except (TypeError, ValueError) as error: + raise NodeRejected( + f"Node {node.id!r} expand_cells declaration rejected: {error}" + ) from error + + def run_graph( compiled: CompiledGraph, *, @@ -1305,11 +1922,12 @@ def run_graph( raise TypeError("decisions must contain Decision records or mappings.") decisions = tuple(normalized_decisions) + _preflight_expand_declarations(compiled) started_at = _now() source_paths, source_keys = _source_paths_and_keys(compiled, sources, store) keys, implementations = _all_node_keys(compiled, kernels, source_keys) if resume == "require": - _preflight_require(compiled, store, keys, implementations) + _preflight_require(compiled, store, keys, implementations, kernels) populations: dict[str, Population] = {} receipts: dict[str, NodeReceipt] = {} @@ -1333,8 +1951,22 @@ def run_graph( assert node.base is not None incumbent = populations[node.base] _validate_population_declaration(node, incumbent) + _validate_materialized_expand_outputs(compiled, node, incumbent, receipts) + input_writers = _input_writers(compiled, node_id, receipts=receipts) + input_numerics = _input_numerics( + compiled, node_id, kernels, writers=input_writers + ) + input_tolerances = _input_tolerances( + compiled, + node_id, + kernels, + writers=input_writers, + numerics=input_numerics, + ) + tolerance_writers = _tolerance_writer_payload(input_writers) hit = False + replace_stale_record = False result: KernelResult | None = None record: dict[str, object] | None = None manifest_artifacts: dict[tuple[str, str], str] = {} @@ -1345,17 +1977,35 @@ def run_graph( node, key=key, kernel_impl_hash=implementation, + capabilities=kernel.capabilities, ) + try: + _require_tolerance_writer_receipt( + node, record, input_writers, exact=True + ) + except StoreMiss: + # This key predates the writer-provenance contract or was + # produced for different runtime entrant lineage. + replace_stale_record = True + raise result, manifest_artifacts = _load_cached_result( store, node, incumbent, record ) hit = True except StoreMiss: + replace_stale_record = store.has(_cache_record_key(key)) if resume == "require": # defended by preflight; handles races raise if result is None: - context = _project_context(node, incumbent, key=key, sources=source_paths) + context = _project_context( + node, + incumbent, + key=key, + sources=source_paths, + tolerances=input_tolerances, + numerics=input_numerics, + ) before = _context_digest(context) try: result = kernel.run(context) @@ -1385,16 +2035,96 @@ def run_graph( incumbent, cache_hit=hit, ) + _validate_entrant_materialization_contract( + compiled, node, incumbent, normalized_receipt + ) if kernel.capabilities.role is KernelRole.RELEASE: derived_tier, gate_ids = _release_tier(compiled, node_id, receipts) _validate_release_tier(node, result, derived_tier) + required_decisions = _required_decision_names(node) normalized_receipt["tier"] = derived_tier normalized_receipt["outcome"] = ( "pass" if derived_tier == "certified" else "fail" ) normalized_receipt["gate_ancestry"] = list(gate_ids) - normalized_receipt["capabilities"] = _capabilities_payload(kernel.capabilities) - updated = _apply_result(node, result, incumbent, cache_hit=hit) + # Required names are derived from normative node params and live in + # authenticated release provenance. The signed records themselves + # remain top-level run provenance and never enter a node key. + normalized_receipt["requires_decisions"] = list(required_decisions) + receipt_capabilities = _capabilities_projection(kernel.capabilities) + if tolerance_writers: + receipt_capabilities["tolerance_writers"] = tolerance_writers + normalized_receipt["capabilities"] = receipt_capabilities + evidence = normalized_receipt.get("evidence") + if ( + tolerance_writers + and isinstance(evidence, Mapping) + and "tolerance" in evidence + ): + normalized_receipt["evidence"] = { + **evidence, + "tolerance_writers": tolerance_writers, + } + expand_rewrites = _expand_rewrite_coordinates(compiled, node) + updated = _apply_result( + node, + result, + incumbent, + cache_hit=hit, + mass_partition=compiled.graph.mass_partition, + rewrite_coordinates=expand_rewrites, + ) + if node.structural is StructuralDelta.EXPAND: + assert incumbent is not None + normalized_receipt["expand_declared"] = _expand_declared_payload(node) + try: + authored_expand_writes = expand_writes_receipt( + incumbent.frame, + updated.frame, + node, + normalized_receipt, + rewrite_coordinates=expand_rewrites, + ) + except (TypeError, ValueError) as error: + raise NodeRejected( + f"Node {node.id!r} expand_writes receipt rejected: {error}" + ) from error + if hit: + try: + stored_expand_writes = _parse_expand_writes( + node, normalized_receipt.get("expand_writes") + ) + except ValueError as error: # defended by cached-record validation + raise StoreCorrupt( + f"Cached EXPAND node {node.id!r} has malformed " + "expand_writes provenance." + ) from error + stored_payload = { + f"{entity}.{column}": list(classes) + for (entity, column), classes in stored_expand_writes.items() + } + if stored_payload != authored_expand_writes: + raise StoreCorrupt( + f"Cached EXPAND node {node.id!r} expand_writes provenance " + "disagrees with its materialized frame." + ) + normalized_receipt["expand_writes"] = authored_expand_writes + if node.structural not in { + StructuralDelta.NONE, + StructuralDelta.CREATE, + }: + existing_mass = normalized_receipt.get("mass", {}) + if not isinstance(existing_mass, Mapping): # defended by mass validation + raise NodeRejected( + f"Node {node.id!r} receipt['mass'] is not a mapping." + ) + try: + authored_mass = mass_record_receipt(updated.mass_ledger[-1]) + except (TypeError, ValueError) as error: + raise NodeRejected( + f"Node {node.id!r} mass receipt rejected: {error}" + ) from error + normalized_receipt["mass"] = {**existing_mass, **authored_mass} normalized_receipt.update(weight_cap_receipt(updated, node)) cache_receipt = normalized_receipt run_receipt = dict(cache_receipt) @@ -1418,7 +2148,7 @@ def run_graph( population=updated, receipt=cache_receipt, opaque_artifacts=opaque, - verify_existing=resume != "forbid", + verify_existing=(resume != "forbid" and not replace_stale_record), ) assert record is not None diff --git a/packages/microcosm-graph/src/microcosm/graph/explain.py b/packages/microcosm-graph/src/microcosm/graph/explain.py index 2bc4f8996..9d10d8be6 100644 --- a/packages/microcosm-graph/src/microcosm/graph/explain.py +++ b/packages/microcosm-graph/src/microcosm/graph/explain.py @@ -14,6 +14,7 @@ from .decl import GATE_OUTCOMES, CompiledGraph, StructuralDelta from .manifest import NodeReceipt, RunManifest +from .population import mass_record_receipt from .view import describe if TYPE_CHECKING: @@ -295,7 +296,13 @@ def _short(value: object, limit: int = 28) -> str: def _role(receipt: NodeReceipt) -> str: - return str(_value(receipt.capabilities.role)) + capabilities = receipt.capabilities + role = ( + capabilities.get("role", "compute") + if isinstance(capabilities, Mapping) + else capabilities.role + ) + return str(_value(role)) def _node_status(receipt: NodeReceipt) -> tuple[str, str]: @@ -460,6 +467,9 @@ def _render_graph( def _capabilities(receipt: NodeReceipt) -> dict[str, object]: capabilities = receipt.capabilities + if isinstance(capabilities, Mapping): + return {str(key): _plain(value) for key, value in capabilities.items()} + tolerance = capabilities.tolerance return { "determinism": _value(capabilities.determinism), "numeric": _value(capabilities.numeric), @@ -468,11 +478,20 @@ def _capabilities(receipt: NodeReceipt) -> dict[str, object]: "role": _value(capabilities.role), "consumes_se": capabilities.consumes_se, "dependencies": capabilities.dependencies, + "tolerance": ( + None + if tolerance is None + else { + "rtol": tolerance.rtol, + "atol": tolerance.atol, + "ulps": tolerance.ulps, + } + ), } def _receipt_payload(receipt: NodeReceipt) -> dict[str, object]: - return { + payload = { "node_key": receipt.key, "store_hit": receipt.hit, "seed": receipt.seed, @@ -489,6 +508,9 @@ def _receipt_payload(receipt: NodeReceipt) -> dict[str, object]: "opaque_artifacts": receipt.opaque_artifacts, "wall_time": receipt.wall_time, } + if receipt.legacy_capabilities: + payload["legacy_capabilities"] = True + return payload def _render_node_detail( @@ -577,14 +599,16 @@ def _run_metadata(manifest: RunManifest) -> str: hits = sum(receipt.hit for receipt in manifest.nodes.values()) try: tier = manifest.tier or "Not applicable" + manifest_key = manifest.key except ValueError: tier = "Invalid release evidence" + manifest_key = "Invalid manifest" decisions = ", ".join(decision.kind for decision in manifest.decisions) or "None" return ( '
Run metadata' '
' f"
Country
{_escape(manifest.country)}
" - f"
Manifest key
{_escape(manifest.key)}
" + f"
Manifest key
{_escape(manifest_key)}
" f"
Tier
{_escape(tier)}
" f"
Nodes
{len(manifest.nodes)}
" f"
Store hits
{hits}
" @@ -995,22 +1019,19 @@ def _mass_payload( ) -> dict[str, object] | None: raw = receipt.receipt.get("mass") if isinstance(raw, Mapping): - return { + payload = { "before": raw.get("before"), "after": raw.get("after"), "stratum_before": raw.get("stratum_before", {}), "stratum_after": raw.get("stratum_after", {}), "policy": raw.get("policy", node.mass), } + if isinstance(raw.get("partition"), Mapping): + payload["partition"] = raw["partition"] + return payload for record in reversed(manifest.mass_ledgers.get(node.id, ())): if record.node_id == node.id: - return { - "before": record.before_total, - "after": record.after_total, - "stratum_before": dict(record.before_by_stratum), - "stratum_after": dict(record.after_by_stratum), - "policy": record.policy, - } + return mass_record_receipt(record) if node.base is not None: before = manifest.populations.get(node.base) after = manifest.populations.get(node.id) @@ -1031,6 +1052,50 @@ def _mass_payload( return None +def _partition_mass_table(mass: Mapping[str, object]) -> str: + raw_partition = mass.get("partition") + if not isinstance(raw_partition, Mapping): + return "" + raw_before = raw_partition.get("stratum_before", {}) + raw_after = raw_partition.get("stratum_after", {}) + by_before = raw_before if isinstance(raw_before, Mapping) else {} + by_after = raw_after if isinstance(raw_after, Mapping) else {} + partitions = sorted(set(by_before) | set(by_after), key=str) + rows: list[str] = [] + for partition in partitions: + raw_before_strata = by_before.get(partition, {}) + raw_after_strata = by_after.get(partition, {}) + before_strata = ( + raw_before_strata if isinstance(raw_before_strata, Mapping) else {} + ) + after_strata = raw_after_strata if isinstance(raw_after_strata, Mapping) else {} + strata = sorted(set(before_strata) | set(after_strata), key=str) + for stratum in strata: + before = before_strata.get(stratum, 0.0) + after = after_strata.get(stratum, 0.0) + delta = ( + float(after) - float(before) + if isinstance(before, int | float) and isinstance(after, int | float) + else None + ) + rows.append( + f"{_escape(partition)}{_escape(stratum)}" + f"{_escape(_number(before))}" + f"{_escape(_number(after))}" + f"{_escape(_number(delta))}" + ) + entity = raw_partition.get("entity", "Not recorded") + column = raw_partition.get("column", "Not recorded") + heading = f"
Mass by {_escape(f'{entity}.{column}')} partition
" + if not rows: + return heading + '

Per-partition mass was not recorded.

' + return ( + heading + '
' + "" + f"{''.join(rows)}
Partition valueStratumBeforeAfterChange
" + ) + + def _mass_tables(mass: Mapping[str, object] | None) -> str: if mass is None: return '

Mass ledger values are not present in the portable receipt.

' @@ -1053,8 +1118,13 @@ def _mass_tables(mass: Mapping[str, object] | None) -> str: by_before = raw_before if isinstance(raw_before, Mapping) else {} by_after = raw_after if isinstance(raw_after, Mapping) else {} strata = sorted(set(by_before) | set(by_after), key=str) + partition_table = _partition_mass_table(mass) if not strata: - return totals + '

Per-stratum mass was not recorded.

' + return ( + totals + + '

Per-stratum mass was not recorded.

' + + partition_table + ) rows = "".join( f"{_escape(stratum)}{_escape(_number(by_before.get(stratum)))}" f"{_escape(_number(by_after.get(stratum)))}" @@ -1064,6 +1134,7 @@ def _mass_tables(mass: Mapping[str, object] | None) -> str: totals + '
' f"{rows}
StratumBeforeAfter
" + + partition_table ) diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index fbe66b2a5..567a57ea5 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -17,7 +17,10 @@ - ``EXPAND`` returns the clone lineage as :attr:`KernelResult.expand` (per entity, new ids to the source ids they copy) plus the new weights; the executor carries every column from the source rows, records the lineage - in the receipt, and records mass. + in the receipt, and records mass. A node declared ``entrants=True`` may + also add rows with null lineage; the kernel then materializes their + columns, and for entrant persons their stratum through + :attr:`KernelResult.strata` (amendments 11 and 14). - ``REWEIGHT`` (and any node with a declared weight transition) returns :attr:`KernelResult.weights`; the executor validates the kind transition and the mass policy. @@ -30,6 +33,12 @@ is derived from the gate verdicts in its ancestry, and its receipt reports ``unreached`` when a required human decision is absent from the run). +Numbers carry their own contract. A kernel whose :class:`Numeric` claim is +``tolerance_bound`` declares a :class:`Tolerance`; the executor records it +in the receipt and hands every reader the declared tolerance of each input +cell's owner through :attr:`KernelContext.tolerances`, so a gate compares +against a declaration rather than a guess (amendment 13). + This file is a frozen interface (see ``docs/graph-acceptance.md``). """ @@ -63,6 +72,7 @@ "KernelRole", "Numeric", "SeedSource", + "Tolerance", "source_hash", ] @@ -76,12 +86,66 @@ class Determinism(StrEnum): class Numeric(StrEnum): - """How reproducible a kernel's numbers are across runs.""" + """How reproducible a kernel's numbers are across runs. + + ``bitwise``: identical bytes on every platform. ``platform_bitwise``: + identical bytes on one platform (architecture and locked dependencies), + with no bound on how far a cell may move across platforms; a quantile + forest is the model case, where a one-ulp difference can flip which + donor a draw lands on (amendment 16). ``tolerance_bound``: every cell + within a declared :class:`Tolerance` across platforms. + """ BITWISE = "bitwise" + PLATFORM_BITWISE = "platform_bitwise" TOLERANCE_BOUND = "tolerance_bound" +@dataclass(frozen=True) +class Tolerance: + """How far a ``tolerance_bound`` kernel's numbers may move between runs. + + Two values agree when they are within ``atol`` absolutely, or within + ``rtol`` relatively, or within ``ulps`` last-place units of each other. + A bitwise kernel declares no tolerance at all. + + Attributes: + rtol: Relative tolerance; non-negative and finite. + atol: Absolute tolerance; non-negative and finite. + ulps: Units in the last place; non-negative. + """ + + rtol: float = 0.0 + atol: float = 0.0 + ulps: int = 0 + + def __post_init__(self) -> None: + for name in ("rtol", "atol"): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"Tolerance.{name} must be a number.") + try: + # Keys and manifests carry the float; an integer too large for + # one would only fail later, at identity time. + as_float = float(value) + except OverflowError as error: + raise ValueError( + f"Tolerance.{name} must be representable as a finite float." + ) from error + if not (as_float >= 0.0) or as_float == float("inf"): + raise ValueError(f"Tolerance.{name} must be non-negative and finite.") + object.__setattr__(self, name, as_float) + if isinstance(self.ulps, bool) or not isinstance(self.ulps, int): + raise ValueError("Tolerance.ulps must be an integer.") + if self.ulps < 0: + raise ValueError("Tolerance.ulps must be non-negative.") + if self.rtol == 0.0 and self.atol == 0.0 and self.ulps == 0: + raise ValueError( + "Tolerance must allow some movement; a bitwise kernel declares " + "no tolerance instead." + ) + + class SeedSource(StrEnum): """Where a seeded kernel takes its randomness from.""" @@ -113,6 +177,9 @@ class Capabilities: standard errors. A kernel that ignores them says so here. dependencies: Installed distributions whose versions enter the implementation hash. + tolerance: Required when ``numeric`` is ``tolerance_bound`` and + forbidden otherwise: how far the kernel's numbers may move + between runs or machines. """ determinism: Determinism @@ -122,6 +189,85 @@ class Capabilities: role: KernelRole = KernelRole.COMPUTE consumes_se: bool = False dependencies: tuple[str, ...] = () + tolerance: Tolerance | None = None + + def __post_init__(self) -> None: + # Every field is validated here, so a registered contract is a real + # one: a string that spells an enum member does not pass as the member. + for name, kind in ( + ("determinism", Determinism), + ("numeric", Numeric), + ("seed_source", SeedSource), + ("structural", StructuralDelta), + ("role", KernelRole), + ): + if not isinstance(getattr(self, name), kind): + raise TypeError(f"Capabilities.{name} must be a {kind.__name__}.") + if not isinstance(self.consumes_se, bool): + raise TypeError("Capabilities.consumes_se must be a boolean.") + if not isinstance(self.dependencies, tuple) or any( + not isinstance(name, str) or not name for name in self.dependencies + ): + raise TypeError( + "Capabilities.dependencies must be a tuple of distribution names." + ) + if self.tolerance is not None and not isinstance(self.tolerance, Tolerance): + raise TypeError("Capabilities.tolerance must be a Tolerance or None.") + if self.numeric is Numeric.TOLERANCE_BOUND and self.tolerance is None: + raise ValueError( + "A tolerance_bound kernel must declare its Tolerance; a claim of " + "bounded movement without a bound is not a claim." + ) + if ( + self.numeric in (Numeric.BITWISE, Numeric.PLATFORM_BITWISE) + and self.tolerance is not None + ): + raise ValueError("A bitwise kernel declares no Tolerance.") + + +@dataclass(frozen=True) +class NumericScope: + """The numeric contract a gate may hold one input coordinate to. + + The executor derives it from the coordinate's writers under the + loosest-writer rule, ordered ``bitwise`` < ``platform_bitwise`` < + ``tolerance_bound``. A platform-bitwise writer never disappears into + a bound: it leaves ``platform`` set, which says the contract holds on + that platform only and that a cross-platform comparison has no bound + (amendment 17). + + Attributes: + numeric: The loosest :class:`Numeric` class among the writers. + tolerance: The loosest declared :class:`Tolerance` among the + ``tolerance_bound`` writers; ``None`` for the other classes. + platform: The platform fingerprint the contract holds on, or + ``None`` when it holds on every platform. Required when + ``numeric`` is ``platform_bitwise``; permitted on + ``tolerance_bound`` when a platform-bitwise writer contributed; + forbidden on ``bitwise``. + """ + + numeric: Numeric = Numeric.BITWISE + tolerance: Tolerance | None = None + platform: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.numeric, Numeric): + raise ValueError("NumericScope.numeric must be a Numeric.") + if self.tolerance is not None and not isinstance(self.tolerance, Tolerance): + raise ValueError("NumericScope.tolerance must be a Tolerance or None.") + if self.platform is not None and ( + not isinstance(self.platform, str) or not self.platform + ): + raise ValueError("NumericScope.platform must be a non-empty str or None.") + if self.numeric is Numeric.TOLERANCE_BOUND and self.tolerance is None: + raise ValueError("A tolerance_bound scope must carry its Tolerance.") + if self.numeric is not Numeric.TOLERANCE_BOUND and self.tolerance is not None: + raise ValueError(f"A {self.numeric.value} scope carries no Tolerance.") + if self.numeric is Numeric.PLATFORM_BITWISE and self.platform is None: + raise ValueError("A platform_bitwise scope must name its platform.") + if self.numeric is Numeric.BITWISE and self.platform is not None: + raise ValueError("A bitwise scope holds on every platform.") @dataclass(frozen=True) @@ -144,6 +290,14 @@ class KernelContext: kernel may use. sources: Source name to a content-verified path, for declared sources only. + tolerances: ``(entity, column)`` of each declared input column to + the :class:`Tolerance` its owning kernel declared, or ``None`` + for a bitwise owner. A gate compares against these. + numerics: ``(entity, column)`` of each declared input column to + its :class:`NumericScope`; ``tolerances`` is its projection + (``numerics[c].tolerance == tolerances[c]``). A gate that + compares across platforms consults the scope's ``platform`` + (amendment 17). """ node: Node @@ -153,6 +307,8 @@ class KernelContext: params: Mapping[str, Param] rng: np.random.Generator sources: Mapping[str, Path] = field(default_factory=dict) + tolerances: Mapping[tuple[str, str], Tolerance | None] = field(default_factory=dict) + numerics: Mapping[tuple[str, str], NumericScope] = field(default_factory=dict) @dataclass(frozen=True) @@ -176,13 +332,22 @@ class KernelResult: ``weights``. weights: ``REWEIGHT`` kernels and declared weight transitions only: the new explicit weights of the transition's entity. + strata: ``EXPAND`` kernels on a node with ``entrants=True`` only: the + stratum label of every entrant person, indexed by its new id. + Copied persons inherit their source's stratum and must not + appear here; an entrant person absent from it rejects the node. artifacts: Opaque bytes stored beside the node's outputs (a fitted model, a diagnostic table), keyed by name. receipt: Descriptive facts for the manifest. Never hashed into a - key. A gate kernel puts its verdict under ``"outcome"`` and its - evidence under ``"evidence"``; a mass-changing kernel may put - its own accounting under ``"mass"`` (the executor records its - own regardless). + node key; the run manifest's key does hash every node receipt + (less the executor's run-level fields), so a receipt must be a + deterministic function of the computation. A gate kernel puts + its verdict under ``"outcome"`` and its evidence under + ``"evidence"``; a mass-changing kernel may put its own + accounting under ``"mass"`` (the executor records its own + regardless). Timings, host names, and iteration diagnostics + that vary between runs of one computation do not belong here + (amendment 18). """ columns: Mapping[tuple[str, str], pd.Series] = field(default_factory=dict) @@ -192,6 +357,7 @@ class KernelResult: weights: Weights | None = None artifacts: Mapping[str, bytes] = field(default_factory=dict) receipt: Mapping[str, object] = field(default_factory=dict) + strata: pd.Series | None = None @runtime_checkable @@ -281,6 +447,11 @@ def __init__(self) -> None: def register(self, kernel: Kernel) -> Kernel: if not isinstance(kernel, Kernel): raise TypeError(f"{kernel!r} does not satisfy the Kernel protocol.") + if not isinstance(kernel.capabilities, Capabilities): + raise TypeError( + f"Kernel {getattr(kernel, 'ref', kernel)!r} must carry a Capabilities " + "instance, not a look-alike." + ) if kernel.ref in self._kernels and self._kernels[kernel.ref] is not kernel: raise ValueError(f"Kernel {kernel.ref!r} is already registered.") self._kernels[kernel.ref] = kernel diff --git a/packages/microcosm-graph/src/microcosm/graph/keys.py b/packages/microcosm-graph/src/microcosm/graph/keys.py index cab41c700..43939dbae 100644 --- a/packages/microcosm-graph/src/microcosm/graph/keys.py +++ b/packages/microcosm-graph/src/microcosm/graph/keys.py @@ -3,13 +3,17 @@ from __future__ import annotations import hashlib +import platform as _platform +import sys from collections.abc import Mapping from pathlib import Path from .canonical import canonical_json, normative, sha256_domain from .decl import CompiledGraph, StructuralDelta +from .kernel import Capabilities, Numeric __all__ = [ + "platform_fingerprint", "artifact_key", "frame_key", "node_key", @@ -66,6 +70,18 @@ def source_content_key(name: str, path: str | Path) -> str: return _hash_parts("source", name, content_hash, size) +def platform_fingerprint() -> str: + """The platform a platform-bitwise kernel's bytes belong to. + + Architecture, operating system, and Python minor version: the axes along + which ``fit.qrf@1`` was measured to move (``docs/graph-qrf-cross-platform.md``). + """ + return ( + f"{_platform.machine()}/{sys.platform}/" + f"py{sys.version_info.major}.{sys.version_info.minor}" + ) + + def artifact_key(node_key: str, entity: str, column: str) -> str: """Derive one column artifact identity from its producing node.""" @@ -94,12 +110,43 @@ def _required_key(keys: Mapping[str, str], node_id: str, consumer: str) -> str: ) from error +def _canonical_tolerance_float(value: int | float) -> float: + number = float(value) + return 0.0 if number == 0.0 else number + + +def _capabilities_projection(capabilities: Capabilities) -> dict[str, object]: + """Return the complete canonical payload for a kernel contract.""" + + tolerance = capabilities.tolerance + return { + "determinism": capabilities.determinism.value, + "numeric": capabilities.numeric.value, + "seed_source": capabilities.seed_source.value, + "structural": capabilities.structural.value, + "role": capabilities.role.value, + "consumes_se": capabilities.consumes_se, + "dependencies": list(capabilities.dependencies), + "tolerance": ( + None + if tolerance is None + else { + "rtol": _canonical_tolerance_float(tolerance.rtol), + "atol": _canonical_tolerance_float(tolerance.atol), + "ulps": tolerance.ulps, + } + ), + } + + def node_key( compiled: CompiledGraph, node_id: str, input_keys: Mapping[str, str], kernel_impl_hash: str, source_keys: Mapping[str, str], + *, + kernel_capabilities: Capabilities, ) -> str: """Derive a node key from its declaration and resolved input identities. @@ -119,12 +166,19 @@ def node_key( input_version = node.base resolved: dict[tuple[str, str], str] = {} + rewritten = { + (owned.entity, owned.column) for owned in node.outputs if owned.rewrite + } if input_version is not None: for slice_ in node.inputs: for column in slice_.columns: coordinate = (slice_.entity, column) - producer = compiled.owners.get( - (input_version, slice_.entity, column), input_version + producer = ( + input_version + if coordinate in rewritten + else compiled.owners.get( + (input_version, slice_.entity, column), input_version + ) ) producer_key = _required_key(input_keys, producer, node_id) resolved[coordinate] = artifact_key(producer_key, slice_.entity, column) @@ -169,6 +223,23 @@ def node_key( else: resolved_sources = {} + # Graph-level mass semantics (amendment 12) change what a structural node + # computes, so they enter its key; an ordinary node's key is unaffected. + graph_facts = ( + {} if node.structural is StructuralDelta.NONE else compiled.graph.normative() + ) + # Capabilities are executable contract, independent of implementation + # bytes. Bind the complete declaration so a cache entry produced under one + # contract cannot satisfy another kernel with the same ref and code hash. + capabilities = _capabilities_projection(kernel_capabilities) + # A platform-bitwise kernel's bytes belong to one platform (amendment 16): + # its key carries the platform, so a shared store never serves another + # platform's output. Other kernels' keys are unchanged by this. + platform_scope = ( + (platform_fingerprint(),) + if kernel_capabilities.numeric is Numeric.PLATFORM_BITWISE + else () + ) return _hash_parts( "node", normative(node), @@ -176,6 +247,9 @@ def node_key( population_input, kernel_impl_hash, resolved_sources, + graph_facts, + capabilities, + *platform_scope, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/manifest.py b/packages/microcosm-graph/src/microcosm/graph/manifest.py index 8f0a9857b..90b4fa3bc 100644 --- a/packages/microcosm-graph/src/microcosm/graph/manifest.py +++ b/packages/microcosm-graph/src/microcosm/graph/manifest.py @@ -9,25 +9,90 @@ from enum import Enum from pathlib import Path from types import MappingProxyType -from typing import TYPE_CHECKING, Self +from typing import TYPE_CHECKING, ClassVar, Self + +from microcosm.frame import Frame from .canonical import canonical_json, sha256_domain from .decl import GATE_OUTCOMES, StructuralDelta from .errors import NodeRejectedError, StoreCorruptError -from .kernel import Capabilities, Determinism, KernelRole, Numeric, SeedSource +from .kernel import ( + Capabilities, + Determinism, + KernelRole, + Numeric, + SeedSource, + Tolerance, +) from .population import MassRecord if TYPE_CHECKING: - from microcosm.frame import Frame + import pandas as pd from .store import ContentStore -__all__ = ["Decision", "NodeReceipt", "RunManifest"] +__all__ = ["Decision", "NodeReceipt", "PopulationView", "RunManifest"] -_SCHEMA_VERSION = 1 +_SCHEMA_VERSION = 2 +_LEGACY_SCHEMA_VERSION = 1 _CERTIFYING_GATE_OUTCOMES = frozenset({"pass", "not_applicable"}) +class PopulationView(Frame): + """Zero-copy manifest view with entity-name table access. + + All attached populations use this type. Existing :class:`Frame` accessors + remain available, and a non-colliding group entity can also be read by name + (for example, ``view.household`` is equivalent to + ``view.entity("household")``). If an entity name collides with a public + :class:`Frame` attribute such as ``metadata``, shorthand attribute access + returns the Frame member, exactly as it would on a plain Frame; the + entity stays reachable through :meth:`entity`, which is the reliable + accessor for every entity name. The source Frame keeps its original type. + """ + + __slots__ = () + + def __init__(self, frame: Frame) -> None: + if not isinstance(frame, Frame): + raise TypeError("PopulationView requires a Frame") + for slot in Frame.__slots__: + object.__setattr__(self, slot, getattr(frame, slot)) + + def entity(self, name: str) -> pd.DataFrame: + """Return an entity table, including names colliding with Frame APIs. + + Args: + name: An entity declared by the attached frame's schema. + + Returns: + The entity table. Treat as read-only. + + Raises: + ValueError: If ``name`` is not declared by the schema. + """ + + return Frame.table(self, name) + + def __getattr__(self, name: str) -> object: + """Entity tables by attribute, for names that collide with nothing. + + Python reaches here only when ordinary lookup fails, so an entity + whose name collides with a Frame attribute resolves to the Frame + attribute (as on a plain Frame) and stays reachable through + :meth:`entity`; inherited Frame methods keep working either way. + """ + try: + schema = object.__getattribute__(self, "_schema") + except AttributeError: + raise AttributeError( + f"{type(self).__name__!s} has no attribute {name!r}" + ) from None + if name in schema.entities: + return Frame.table(self, name) + raise AttributeError(f"{type(self).__name__!s} has no attribute {name!r}") + + def _freeze_json(value: object) -> object: """Copy JSON-like receipt data into immutable containers.""" @@ -55,6 +120,16 @@ def _enum_value(value: object) -> object: return value.value if isinstance(value, Enum) else value +def _tolerance_payload(tolerance: Tolerance | None) -> dict[str, object] | None: + if tolerance is None: + return None + return { + "rtol": float(tolerance.rtol), + "atol": float(tolerance.atol), + "ulps": tolerance.ulps, + } + + @dataclass(frozen=True) class Decision(Mapping[str, str]): """A signed human decision carried as provenance, never as a node input.""" @@ -137,13 +212,14 @@ class NodeReceipt: seed: int kernel_ref: str kernel_impl_hash: str - capabilities: Capabilities + capabilities: Capabilities | Mapping[str, object] receipt: Mapping[str, object] = field(default_factory=dict) artifacts: Mapping[tuple[str, str], str] = field(default_factory=dict) wall_time: float = 0.0 frame_key: str | None = None weight_key: str | None = None opaque_artifacts: Mapping[str, str] = field(default_factory=dict) + legacy_capabilities: bool = field(default=False, kw_only=True) def __post_init__(self) -> None: if not isinstance(self.key, str): @@ -156,7 +232,17 @@ def __post_init__(self) -> None: raise TypeError("NodeReceipt.kernel_ref must be a string") if not isinstance(self.kernel_impl_hash, str): raise TypeError("NodeReceipt.kernel_impl_hash must be a string") - if not isinstance(self.capabilities, Capabilities): + if not isinstance(self.legacy_capabilities, bool): + raise TypeError("NodeReceipt.legacy_capabilities must be a bool") + if self.legacy_capabilities: + if not isinstance(self.capabilities, Mapping) or isinstance( + self.capabilities, Capabilities + ): + raise TypeError("legacy NodeReceipt.capabilities must be a raw mapping") + frozen_capabilities = _legacy_capabilities_from_payload(self.capabilities) + object.__setattr__(self, "capabilities", frozen_capabilities) + object.__setattr__(self, "hit", False) + elif not isinstance(self.capabilities, Capabilities): raise TypeError("NodeReceipt.capabilities must be Capabilities") frozen_receipt = _freeze_json(self.receipt) if not isinstance(frozen_receipt, Mapping): @@ -227,13 +313,15 @@ def wall_time_s(self) -> float: def _payload(self) -> dict[str, object]: capabilities = self.capabilities - return { - "key": self.key, - "hit": self.hit, - "seed": self.seed, - "kernel_ref": self.kernel_ref, - "kernel_impl_hash": self.kernel_impl_hash, - "capabilities": { + capabilities_payload: Mapping[str, object] + if self.legacy_capabilities: + if not isinstance(capabilities, Mapping): # pragma: no cover + raise TypeError("legacy capabilities lost their raw mapping") + capabilities_payload = capabilities + else: + if not isinstance(capabilities, Capabilities): # pragma: no cover + raise TypeError("current capabilities lost their typed contract") + capabilities_payload = { "determinism": _enum_value(capabilities.determinism), "numeric": _enum_value(capabilities.numeric), "seed_source": _enum_value(capabilities.seed_source), @@ -241,7 +329,16 @@ def _payload(self) -> dict[str, object]: "role": _enum_value(capabilities.role), "consumes_se": capabilities.consumes_se, "dependencies": capabilities.dependencies, - }, + "tolerance": _tolerance_payload(capabilities.tolerance), + } + return { + "key": self.key, + "hit": self.hit, + "seed": self.seed, + "kernel_ref": self.kernel_ref, + "kernel_impl_hash": self.kernel_impl_hash, + "capabilities": capabilities_payload, + "legacy_capabilities": self.legacy_capabilities, "receipt": self.receipt, "artifacts": tuple( {"entity": entity, "column": column, "key": key} @@ -253,14 +350,49 @@ def _payload(self) -> dict[str, object]: "wall_time": self.wall_time, } + #: Fields a run may change without changing what was computed. + RUN_LEVEL_FIELDS: ClassVar[frozenset[str]] = frozenset({"hit", "wall_time"}) + #: Receipt entries of a release-role node that derive from the decisions a + #: run supplied rather than from computation (F5: decisions never feed a key). + RELEASE_RUN_LEVEL_RECEIPT_FIELDS: ClassVar[frozenset[str]] = frozenset({"outcome"}) + + def _content_payload(self) -> dict[str, object]: + """The receipt less its run-level fields; this is what the manifest key hashes. + + For a release-role node the decision-derived ``outcome`` is also left + out, so two runs of one computation share a key whatever decisions + each was handed; the certified loader authenticates decisions instead. + """ + + payload = self._payload() + body = {k: v for k, v in payload.items() if k not in self.RUN_LEVEL_FIELDS} + capabilities = body.get("capabilities") + role = capabilities.get("role") if isinstance(capabilities, Mapping) else None + receipt = body.get("receipt") + if role == "release" and isinstance(receipt, Mapping): + body["receipt"] = { + k: v + for k, v in receipt.items() + if k not in self.RELEASE_RUN_LEVEL_RECEIPT_FIELDS + } + return body + @dataclass(frozen=True) class RunManifest: """One run's provenance plus its attached, non-serialized populations. - Only sorted node keys and signed decisions form :attr:`key`. Store hits, - timings, host, timestamps, receipts, and attached ``Frame`` instances are - run-level observations and cannot invalidate computational reuse. + Every complete node receipt, less its run-level fields (``hit``, + ``wall_time``, and a release node's decision-derived ``outcome``), and the + derived release tier form :attr:`key`, so two runs that computed the same + thing share a key whether or not either was served from the store and + whatever decisions each was handed. Signed decisions and the outcome they + yield stay outside the key by interface ruling; + the release receipt's required decision names are authenticated so a + certified load can revalidate those carried records. Country, host, + timestamps, and attached ``Frame`` instances are also outside the + content-addressed body. Computational reuse continues to use node keys, + not this run-manifest identity. """ country: str @@ -301,8 +433,17 @@ def __post_init__(self) -> None: for name in ("started_at", "finished_at", "host"): if not isinstance(getattr(self, name), str): raise TypeError(f"RunManifest.{name} must be a string") + populations: dict[str, PopulationView] = {} + for version_id, frame in self.populations.items(): + if not isinstance(version_id, str): + raise TypeError("RunManifest.populations keys must be strings") + if not isinstance(frame, Frame): + raise TypeError("RunManifest.populations values must be Frame") + populations[version_id] = PopulationView(frame) object.__setattr__( - self, "populations", MappingProxyType(dict(self.populations)) + self, + "populations", + MappingProxyType(populations), ) mass_ledgers: dict[str, tuple[MassRecord, ...]] = {} for version_id, records in self.mass_ledgers.items(): @@ -320,14 +461,14 @@ def __post_init__(self) -> None: def content_addressed(self) -> Mapping[str, object]: """The exact projection hashed by :attr:`key`.""" - decisions = sorted( - (decision._payload() for decision in self.decisions), - key=canonical_json, - ) + nodes = { + node_id: self.nodes[node_id]._content_payload() + for node_id in sorted(self.nodes) + } return MappingProxyType( { - "node_keys": tuple(sorted(item.key for item in self.nodes.values())), - "decisions": tuple(decisions), + "nodes": MappingProxyType(nodes), + "tier": self.tier, } ) @@ -338,7 +479,7 @@ def tier(self) -> str | None: releases = [ (node_id, node) for node_id, node in self.nodes.items() - if node.capabilities.role is KernelRole.RELEASE + if _capability_role(node) is KernelRole.RELEASE ] if not releases: return None @@ -363,7 +504,7 @@ def tier(self) -> str | None: raise ValueError( f"release node {node_id!r} names missing gate {gate_id!r}" ) from error - if gate.capabilities.role is not KernelRole.GATE: + if _capability_role(gate) is not KernelRole.GATE: raise ValueError( f"release node {node_id!r} names non-gate ancestor {gate_id!r}" ) @@ -389,7 +530,7 @@ def known_failures(self) -> tuple[str, ...]: failures: set[str] = set() for node_id, node in self.nodes.items(): outcome = node.receipt.get("outcome") - if node.capabilities.role is KernelRole.GATE and ( + if _capability_role(node) is KernelRole.GATE and ( _gate_outcome(node_id, node) not in _CERTIFYING_GATE_OUTCOMES ): failures.add(node_id) @@ -416,7 +557,7 @@ def node(self, node_id: str) -> NodeReceipt: def receipt(self, node_id: str) -> NodeReceipt: return self.node(node_id) - def population(self, version_id: str) -> Frame: + def population(self, version_id: str) -> PopulationView: """Return an attached final population version. Population frames are deliberately not serialized in manifest JSON; @@ -425,11 +566,14 @@ def population(self, version_id: str) -> Frame: """ try: - return self.populations[version_id] + population = self.populations[version_id] except KeyError as error: raise KeyError( f"Population {version_id!r} is not attached to this manifest." ) from error + if not isinstance(population, PopulationView): # __post_init__ invariant + raise RuntimeError("attached population was not normalized") + return population def mass_ledger(self, version_id: str) -> tuple[MassRecord, ...]: """Return the transient mass audit trail for one attached version.""" @@ -485,18 +629,20 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: raw = json.loads(value) if not isinstance(raw, dict): raise ValueError("manifest JSON must contain an object") - if type(raw.get("schema_version")) is not int or ( - raw["schema_version"] != _SCHEMA_VERSION - ): - raise ValueError( - f"unsupported manifest schema version {raw.get('schema_version')!r}" - ) + schema_version = raw.get("schema_version") + if type(schema_version) is not int or schema_version not in { + _LEGACY_SCHEMA_VERSION, + _SCHEMA_VERSION, + }: + raise ValueError(f"unsupported manifest schema version {schema_version!r}") nodes_raw = raw.get("nodes") if not isinstance(nodes_raw, dict): raise ValueError("manifest nodes must be an object") nodes = { - str(node_id): _node_receipt_from_payload(payload) + str(node_id): _node_receipt_from_payload( + payload, schema_version=schema_version + ) for node_id, payload in nodes_raw.items() } decisions_raw = raw.get("decisions", []) @@ -512,23 +658,31 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: host=_string_field(raw, "host"), ) body = raw.get("content_addressed") - if "content_addressed" in raw: - if not isinstance(body, Mapping): - raise ValueError("manifest content-addressed body must be an object") - recomputed_body = json.loads(canonical_json(manifest.content_addressed)) + if not isinstance(body, Mapping): + raise ValueError("manifest content-addressed body must be an object") + if schema_version == _LEGACY_SCHEMA_VERSION: + recomputed_body = json.loads( + canonical_json(_legacy_content_addressed(manifest)) + ) if body != recomputed_body: raise ValueError( - "manifest content key mismatch: the content-addressed body " - "does not match portable provenance" + "manifest content key mismatch: the legacy content-addressed " + "body does not match portable provenance" ) - recomputed_key = sha256_domain("manifest", canonical_json(body)) else: - recomputed_key = manifest.key + recomputed_body = manifest.content_addressed + _validate_current_content_addressed_body(body, recomputed_body, manifest) + recomputed_key = sha256_domain("manifest", canonical_json(body)) serialized_key = raw.get("key") - if serialized_key != recomputed_key or recomputed_key != manifest.key: + if serialized_key != recomputed_key: raise ValueError( "manifest content key mismatch: serialized provenance was altered" ) + if schema_version == _SCHEMA_VERSION and serialized_key != manifest.key: + raise ValueError( + "manifest content key mismatch: serialized key differs from " + "reconstructed portable provenance" + ) if "tier" in raw and raw["tier"] != manifest.tier: raise ValueError( f"manifest tier mismatch: stored {raw['tier']!r}, " @@ -611,15 +765,38 @@ def load_certified(cls, path: str | Path, store: ContentStore) -> Self: """Load ``path`` only when its reached release is certified.""" manifest = cls.load(path, store) - releases = [ - node - for node in manifest.nodes.values() - if node.capabilities.role is KernelRole.RELEASE - ] - if any(node.receipt.get("outcome") == "unreached" for node in releases): + legacy_nodes = sorted( + node_id + for node_id, node in manifest.nodes.items() + if node.legacy_capabilities + ) + if legacy_nodes: raise NodeRejectedError( - f"Manifest {manifest.key} release outcome is unreached." + f"Manifest {manifest.key} is unreached: legacy_capabilities on " + f"nodes {legacy_nodes!r} omit the schema-v2 tolerance field." ) + releases = [ + (node_id, node) + for node_id, node in manifest.nodes.items() + if _capability_role(node) is KernelRole.RELEASE + ] + decision_names = _decision_names(manifest.decisions) + for node_id, node in releases: + required = _required_decision_names(node_id, node) + missing = sorted(set(required) - decision_names) + if missing: + raise NodeRejectedError( + f"Manifest {manifest.key} release outcome is unreached: " + f"missing required signed decisions {missing!r}." + ) + expected_outcome = "pass" if manifest.tier == "certified" else "fail" + stored_outcome = node.receipt.get("outcome") + if stored_outcome != expected_outcome: + raise NodeRejectedError( + f"Manifest {manifest.key} release node {node_id!r} outcome " + f"{stored_outcome!r} disagrees with revalidated outcome " + f"{expected_outcome!r}." + ) if manifest.tier != "certified": raise NodeRejectedError( f"Manifest {manifest.key} is evidence-tier, not certified." @@ -627,6 +804,84 @@ def load_certified(cls, path: str | Path, store: ContentStore) -> Self: return manifest +def _legacy_content_addressed(manifest: RunManifest) -> Mapping[str, object]: + """Reconstruct the schema-v1 identity projection for legacy validation.""" + + decisions = sorted( + (decision._payload() for decision in manifest.decisions), + key=canonical_json, + ) + return MappingProxyType( + { + "node_keys": tuple(sorted(node.key for node in manifest.nodes.values())), + "decisions": tuple(decisions), + } + ) + + +def _validate_current_content_addressed_body( + body: Mapping[str, object], + expected: Mapping[str, object], + manifest: RunManifest, +) -> None: + """Match a schema-v2 body to portable receipts, naming the first node.""" + + expected_nodes = expected.get("nodes") + if not isinstance(expected_nodes, Mapping): # pragma: no cover - internal shape + raise RuntimeError("current manifest identity lost its node mapping") + first_node = min(manifest.nodes, default="") + body_nodes = body.get("nodes") + if not isinstance(body_nodes, Mapping): + raise ValueError( + f"manifest content key mismatch at node {first_node!r}: " + "the content-addressed body has no node receipt mapping" + ) + for node_id in sorted(set(body_nodes) | set(expected_nodes)): + if node_id not in body_nodes: + detail = "the content-addressed body omits its receipt" + elif node_id not in expected_nodes: + detail = "the content-addressed body names an absent node" + elif canonical_json(body_nodes[node_id]) != canonical_json( + expected_nodes[node_id] + ): + detail = "the content-addressed receipt differs from portable provenance" + else: + continue + raise ValueError(f"manifest content key mismatch at node {node_id!r}: {detail}") + + expected_fields = {"nodes", "tier"} + if set(body) != expected_fields: + raise ValueError( + f"manifest content key mismatch after node {first_node!r}: body fields " + f"{sorted(body)!r} do not equal {sorted(expected_fields)!r}" + ) + if canonical_json(body.get("tier")) != canonical_json(expected.get("tier")): + release_nodes = sorted( + node_id + for node_id, node in manifest.nodes.items() + if _capability_role(node) is KernelRole.RELEASE + ) + tier_node = release_nodes[0] if release_nodes else first_node + raise ValueError( + f"manifest content key mismatch at node {tier_node!r}: stored tier " + f"{body.get('tier')!r} differs from derived tier " + f"{expected.get('tier')!r}" + ) + + +def _capability_role(node: NodeReceipt) -> KernelRole: + """Return a receipt role without constructing a legacy contract.""" + + capabilities = node.capabilities + if isinstance(capabilities, Capabilities): + return capabilities.role + role = capabilities.get("role", KernelRole.COMPUTE.value) + try: + return KernelRole(str(role)) + except ValueError as error: # Parser validation should make this unreachable. + raise ValueError(f"node capabilities have invalid role {role!r}") from error + + def _gate_outcome(node_id: str, node: NodeReceipt) -> str: """Return one authenticated-shape gate outcome or reject the manifest.""" @@ -639,6 +894,50 @@ def _gate_outcome(node_id: str, node: NodeReceipt) -> str: return str(outcome) +def _decision_names(decisions: tuple[Decision, ...]) -> frozenset[str]: + """Validate the two signed-decision record shapes and return their names.""" + + names: set[str] = set() + for decision in decisions: + payload = dict(decision) + if set(payload) == {"name", "owner", "signature"}: + fields = ("name", "owner", "signature") + name = payload["name"] + elif set(payload) == {"owner", "kind", "text", "signed_at"}: + fields = ("owner", "kind", "text", "signed_at") + name = payload["kind"] + else: # defended by Decision.from_mapping + raise NodeRejectedError( + "Certified manifests require a recognized signed decision record." + ) + empty = [field for field in fields if not payload[field].strip()] + if empty: + raise NodeRejectedError( + "Certified manifests require non-empty signed decision fields; " + f"record {name!r} has empty fields {empty!r}." + ) + names.add(name) + return frozenset(names) + + +def _required_decision_names(node_id: str, node: NodeReceipt) -> tuple[str, ...]: + """Read authenticated release decision requirements, failing closed.""" + + raw = node.receipt.get("requires_decisions") + if not isinstance(raw, tuple) or any( + not isinstance(name, str) or not name for name in raw + ): + raise NodeRejectedError( + f"Release node {node_id!r} has no valid authenticated " + "requires_decisions provenance." + ) + if len(set(raw)) != len(raw): + raise NodeRejectedError( + f"Release node {node_id!r} repeats an authenticated required decision name." + ) + return raw + + def _validate_artifacts(manifest: RunManifest, store: ContentStore) -> None: """Confirm that every manifest artifact exists with its declared kind.""" @@ -676,29 +975,111 @@ def _invalid_decision(value: object) -> Decision: ) -def _capabilities_from_payload(value: object) -> Capabilities: +def _capability_contract_fields( + value: object, +) -> tuple[ + Determinism, + Numeric, + SeedSource, + StructuralDelta, + KernelRole, + bool, + tuple[str, ...], +]: if not isinstance(value, Mapping): raise ValueError("node capabilities must be an object") consumes_se = value.get("consumes_se") dependencies = value.get("dependencies") if not isinstance(consumes_se, bool): raise ValueError("capabilities.consumes_se must be a bool") - if not isinstance(dependencies, list): + if not isinstance(dependencies, list | tuple): raise ValueError("capabilities.dependencies must be an array") - if not all(isinstance(item, str) for item in dependencies): - raise ValueError("capabilities.dependencies must contain strings") + if not all(isinstance(item, str) and item for item in dependencies): + raise ValueError("capabilities.dependencies must contain non-empty strings") + try: + return ( + Determinism(_string_field(value, "determinism")), + Numeric(_string_field(value, "numeric")), + SeedSource(_string_field(value, "seed_source")), + StructuralDelta(_string_field(value, "structural")), + KernelRole(str(value.get("role", KernelRole.COMPUTE.value))), + consumes_se, + tuple(dependencies), + ) + except ValueError as error: + raise ValueError(f"invalid node capabilities: {error}") from error + + +def _legacy_capabilities_from_payload(value: object) -> Mapping[str, object]: + """Validate schema-v1 capabilities without inventing a tolerance.""" + + if not isinstance(value, Mapping): + raise ValueError("node capabilities must be an object") + if "tolerance" in value: + raise ValueError("legacy capabilities must omit the tolerance field") + _capability_contract_fields(value) + frozen = _freeze_json(value) + if not isinstance(frozen, Mapping): # pragma: no cover + raise ValueError("legacy node capabilities must be an object") + return frozen + + +def _capabilities_from_payload(value: object) -> Capabilities: + if not isinstance(value, Mapping): + raise ValueError("node capabilities must be an object") + if "tolerance" not in value: + raise ValueError("capabilities.tolerance is required by schema v2") + raw_tolerance = value["tolerance"] + if raw_tolerance is None: + tolerance = None + else: + if not isinstance(raw_tolerance, Mapping) or set(raw_tolerance) != { + "rtol", + "atol", + "ulps", + }: + raise ValueError( + "capabilities.tolerance must be null or an object containing " + "rtol, atol, and ulps" + ) + rtol = raw_tolerance["rtol"] + atol = raw_tolerance["atol"] + ulps = raw_tolerance["ulps"] + if ( + isinstance(rtol, bool) + or not isinstance(rtol, int | float) + or isinstance(atol, bool) + or not isinstance(atol, int | float) + or isinstance(ulps, bool) + or not isinstance(ulps, int) + ): + raise ValueError( + "capabilities.tolerance rtol/atol must be numeric and ulps " + "must be an integer" + ) + tolerance = Tolerance(rtol=rtol, atol=atol, ulps=ulps) + ( + determinism, + numeric, + seed_source, + structural, + role, + consumes_se, + dependencies, + ) = _capability_contract_fields(value) return Capabilities( - determinism=Determinism(_string_field(value, "determinism")), - numeric=Numeric(_string_field(value, "numeric")), - seed_source=SeedSource(_string_field(value, "seed_source")), - structural=StructuralDelta(_string_field(value, "structural")), - role=KernelRole(str(value.get("role", KernelRole.COMPUTE.value))), + determinism=determinism, + numeric=numeric, + seed_source=seed_source, + structural=structural, + role=role, consumes_se=consumes_se, - dependencies=tuple(dependencies), + dependencies=dependencies, + tolerance=tolerance, ) -def _node_receipt_from_payload(value: object) -> NodeReceipt: +def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeReceipt: if not isinstance(value, Mapping): raise ValueError("manifest node receipts must be objects") hit = value.get("hit") @@ -709,6 +1090,33 @@ def _node_receipt_from_payload(value: object) -> NodeReceipt: frame_key = value.get("frame_key") weight_key = value.get("weight_key") opaque_artifacts = value.get("opaque_artifacts", {}) + capabilities_payload = value.get("capabilities") + if schema_version == _LEGACY_SCHEMA_VERSION: + # Every schema-v1 receipt is legacy: v1 never recorded a tolerance, so + # a v1 receipt that carries one, or the v2 legacy flag, is a hybrid + # that no writer produced and is refused rather than promoted. + if "legacy_capabilities" in value: + raise ValueError("schema-v1 node receipts cannot carry legacy_capabilities") + if ( + isinstance(capabilities_payload, Mapping) + and "tolerance" in capabilities_payload + ): + raise ValueError( + "schema-v1 node receipts cannot carry capabilities.tolerance; " + "a v1 manifest is legacy in full" + ) + legacy_capabilities = True + else: + legacy_capabilities = value.get("legacy_capabilities") + if not isinstance(legacy_capabilities, bool): + raise ValueError( + "schema-v2 node receipt legacy_capabilities must be a bool" + ) + capabilities = ( + _legacy_capabilities_from_payload(capabilities_payload) + if legacy_capabilities + else _capabilities_from_payload(capabilities_payload) + ) if not isinstance(hit, bool): raise ValueError("node receipt hit must be a bool") if not isinstance(seed_value, int) or isinstance(seed_value, bool): @@ -747,11 +1155,12 @@ def _node_receipt_from_payload(value: object) -> NodeReceipt: seed=seed_value, kernel_ref=_string_field(value, "kernel_ref"), kernel_impl_hash=_string_field(value, "kernel_impl_hash"), - capabilities=_capabilities_from_payload(value.get("capabilities")), + capabilities=capabilities, receipt=receipt, artifacts=artifacts, wall_time=float(wall_time), frame_key=frame_key, weight_key=weight_key, opaque_artifacts=opaque_artifacts, + legacy_capabilities=legacy_capabilities, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/population.py b/packages/microcosm-graph/src/microcosm/graph/population.py index 98a551303..1a52baf3f 100644 --- a/packages/microcosm-graph/src/microcosm/graph/population.py +++ b/packages/microcosm-graph/src/microcosm/graph/population.py @@ -16,6 +16,7 @@ from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights +from .canonical import canonical_json from .decl import ( MASS_POLICIES, ROWS_ALL, @@ -34,6 +35,9 @@ "dtype_for_token", "dtype_matches", "expand_lineage_receipt", + "entrant_strata_receipt", + "mass_record_receipt", + "expand_writes_receipt", "owned_ids", "patch", "population_from_frame", @@ -119,6 +123,14 @@ class MassRecord: before_by_stratum: tuple[tuple[object, float], ...] after_by_stratum: tuple[tuple[object, float], ...] entity: str | None = None + partition_entity: str | None = None + partition_column: str | None = None + before_by_partition_stratum: tuple[ + tuple[object, tuple[tuple[object, float], ...]], ... + ] = () + after_by_partition_stratum: tuple[ + tuple[object, tuple[tuple[object, float], ...]], ... + ] = () @property def before_strata(self) -> Mapping[object, float]: @@ -128,6 +140,24 @@ def before_strata(self) -> Mapping[object, float]: def after_strata(self) -> Mapping[object, float]: return MappingProxyType(dict(self.after_by_stratum)) + @property + def before_partitions(self) -> Mapping[object, Mapping[object, float]]: + return MappingProxyType( + { + partition: MappingProxyType(dict(strata)) + for partition, strata in self.before_by_partition_stratum + } + ) + + @property + def after_partitions(self) -> Mapping[object, Mapping[object, float]]: + return MappingProxyType( + { + partition: MappingProxyType(dict(strata)) + for partition, strata in self.after_by_partition_stratum + } + ) + @property def old_total(self) -> float: """Compatibility spelling used by :mod:`microcosm.frame`.""" @@ -141,6 +171,66 @@ def new_total(self) -> float: return self.after_total +def _receipt_key(value: object) -> str: + """Return the JSON-object-key spelling of a partition or stratum value.""" + + if isinstance(value, np.generic): + value = value.item() + return str(value) + + +def _partition_receipt_mapping( + values: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], +) -> dict[str, dict[str, float]]: + result: dict[str, dict[str, float]] = {} + for partition, strata in values: + partition_key = _receipt_key(partition) + if partition_key in result: + raise PopulationError( + f"Partition values collide as JSON key {partition_key!r}." + ) + converted: dict[str, float] = {} + for stratum, mass in strata: + stratum_key = _receipt_key(stratum) + if stratum_key in converted: + raise PopulationError( + f"Strata collide as JSON key {stratum_key!r} inside partition " + f"{partition_key!r}." + ) + converted[stratum_key] = float(mass) + result[partition_key] = converted + return result + + +def mass_record_receipt(record: MassRecord) -> dict[str, object]: + """Return executor-authored public mass accounting for one ledger record.""" + + payload: dict[str, object] = { + "policy": record.policy, + "before": record.before_total, + "after": record.after_total, + "stratum_before": _receipt_mass_mapping( + dict(record.before_by_stratum), label=f"Node {record.node_id!r} mass" + ), + "stratum_after": _receipt_mass_mapping( + dict(record.after_by_stratum), label=f"Node {record.node_id!r} mass" + ), + } + if record.partition_entity is not None: + assert record.partition_column is not None + payload["partition"] = { + "entity": record.partition_entity, + "column": record.partition_column, + "stratum_before": _partition_receipt_mapping( + record.before_by_partition_stratum + ), + "stratum_after": _partition_receipt_mapping( + record.after_by_partition_stratum + ), + } + return payload + + @dataclass(frozen=True) class Population: """One immutable graph view over a validated :class:`Frame`.""" @@ -272,9 +362,15 @@ def population_from_frame( ) -def _lineage_json_scalar(value: object) -> str | int | float | bool: +def _lineage_json_scalar( + value: object, *, allow_null: bool = False +) -> str | int | float | bool | None: """Detach one entity id into the scalar vocabulary accepted by receipts.""" + if pd.isna(value): + if allow_null: + return None + raise PopulationError(f"EXPAND lineage id {value!r} is not a JSON scalar.") if isinstance(value, np.generic): value = value.item() if not isinstance(value, str | int | float | bool): @@ -284,6 +380,46 @@ def _lineage_json_scalar(value: object) -> str | int | float | bool: return value +def _stratum_receipt_scalar(value: object) -> object: + """Encode one cache-safe entrant stratum label for a JSON receipt.""" + + missing = pd.isna(value) + if isinstance(missing, bool | np.bool_) and bool(missing): + raise PopulationError("EXPAND entrant stratum labels cannot be null.") + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, bytes): + return {"bytes_hex": value.hex()} + if not isinstance(value, str | int | float | bool): + raise PopulationError( + f"EXPAND entrant stratum label {value!r} is not a cache-safe scalar." + ) + if isinstance(value, float) and not np.isfinite(value): + raise PopulationError(f"EXPAND entrant stratum label {value!r} is not finite.") + return value + + +def _stratum_from_receipt_scalar(value: object) -> object: + """Decode one executor-authored entrant stratum label from a receipt.""" + + if isinstance(value, Mapping): + if set(value) != {"bytes_hex"} or not isinstance(value["bytes_hex"], str): + raise PopulationError("EXPAND entrant stratum receipt label is malformed.") + encoded = value["bytes_hex"] + try: + decoded = bytes.fromhex(encoded) + except ValueError as error: + raise PopulationError( + "EXPAND entrant stratum receipt bytes are malformed." + ) from error + if encoded != decoded.hex(): + raise PopulationError( + "EXPAND entrant stratum receipt bytes are not canonical." + ) + return decoded + return _stratum_receipt_scalar(value) + + def expand_lineage_receipt( expand: Mapping[str, pd.Series], ) -> dict[str, list[list[object]]]: @@ -301,7 +437,10 @@ def expand_lineage_receipt( f"EXPAND lineage for {entity!r} is not a pandas Series." ) payload[entity] = [ - [_lineage_json_scalar(target), _lineage_json_scalar(source)] + [ + _lineage_json_scalar(target), + _lineage_json_scalar(source, allow_null=True), + ] for target, source in zip( lineage.index.tolist(), lineage.tolist(), strict=True ) @@ -309,6 +448,195 @@ def expand_lineage_receipt( return payload +def _entrant_person_ids(frame: Frame, lineage: Mapping[str, pd.Series]) -> pd.Index: + """Return null-lineage person targets in declared lineage order.""" + + person = frame.schema.person_entity + id_column = frame.schema.entity_id_column(person) + person_lineage = lineage[person] + entrant_positions = np.flatnonzero( + person_lineage.isna().to_numpy(dtype=np.bool_, copy=False) + ) + return pd.Index(person_lineage.index.take(entrant_positions), name=id_column) + + +def _validated_entrant_strata( + frame: Frame, + node: Node, + lineage: Mapping[str, pd.Series], + raw: object, +) -> pd.Series | None: + """Validate and align the iff contract for entrant-person strata.""" + + person = frame.schema.person_entity + id_column = frame.schema.entity_id_column(person) + id_dtype = frame.table(person)[id_column].dtype + entrant_ids = _entrant_person_ids(frame, lineage) + if not len(entrant_ids): + if raw is not None: + raise PopulationError( + f"EXPAND node {node.id!r} returned strata without entrant persons." + ) + return None + if raw is None: + raise PopulationError( + f"EXPAND node {node.id!r} omitted strata for entrant persons " + f"{entrant_ids[:5].tolist()}." + ) + if not isinstance(raw, pd.Series): + raise PopulationError( + f"EXPAND node {node.id!r} entrant strata is not a Series." + ) + labels_index = pd.Index(raw.index, name=id_column) + if labels_index.nlevels != 1 or labels_index.dtype != id_dtype: + raise PopulationError( + f"EXPAND node {node.id!r} entrant strata index must use " + f"{id_dtype!s} person ids." + ) + if not labels_index.is_unique: + raise PopulationError( + f"EXPAND node {node.id!r} repeats entrant strata person ids." + ) + if labels_index.isna().any(): + raise PopulationError( + f"EXPAND node {node.id!r} entrant strata contains null person ids." + ) + missing = entrant_ids[~entrant_ids.isin(labels_index)] + extra = labels_index[~labels_index.isin(entrant_ids)] + if len(missing) or len(extra): + raise PopulationError( + f"EXPAND node {node.id!r} entrant strata must name exactly the entrant " + f"persons; missing={missing[:5].tolist()}, extra={extra[:5].tolist()}." + ) + if not ( + pd.api.types.is_object_dtype(raw.dtype) or isinstance(raw.dtype, pd.StringDtype) + ): + raise PopulationError( + f"EXPAND node {node.id!r} entrant strata must use object or string " + f"labels, got {raw.dtype!s}." + ) + if raw.isna().any(): + raise PopulationError( + f"EXPAND node {node.id!r} entrant strata contains missing labels." + ) + aligned = raw.reindex(entrant_ids).copy() + for value in aligned.array: + _stratum_receipt_scalar(value) + return aligned + + +def entrant_strata_receipt( + frame: Frame, + node: Node, + expand: Mapping[str, pd.Series], + strata: pd.Series | None, +) -> list[list[object]] | None: + """Return executor-authored entrant-person strata in lineage order.""" + + lineage = _validate_expand_lineage(frame, node, expand) + aligned = _validated_entrant_strata(frame, node, lineage, strata) + if aligned is None: + return None + return [ + [_lineage_json_scalar(target), _stratum_receipt_scalar(label)] + for target, label in zip(aligned.index, aligned.array, strict=True) + ] + + +def _cached_entrant_strata( + frame: Frame, + node: Node, + lineage: Mapping[str, pd.Series], + receipt: Mapping[str, object], +) -> pd.Series | None: + """Parse the executor-authored entrant-strata cache attestation.""" + + person = frame.schema.person_entity + id_column = frame.schema.entity_id_column(person) + id_dtype = frame.table(person)[id_column].dtype + entrant_ids = _entrant_person_ids(frame, lineage) + if not len(entrant_ids): + if "entrant_strata" in receipt: + raise PopulationError( + f"Cached EXPAND node {node.id!r} has entrant strata without " + "entrant persons." + ) + return None + raw = receipt.get("entrant_strata") + if not isinstance(raw, list): + raise PopulationError( + f"Cached EXPAND node {node.id!r} has no entrant-strata receipt." + ) + targets: list[object] = [] + labels: list[object] = [] + for entry in raw: + if not isinstance(entry, list) or len(entry) != 2: + raise PopulationError( + f"Cached EXPAND node {node.id!r} has malformed entrant strata." + ) + targets.append(entry[0]) + labels.append(_stratum_from_receipt_scalar(entry[1])) + try: + target_index = pd.Index( + pd.Series(targets, dtype=id_dtype).array, name=id_column + ) + except (TypeError, ValueError) as error: + raise PopulationError( + f"Cached EXPAND node {node.id!r} entrant strata contain invalid " + f"{id_dtype!s} person ids." + ) from error + if not target_index.equals(entrant_ids): + raise PopulationError( + f"Cached EXPAND node {node.id!r} entrant strata do not name its " + "entrant persons in lineage order." + ) + return pd.Series(labels, index=entrant_ids, dtype=object) + + +def _assert_cached_expand_strata( + before: Frame, + after: Frame, + node: Node, + lineage: Mapping[str, pd.Series], + receipt: Mapping[str, object], +) -> None: + """Verify cached incumbent, copied, and entrant person strata by lineage.""" + + person = before.schema.person_entity + id_column = before.schema.entity_id_column(person) + before_ids = pd.Index(before.table(person)[id_column], name=id_column) + person_lineage = lineage[person] + expected_ids = before_ids.append(pd.Index(person_lineage.index, name=id_column)) + after_ids = pd.Index(after.table(person)[id_column], name=id_column) + if not after_ids.equals(expected_ids): + raise PopulationError(f"Cached EXPAND node {node.id!r} reordered person ids.") + entrant_strata = _cached_entrant_strata(before, node, lineage, receipt) + expected = before.strata.astype(object).tolist() + entrant_positions: list[int] = [] + for target, source in zip(person_lineage.index, person_lineage.array, strict=True): + if pd.isna(source): + assert entrant_strata is not None + entrant_positions.append(len(expected)) + expected.append(entrant_strata.loc[target]) + continue + source_position = before_ids.get_loc(source) + expected.append(before.strata.iloc[source_position]) + actual = after.strata.astype(object).reset_index(drop=True) + if not actual.equals(pd.Series(expected, dtype=object)): + raise PopulationError( + f"Cached EXPAND node {node.id!r} strata disagree with its lineage " + "and entrant-strata receipt." + ) + for position in entrant_positions: + actual_label = _stratum_receipt_scalar(actual.iloc[position]) + expected_label = _stratum_receipt_scalar(expected[position]) + if canonical_json(actual_label) != canonical_json(expected_label): + raise PopulationError( + f"Cached EXPAND node {node.id!r} entrant stratum label " + "disagrees with its receipt." + ) + + def _expand_lineage_from_receipt( frame: Frame, node: Node, @@ -340,11 +668,21 @@ def _expand_lineage_from_receipt( ) id_column = frame.schema.entity_id_column(entity) dtype = frame.table(entity)[id_column].dtype + source_dtype: object = dtype + has_null_source = any(pd.isna(value) for value in sources) + if has_null_source and pd.api.types.is_bool_dtype(dtype): + source_dtype = pd.BooleanDtype() + elif has_null_source and pd.api.types.is_integer_dtype(dtype): + numpy_dtype = np.dtype(getattr(dtype, "numpy_dtype", dtype)) + prefix = "UInt" if np.issubdtype(numpy_dtype, np.unsignedinteger) else "Int" + source_dtype = pd.api.types.pandas_dtype( + f"{prefix}{numpy_dtype.itemsize * 8}" + ) lineage[entity] = pd.Series( sources, index=pd.Index(pd.Series(targets, dtype=dtype).array, name=id_column), name=id_column, - dtype=dtype, + dtype=source_dtype, ) return lineage @@ -378,7 +716,19 @@ def _validate_expand_lineage( id_column = frame.schema.entity_id_column(entity) source_ids = pd.Index(frame.table(entity)[id_column], name=id_column) targets = pd.Index(lineage.index, name=id_column) - if targets.dtype != source_ids.dtype or lineage.dtype != source_ids.dtype: + source_is_null = lineage.isna().to_numpy(dtype=np.bool_, copy=False) + nullable_sources = bool(source_is_null.any()) + nullable_source_dtype = getattr(lineage.dtype, "numpy_dtype", None) + # A declared-entrants node may always carry a nullable id dtype, even + # on a run that admits no entrant; only its numpy dtype must match. + if targets.dtype != source_ids.dtype or ( + lineage.dtype != source_ids.dtype + and not ( + (nullable_sources or node.entrants) + and nullable_source_dtype is not None + and np.dtype(nullable_source_dtype) == np.dtype(source_ids.dtype) + ) + ): raise PopulationError( f"EXPAND node {node.id!r} lineage for {entity!r} must use " f"{source_ids.dtype!s} ids for both targets and sources." @@ -387,9 +737,15 @@ def _validate_expand_lineage( raise PopulationError( f"EXPAND node {node.id!r} repeats new target {entity!r} ids." ) - if targets.isna().any() or lineage.isna().any(): + if targets.isna().any(): raise PopulationError( - f"EXPAND node {node.id!r} lineage for {entity!r} contains null ids." + f"EXPAND node {node.id!r} lineage for {entity!r} contains null " + "target ids." + ) + if nullable_sources and not node.entrants: + raise PopulationError( + f"EXPAND node {node.id!r} lineage for {entity!r} contains null " + "source ids without entrants=True." ) collisions = targets.intersection(source_ids) if len(collisions): @@ -397,31 +753,143 @@ def _validate_expand_lineage( f"EXPAND node {node.id!r} lineage target {entity!r} ids collide " f"with incumbents {collisions[:5].tolist()}." ) - source_positions = source_ids.get_indexer(lineage.to_numpy(copy=False)) + source_positions = np.full(len(lineage), -1, dtype=np.int64) + copied = ~source_is_null + source_positions[copied] = source_ids.get_indexer( + lineage.iloc[np.flatnonzero(copied)].to_numpy(copy=False) + ) if (source_positions < 0).any(): - bad = lineage.iloc[np.flatnonzero(source_positions < 0)[:5]].tolist() - raise PopulationError( - f"EXPAND node {node.id!r} lineage names unknown {entity!r} " - f"source ids {bad}." - ) + unknown = copied & (source_positions < 0) + if unknown.any(): + bad = lineage.iloc[np.flatnonzero(unknown)[:5]].tolist() + raise PopulationError( + f"EXPAND node {node.id!r} lineage names unknown {entity!r} " + f"source ids {bad}." + ) if after is not None: after_ids = pd.Index(after.table(entity)[id_column], name=id_column) if not source_ids.isin(after_ids).all(): raise PopulationError( f"Cached EXPAND node {node.id!r} dropped incumbent {entity!r} ids." ) - additions = after_ids[~after_ids.isin(source_ids)] - if not additions.equals(targets): + expected_ids = source_ids.append(targets) + if not after_ids.equals(expected_ids): raise PopulationError( - f"Cached EXPAND node {node.id!r} frame additions for " - f"{entity!r} disagree with its lineage receipt." + f"Cached EXPAND node {node.id!r} final {entity!r} ids " + "disagree with its lineage receipt." ) validated[entity] = lineage return validated +def _assert_copied_expand_storage( + before: Frame, + after: Frame, + node: Node, + lineage: Mapping[str, pd.Series], + *, + rewrites: frozenset[tuple[str, str]], +) -> None: + """Revalidate cached EXPAND storage against its cold-path invariants.""" + + person = before.schema.person_entity + declared: dict[str, dict[str, str]] = {entity: {} for entity in before.entities} + for entity, column, dtype in _expand_cells(node): + if entity not in declared: + raise PopulationError( + f"Cached EXPAND node {node.id!r} names unknown entity {entity!r}." + ) + declared[entity][column] = dtype + + for entity in before.entities: + before_table = before.table(entity) + after_table = after.table(entity) + expected_columns = set(before_table.columns) | set(declared[entity]) + actual_columns = set(after_table.columns) + if actual_columns != expected_columns: + missing = sorted(expected_columns - actual_columns, key=str) + extra = sorted(actual_columns - expected_columns, key=str) + raise PopulationError( + f"Cached EXPAND node {node.id!r} {entity!r} column set differs " + f"from its declaration (missing={missing}, extra={extra})." + ) + for column, dtype in declared[entity].items(): + assert_dtype( + after_table[column], + dtype, + label=f"Cached EXPAND node {node.id!r} cell {entity}.{column}", + ) + for column in before_table.columns: + incumbent = ( + after_table[column].iloc[: len(before_table)].reset_index(drop=True) + ) + if not storage_equal(before_table[column], incumbent): + raise PopulationError( + f"Cached EXPAND node {node.id!r} changed incumbent storage " + f"in {entity}.{column}." + ) + + entity_lineage = lineage[entity] + copied = ~entity_lineage.isna().to_numpy(dtype=np.bool_, copy=False) + if not copied.any(): + continue + + id_column = before.schema.entity_id_column(entity) + structural = {id_column} + if entity == person: + structural.update( + before.schema.membership_column(group) + for group in before.schema.group_entities + ) + before_ids = pd.Index(before_table[id_column], name=id_column) + after_ids = pd.Index(after_table[id_column], name=id_column) + copied_lineage = entity_lineage.iloc[np.flatnonzero(copied)] + source_ids = copied_lineage.tolist() + target_ids = copied_lineage.index.tolist() + source_positions = before_ids.get_indexer(source_ids) + target_positions = after_ids.get_indexer(target_ids) + if (source_positions < 0).any() or (target_positions < 0).any(): + raise PopulationError( + f"EXPAND node {node.id!r} cannot align copied {entity!r} rows." + ) + + for column in before_table.columns: + column = str(column) + if column in structural or (entity, column) in rewrites: + continue + copied_pairs = list(zip(target_ids, source_ids, strict=True)) + if column not in after_table: + mismatches = copied_pairs[:5] + else: + source_values = ( + before_table[column].iloc[source_positions].reset_index(drop=True) + ) + target_values = ( + after_table[column].iloc[target_positions].reset_index(drop=True) + ) + if storage_equal(source_values, target_values): + continue + mismatches = [ + pair + for position, pair in enumerate(copied_pairs) + if not storage_equal( + source_values.iloc[[position]].reset_index(drop=True), + target_values.iloc[[position]].reset_index(drop=True), + ) + ][:5] + raise PopulationError( + f"EXPAND node {node.id!r} changed carried storage in " + f"{entity}.{column} for copied target/source ids {mismatches}." + ) + + def restore_cached_expand( - population: Population, node: Node, result: KernelResult + population: Population, + node: Node, + result: KernelResult, + *, + mass_partition: tuple[str, str] | None = None, + rewrite_coordinates: frozenset[tuple[str, str]] = frozenset(), ) -> Population: """Restore a previously validated EXPAND frame against its keyed base. @@ -430,10 +898,18 @@ def restore_cached_expand( to this base version. This function re-establishes graph ownership, design-weight ancestry, and the executor mass ledger without relaxing the miss-path lineage validation. + ``rewrite_coordinates`` is the executor-validated set of full-cell rewrite + claims in this EXPAND version; only those carried cells may differ between + a copied row and its source. """ if node.structural is not StructuralDelta.EXPAND or result.frame is None: raise PopulationError("restore_cached_expand requires an EXPAND Frame.") + if result.strata is not None: + raise PopulationError( + f"Cached EXPAND node {node.id!r} returned kernel strata instead of " + "its executor frame artifact." + ) frame = result.frame if frame.schema != population.frame.schema: raise PopulationError(f"Cached EXPAND node {node.id!r} changed schema.") @@ -443,6 +919,20 @@ def restore_cached_expand( lineage = _validate_expand_lineage( population.frame, node, receipt_lineage, after=frame ) + _assert_expand_memberships( + population.frame, + {entity: frame.table(entity) for entity in frame.entities}, + lineage, + node, + ) + _assert_copied_expand_storage( + population.frame, + frame, + node, + lineage, + rewrites=rewrite_coordinates, + ) + _assert_cached_expand_strata(population.frame, frame, node, lineage, result.receipt) _assert_expand_weights(population, frame, node, result) design_weights: dict[str, np.ndarray] = {} @@ -457,18 +947,46 @@ def restore_cached_expand( values[retained] = old_anchor[positions[retained]] if not retained.all(): sources = lineage[entity].reindex(after_ids[~retained]) - source_positions = before_ids.get_indexer(sources.to_numpy(copy=False)) + source_is_null = sources.isna().to_numpy(dtype=np.bool_, copy=False) + introduced_positions = np.flatnonzero(~retained) + copied_positions = introduced_positions[~source_is_null] + source_positions = before_ids.get_indexer( + sources.iloc[np.flatnonzero(~source_is_null)].to_numpy(copy=False) + ) if (source_positions < 0).any(): # defended by lineage validation raise PopulationError( f"Cached EXPAND node {node.id!r} has unknown design lineage " f"for new {entity!r} ids." ) - values[~retained] = old_anchor[source_positions] + values[copied_positions] = old_anchor[source_positions] + entrant_positions = introduced_positions[source_is_null] + if len(entrant_positions): + try: + current = frame.weights_for(entity) + except ValueError as error: + raise PopulationError( + f"Cached EXPAND node {node.id!r} has no design anchor for " + f"entrant {entity!r} ids." + ) from error + if current.kind is not WeightKind.DESIGN: + raise PopulationError( + f"Cached EXPAND node {node.id!r} cannot anchor entrant " + f"{entity!r} ids from {current.kind.value!r} weights; " + "explicit design weights are required." + ) + values[entrant_positions] = current.values[entrant_positions] design_weights[entity] = values ledger = ( *population.mass_ledger, - _mass_record(population.frame, frame, node, result, _mass_policy(node)), + _mass_record( + population.frame, + frame, + node, + result, + _mass_policy(node), + mass_partition=mass_partition, + ), ) owners = { (entity, str(column)): node.id @@ -531,7 +1049,14 @@ def storage_equal( return _storage_parts(left, selected) == _storage_parts(right, selected) -def patch(population: Population, node: Node, result: KernelResult) -> Population: +def patch( + population: Population, + node: Node, + result: KernelResult, + *, + mass_partition: tuple[str, str] | None = None, + rewrite_coordinates: frozenset[tuple[str, str]] = frozenset(), +) -> Population: """Validate and apply one node result without mutating ``population``. ``EXPAND`` kernels return only the new-id to source-id mapping through @@ -540,6 +1065,9 @@ def patch(population: Population, node: Node, result: KernelResult) -> Populatio overlays enumerated by ``params['expand_cells']``. ``result.weights`` is the full target vector named by ``params['expand_weight_entity']`` and ``params['expand_weight_kind']``. + ``rewrite_coordinates`` is the executor-validated set of full-cell rewrite + claims in this EXPAND version; only those carried cells may differ between + a copied row and its source. """ if node.structural is StructuralDelta.CREATE: @@ -554,6 +1082,10 @@ def patch(population: Population, node: Node, result: KernelResult) -> Populatio _assert_no_ordinary_structural_outputs(population, node) expected_columns = {(owned.entity, owned.column) for owned in node.outputs} lineage_expand = node.structural is StructuralDelta.EXPAND and result.frame is None + if result.strata is not None and not lineage_expand: + raise PopulationError( + f"Node {node.id!r} returned entrant strata outside a lineage EXPAND." + ) if not lineage_expand and set(result.columns) != expected_columns: raise PopulationError( f"Node {node.id!r} returned columns {sorted(result.columns)}; " @@ -566,7 +1098,9 @@ def patch(population: Population, node: Node, result: KernelResult) -> Populatio raise PopulationError(f"Non-structural node {node.id!r} returned a Frame.") frame, owners = _patch_columns(population, node, result) elif lineage_expand: - frame, owners = _patch_expand(population, node, result) + frame, owners = _patch_expand( + population, node, result, rewrite_coordinates=rewrite_coordinates + ) else: frame, owners = _patch_structural(population, node, result) @@ -602,7 +1136,14 @@ def patch(population: Population, node: Node, result: KernelResult) -> Populatio ledger = population.mass_ledger if node.structural is not StructuralDelta.NONE or node.weights is not None: policy = _mass_policy(node) - record = _mass_record(before, frame, node, result, policy) + record = _mass_record( + before, + frame, + node, + result, + policy, + mass_partition=mass_partition, + ) ledger = (*ledger, record) return Population.from_frame( @@ -633,6 +1174,11 @@ def _expand_cells(node: Node) -> tuple[tuple[str, str, str], ...]: f"EXPAND node {node.id!r} has malformed expand_cells entry {item!r}." ) entity, column, dtype = item + if "." in entity or "." in column: + raise PopulationError( + f"EXPAND node {node.id!r} params['expand_cells'] entity and " + f"column names must be dot-free; got {entity!r}, {column!r}." + ) # Reuse the declaration token validator without importing frozen # declaration internals into this runtime convention. dtype_for_token(dtype) @@ -658,17 +1204,19 @@ def _targets_by_source(lineage: pd.Series) -> dict[object, list[object]]: grouped: dict[object, list[object]] = {} for target, source in zip(lineage.index, lineage.array, strict=True): + if pd.isna(source): + continue grouped.setdefault(source, []).append(target) return grouped -def _remap_expand_memberships( +def _remapped_expand_memberships( before: Frame, tables: Mapping[str, pd.DataFrame], lineage: Mapping[str, pd.Series], node: Node, -) -> None: - """Make each copied person's memberships follow the copied groups. +) -> dict[str, pd.Series]: + """Return memberships whose copied rows follow their copied groups. Multiple copies use strict ordinal alignment: the nth copy of every member follows the nth copy of its source group. A copied group therefore requires @@ -680,14 +1228,17 @@ def _remap_expand_memberships( source_person = before.table(person) person_id = before.schema.entity_id_column(person) source_person_ids = pd.Index(source_person[person_id]) + entrant_mask = person_lineage.isna().to_numpy(dtype=np.bool_, copy=False) + copied_indices = np.flatnonzero(~entrant_mask) source_positions = source_person_ids.get_indexer( - person_lineage.to_numpy(copy=False) + person_lineage.iloc[copied_indices].to_numpy(copy=False) ) if (source_positions < 0).any(): # defended by lineage validation raise PopulationError( f"EXPAND node {node.id!r} cannot align copied person memberships." ) person_targets = _targets_by_source(person_lineage) + result: dict[str, pd.Series] = {} for group in before.schema.group_entities: membership = before.schema.membership_column(group) @@ -708,16 +1259,24 @@ def _remap_expand_memberships( ) seen: dict[object, int] = {} - remapped: list[object] = [] - for source_position, source_person_id in zip( - source_positions, person_lineage.array, strict=True + remapped = ( + tables[person][membership] + .iloc[len(source_person) :] + .reset_index(drop=True) + .copy() + ) + for addition_position, source_position, source_person_id in zip( + copied_indices, + source_positions, + person_lineage.iloc[copied_indices].array, + strict=True, ): # Select the membership Series directly. Selecting a mixed-type # DataFrame row can coerce a large integer group id through float. source_group = source_person[membership].iloc[source_position] candidates = group_targets.get(source_group, []) if not candidates: - remapped.append(source_group) + remapped.iloc[addition_position] = source_group continue ordinal = seen.get(source_person_id, 0) if ordinal >= len(candidates): @@ -725,18 +1284,126 @@ def _remap_expand_memberships( f"EXPAND node {node.id!r} cannot align {membership!r} for " f"copied person {source_person_id!r}." ) - remapped.append(candidates[ordinal]) + remapped.iloc[addition_position] = candidates[ordinal] seen[source_person_id] = ordinal + 1 carried = source_person[membership].reset_index(drop=True) - additions = pd.Series(remapped, dtype=source_person[membership].dtype) - tables[person][membership] = pd.concat( - [carried, additions], ignore_index=True - ).array + result[membership] = pd.concat([carried, remapped], ignore_index=True) + return result + + +def _remap_expand_memberships( + before: Frame, + tables: Mapping[str, pd.DataFrame], + lineage: Mapping[str, pd.Series], + node: Node, +) -> dict[str, pd.Series]: + """Install the lineage-derived memberships for copied persons.""" + + person = before.schema.person_entity + remapped = _remapped_expand_memberships(before, tables, lineage, node) + for membership, values in remapped.items(): + tables[person][membership] = values.array + return remapped + + +def _assert_expand_memberships( + before: Frame, + tables: Mapping[str, pd.DataFrame], + lineage: Mapping[str, pd.Series], + node: Node, + *, + remapped: Mapping[str, pd.Series] | None = None, +) -> frozenset[tuple[str, str]]: + """Validate copied and entrant memberships; return copied re-pointings.""" + + person = before.schema.person_entity + person_lineage = lineage[person] + entrant_mask = person_lineage.isna().to_numpy(dtype=np.bool_, copy=False) + copied_positions = np.flatnonzero(~entrant_mask) + entrant_positions = np.flatnonzero(entrant_mask) + if not len(copied_positions) and not len(entrant_positions): + return frozenset() + + if remapped is None and len(copied_positions): + remapped = _remapped_expand_memberships(before, tables, lineage, node) + overlay_coordinates = { + (entity, column) for entity, column, _ in _expand_cells(node) + } + person_table = tables[person] + source_person_count = before.n(person) + repointed: set[tuple[str, str]] = set() + for group in before.schema.group_entities: + membership = before.schema.membership_column(group) + coordinate = (person, membership) + actual = person_table[membership].reset_index(drop=True) + group_lineage = lineage[group] + group_entrant_mask = group_lineage.isna().to_numpy(dtype=np.bool_, copy=False) + entrant_group_ids = pd.Index(group_lineage.index[group_entrant_mask]) + if len(copied_positions): + assert remapped is not None + expected = remapped[membership] + for addition_position in copied_positions: + row_position = source_person_count + int(addition_position) + if storage_equal( + actual.iloc[[row_position]].reset_index(drop=True), + expected.iloc[[row_position]].reset_index(drop=True), + ): + continue + offending_group_id = actual.iloc[row_position] + if ( + coordinate in overlay_coordinates + and not pd.isna(offending_group_id) + and offending_group_id in entrant_group_ids + ): + repointed.add(coordinate) + continue + person_id = _lineage_json_scalar( + person_lineage.index[addition_position] + ) + named_group_id = _lineage_json_scalar(offending_group_id) + raise PopulationError( + f"EXPAND node {node.id!r} membership overlay " + f"{person}.{membership} re-pointed copied {person} id " + f"{person_id!r} to {group} id {named_group_id!r}; only its " + f"lineage-remapped group or a same-EXPAND entrant {group} " + "is allowed." + ) + + group_id = before.schema.entity_id_column(group) + incumbent_group_ids = pd.Index(before.table(group)[group_id]) + copied_group_ids = pd.Index(group_lineage.index[~group_entrant_mask]) + for addition_position in entrant_positions: + row_position = source_person_count + int(addition_position) + offending_group_id = actual.iloc[row_position] + if not pd.isna(offending_group_id) and ( + offending_group_id in incumbent_group_ids + or offending_group_id in entrant_group_ids + ): + continue + person_id = _lineage_json_scalar(person_lineage.index[addition_position]) + named_group_id = _lineage_json_scalar(offending_group_id, allow_null=True) + group_kind = ( + "copied" + if not pd.isna(offending_group_id) + and offending_group_id in copied_group_ids + else "unknown" + ) + raise PopulationError( + f"EXPAND node {node.id!r} membership overlay {person}.{membership} " + f"assigned entrant {person} id {person_id!r} to {group_kind} " + f"{group} id {named_group_id!r}; entrant persons may join only " + f"incumbent or same-EXPAND entrant {group} ids." + ) + return frozenset(repointed) def _patch_expand( - population: Population, node: Node, result: KernelResult + population: Population, + node: Node, + result: KernelResult, + *, + rewrite_coordinates: frozenset[tuple[str, str]], ) -> tuple[Frame, dict[tuple[str, str], str]]: """Materialize a source-lineage EXPAND result in the executor.""" @@ -746,11 +1413,17 @@ def _patch_expand( f"EXPAND node {node.id!r} cannot yet carry association link tables." ) cells = _expand_cells(node) - for entity, _, _ in cells: + for entity, column, _ in cells: if entity not in before.entities: raise PopulationError( f"EXPAND node {node.id!r} names unknown entity {entity!r}." ) + id_column = before.schema.entity_id_column(entity) + if column == id_column: + raise PopulationError( + f"EXPAND node {node.id!r} cannot overlay entity id column " + f"{entity}.{column}; lineage supplies final ids." + ) cell_coordinates = {(entity, column) for entity, column, _ in cells} if set(result.columns) != cell_coordinates: @@ -761,7 +1434,6 @@ def _patch_expand( lineage = _validate_expand_lineage(before, node, result.expand) - tables: dict[str, pd.DataFrame] = {} lineage_positions: dict[str, np.ndarray] = {} target_ids: dict[str, pd.Index] = {} for entity in before.entities: @@ -772,29 +1444,40 @@ def _patch_expand( source_table[id_column].to_numpy(copy=True), name=id_column ) new_targets = pd.Index(entity_lineage.index, name=id_column) - targets = source_ids.append(new_targets) - source_positions = source_ids.get_indexer(entity_lineage.to_numpy(copy=False)) - positions = np.concatenate( - [np.arange(len(source_ids), dtype=np.int64), source_positions] + target_ids[entity] = source_ids.append(new_targets) + entrants = entity_lineage.isna().to_numpy(dtype=np.bool_, copy=False) + source_positions = np.full(len(entity_lineage), -1, dtype=np.int64) + copied = ~entrants + source_positions[copied] = source_ids.get_indexer( + entity_lineage.iloc[np.flatnonzero(copied)].to_numpy(copy=False) ) - carried = source_table.iloc[positions].reset_index(drop=True) - replacement_ids = pd.Series( - targets.to_numpy(copy=True), dtype=source_table[id_column].dtype + lineage_positions[entity] = np.concatenate( + [np.arange(len(source_ids), dtype=np.int64), source_positions] ) - if len(replacement_ids) != len(carried): - raise PopulationError( - f"EXPAND node {node.id!r} lineage index/value lengths disagree " - f"for {entity!r}." - ) - carried[id_column] = replacement_ids.array - tables[entity] = carried - lineage_positions[entity] = positions - target_ids[entity] = targets - _remap_expand_memberships(before, tables, lineage, node) + if entrants.any(): + carried = { + (entity, str(column)) + for column in source_table.columns + if column != id_column + } + missing = sorted(carried - cell_coordinates) + if missing: + names = [ + f"{carried_entity}.{column}" for carried_entity, column in missing + ] + raise PopulationError( + f"EXPAND node {node.id!r} entrant rows do not materialize " + f"carried columns {names}." + ) + + person = before.schema.person_entity + entrant_strata = _validated_entrant_strata(before, node, lineage, result.strata) + aligned_cells: dict[tuple[str, str], pd.Series] = {} for entity, column, dtype in cells: - incoming = result.columns[(entity, column)] + coordinate = (entity, column) + incoming = result.columns[coordinate] if not isinstance(incoming, pd.Series): raise PopulationError( f"EXPAND node {node.id!r} cell {entity}.{column} is not a Series." @@ -815,8 +1498,67 @@ def _patch_expand( dtype, label=f"EXPAND node {node.id!r} cell {entity}.{column}", ) + source_table = before.table(entity) + if column in source_table: + carried_dtype = token_for_dtype(source_table[column].dtype) + if dtype != carried_dtype: + raise PopulationError( + f"EXPAND node {node.id!r} carried cell {entity}.{column} " + f"declares {dtype!r}; its incumbent dtype is {carried_dtype!r}." + ) + incumbent = aligned.iloc[: len(source_table)].reset_index(drop=True) + if not storage_equal(source_table[column], incumbent): + raise PopulationError( + f"EXPAND node {node.id!r} changed carried storage in " + f"{entity}.{column} for incumbent rows." + ) + aligned_cells[coordinate] = aligned + + tables: dict[str, pd.DataFrame] = {} + for entity in before.entities: + id_column = before.schema.entity_id_column(entity) + source_table = before.table(entity) + positions = lineage_positions[entity] + addition_positions = positions[len(source_table) :] + if len(source_table): + additions = source_table.iloc[ + np.maximum(addition_positions, 0) + ].reset_index(drop=True) + else: + additions = source_table.reindex(range(len(addition_positions))).copy() + carried = pd.concat( + [source_table.reset_index(drop=True), additions], ignore_index=True + ) + replacement_ids = pd.Series( + target_ids[entity].to_numpy(copy=True), + dtype=source_table[id_column].dtype, + ) + if len(replacement_ids) != len(carried): + raise PopulationError( + f"EXPAND node {node.id!r} lineage index/value lengths disagree " + f"for {entity!r}." + ) + carried[id_column] = replacement_ids.array + tables[entity] = carried + + remapped_memberships = _remap_expand_memberships(before, tables, lineage, node) + + for (entity, column), aligned in aligned_cells.items(): tables[entity][column] = aligned.array + _assert_expand_memberships( + before, tables, lineage, node, remapped=remapped_memberships + ) + + for entity, expected_ids in target_ids.items(): + id_column = before.schema.entity_id_column(entity) + final_ids = pd.Index(tables[entity][id_column], name=id_column) + if not final_ids.equals(expected_ids): + raise PopulationError( + f"EXPAND node {node.id!r} final {entity!r} ids disagree with " + "its lineage targets after cell overlays." + ) + weight_entity = _expand_weight_entity(node) assert weight_entity is not None if weight_entity not in before.weighted_entities: @@ -832,16 +1574,39 @@ def _patch_expand( weights[entity] = result.weights continue old = before.weights_for(entity) - weights[entity] = Weights(old.values[lineage_positions[entity]], kind=old.kind) + positions = lineage_positions[entity] + if (positions < 0).any(): + raise PopulationError( + f"EXPAND node {node.id!r} cannot admit entrants on weighted " + f"entity {entity!r}; only {weight_entity!r} has materialized weights." + ) + weights[entity] = Weights(old.values[positions], kind=old.kind) - person = before.schema.person_entity person_positions = lineage_positions[person] - strata = pd.Series( - before.strata.iloc[person_positions].array.copy(), - index=tables[person].index, - name=before.strata.name, - dtype=before.strata.dtype, - ) + if entrant_strata is None: + strata = pd.Series( + before.strata.iloc[person_positions].array.copy(), + index=tables[person].index, + name=before.strata.name, + dtype=before.strata.dtype, + ) + else: + additions: list[object] = [] + entrant_values = iter(entrant_strata.array) + for source_position in person_positions[len(before.table(person)) :]: + additions.append( + next(entrant_values) + if source_position < 0 + else before.strata.iloc[source_position] + ) + strata = pd.concat( + [ + before.strata.astype(object).reset_index(drop=True), + pd.Series(additions, dtype=object), + ], + ignore_index=True, + ) + strata.name = before.strata.name frame = Frame( tables, before.schema, @@ -850,6 +1615,13 @@ def _patch_expand( mass_log=before.mass_log, metadata=before.metadata, ) + _assert_copied_expand_storage( + before, + frame, + node, + lineage, + rewrites=rewrite_coordinates, + ) owners = { (entity, str(column)): node.id for entity in frame.entities @@ -858,6 +1630,65 @@ def _patch_expand( return frame, owners +def expand_writes_receipt( + before: Frame, + after: Frame, + node: Node, + receipt: Mapping[str, object], + *, + rewrite_coordinates: frozenset[tuple[str, str]] = frozenset(), +) -> dict[str, list[str]]: + """Record each EXPAND overlay coordinate and the row classes it wrote.""" + + if node.structural is not StructuralDelta.EXPAND: + raise PopulationError("expand_writes_receipt requires an EXPAND node.") + receipt_lineage = _expand_lineage_from_receipt(before, node, receipt) + lineage = _validate_expand_lineage(before, node, receipt_lineage, after=after) + cells = _expand_cells(node) + membership_coordinates = { + ( + before.schema.person_entity, + before.schema.membership_column(group), + ) + for group in before.schema.group_entities + } + materialized_memberships = membership_coordinates & { + (entity, column) for entity, column, _ in cells + } + copied_membership_rewrites = ( + _assert_expand_memberships( + before, + {entity: after.table(entity) for entity in after.entities}, + lineage, + node, + ) + if materialized_memberships + else frozenset() + ) + + writes: dict[str, list[str]] = {} + for entity, column, _ in sorted(cells): + coordinate = (entity, column) + source_is_null = lineage[entity].isna().to_numpy(dtype=np.bool_, copy=False) + classes: list[str] = [] + if source_is_null.any(): + classes.append("entrant") + if ( + (~source_is_null).any() + and column in before.table(entity) + and ( + coordinate in rewrite_coordinates + or coordinate in copied_membership_rewrites + ) + ): + classes.append("copied-rewrite") + if column not in before.table(entity) and after.n(entity): + classes.append("new-column") + if classes: + writes[f"{entity}.{column}"] = classes + return writes + + def _assert_expand_weights( population: Population, frame: Frame, node: Node, result: KernelResult ) -> None: @@ -874,6 +1705,22 @@ def _assert_expand_weights( raise PopulationError( f"EXPAND node {node.id!r} declares unknown weight kind {raw_kind!r}." ) from error + try: + current = population.frame.weights_for(entity) + except ValueError: + current = None + # Weight kinds only move forward (amendment 6); an EXPAND may carry its + # base's kind or advance it, never regress it, so a reweighted base can + # never be re-declared design to smuggle entrants past the design anchor. + forward = (WeightKind.DESIGN, WeightKind.IMPORTANCE, WeightKind.CALIBRATED) + if current is not None and forward.index(declared_kind) < forward.index( + current.kind + ): + raise PopulationError( + f"EXPAND node {node.id!r} cannot regress {entity!r} weights from " + f"{current.kind.value!r} to {raw_kind!r}; weight kinds only move " + "forward, and entrants need a design base." + ) assert result.weights is not None if result.weights.kind is not declared_kind: raise PopulationError( @@ -1210,13 +2057,11 @@ def _carry_design_weights( introduced = before_positions < 0 if introduced.any(): sources = lineage.reindex(after_ids[introduced]) - if sources.isna().any(): - raise PopulationError( - f"EXPAND node {node.id!r} has incomplete design lineage " - f"for {entity!r}." - ) - before_positions[introduced] = before_ids.get_indexer( - sources.to_numpy(copy=False) + source_is_null = sources.isna().to_numpy(dtype=np.bool_, copy=False) + introduced_positions = np.flatnonzero(introduced) + copied_positions = introduced_positions[~source_is_null] + before_positions[copied_positions] = before_ids.get_indexer( + sources.iloc[np.flatnonzero(~source_is_null)].to_numpy(copy=False) ) values = np.empty(len(after_ids), dtype=np.float64) retained = before_positions >= 0 @@ -1455,6 +2300,8 @@ def _mass_record( node: Node, result: KernelResult, policy: str, + *, + mass_partition: tuple[str, str] | None = None, ) -> MassRecord: if policy not in MASS_POLICIES: raise PopulationError(f"Node {node.id!r} has unknown mass policy {policy!r}.") @@ -1464,11 +2311,23 @@ def _mass_record( after_pairs = tuple((key, float(value)) for key, value in after_mass.items()) before_total = float(before_mass.sum()) after_total = float(after_mass.sum()) + before_partition: tuple[tuple[object, tuple[tuple[object, float], ...]], ...] = () + after_partition: tuple[tuple[object, tuple[tuple[object, float], ...]], ...] = () + if mass_partition is not None: + before_partition = _mass_by_partition(before, mass_partition, node.id) + after_partition = _mass_by_partition(after, mass_partition, node.id) if policy == "conserve": + if mass_partition is not None: + _assert_partition_mass_mapping( + before_partition, + after_partition, + label=f"Node {node.id!r} mass='conserve'", + ) _assert_mass_mapping( dict(before_pairs), dict(after_pairs), label=f"Node {node.id!r} mass='conserve'", + missing_as_zero=True, ) receipt_mass = result.receipt.get("mass") @@ -1481,6 +2340,9 @@ def _mass_record( before=dict(before_pairs), after=dict(after_pairs), node_id=node.id, + mass_partition=mass_partition, + before_partition=before_partition, + after_partition=after_partition, ) elif policy == "declared": raise PopulationError( @@ -1507,6 +2369,89 @@ def _mass_record( and "expand_weight_entity" in node.params else None ), + partition_entity=(None if mass_partition is None else mass_partition[0]), + partition_column=(None if mass_partition is None else mass_partition[1]), + before_by_partition_stratum=before_partition, + after_by_partition_stratum=after_partition, + ) + + +def _partition_values_on_person( + frame: Frame, + mass_partition: tuple[str, str], + node_id: str, +) -> pd.Series: + entity, column = mass_partition + if entity not in frame.entities: + raise PopulationError( + f"Node {node_id!r} mass partition names unknown entity {entity!r}." + ) + table = frame.table(entity) + if column not in table: + raise PopulationError( + f"Node {node_id!r} mass partition column {entity}.{column} is " + "missing at run time." + ) + person = frame.schema.person_entity + if entity == person: + return table[column].reset_index(drop=True) + if entity not in frame.schema.group_entities: + raise PopulationError( + f"Node {node_id!r} cannot broadcast mass partition entity {entity!r} " + "to persons." + ) + id_column = frame.schema.entity_id_column(entity) + membership = frame.schema.membership_column(entity) + positions = pd.Index(table[id_column]).get_indexer( + frame.table(person)[membership].to_numpy(copy=False) + ) + if (positions < 0).any(): # defended by Frame linkage validation + raise PopulationError( + f"Node {node_id!r} cannot align mass partition {entity}.{column} " + "to person memberships." + ) + return table[column].iloc[positions].reset_index(drop=True) + + +def _mass_by_partition( + frame: Frame, + mass_partition: tuple[str, str], + node_id: str, +) -> tuple[tuple[object, tuple[tuple[object, float], ...]], ...]: + partition = _partition_values_on_person(frame, mass_partition, node_id) + person = frame.schema.person_entity + weights = frame.resolve_weights(person).values + strata = frame.strata.reset_index(drop=True) + valid = partition.notna().to_numpy(dtype=np.bool_, copy=False) + if not valid.any(): + return () + grouped = ( + pd.DataFrame( + { + "_partition": partition.loc[valid].reset_index(drop=True), + "_stratum": strata.loc[valid].reset_index(drop=True), + "_mass": weights[valid], + } + ) + .groupby(["_partition", "_stratum"], observed=True, sort=False)["_mass"] + .sum() + ) + nested: dict[object, dict[object, float]] = {} + for (partition_value, stratum), mass in grouped.items(): + nested.setdefault(partition_value, {})[stratum] = float(mass) + return tuple( + ( + partition_value, + tuple( + sorted( + strata_mass.items(), + key=lambda item: _receipt_key(item[0]), + ) + ), + ) + for partition_value, strata_mass in sorted( + nested.items(), key=lambda item: _receipt_key(item[0]) + ) ) @@ -1519,6 +2464,9 @@ def _validate_mass_receipt( before: Mapping[object, float], after: Mapping[object, float], node_id: str, + mass_partition: tuple[str, str] | None, + before_partition: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], + after_partition: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], ) -> None: if not isinstance(raw, Mapping): raise PopulationError(f"Node {node_id!r} receipt['mass'] must be a mapping.") @@ -1535,6 +2483,93 @@ def _validate_mass_receipt( _assert_receipt_mapping( raw.get("stratum_after"), after, f"Node {node_id!r} mass.stratum_after" ) + if mass_partition is None and "partition" in raw: + raise PopulationError( + f"Node {node_id!r} mass.partition is present but the graph " + "declares no mass partition." + ) + if mass_partition is not None and "partition" in raw: + _validate_partition_mass_receipt( + raw["partition"], + mass_partition=mass_partition, + before=before_partition, + after=after_partition, + node_id=node_id, + ) + + +def _validate_partition_mass_receipt( + raw: object, + *, + mass_partition: tuple[str, str], + before: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], + after: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], + node_id: str, +) -> None: + if not isinstance(raw, Mapping) or set(raw) != { + "entity", + "column", + "stratum_before", + "stratum_after", + }: + raise PopulationError( + f"Node {node_id!r} mass.partition must contain entity, column, " + "stratum_before, and stratum_after." + ) + entity, column = mass_partition + if raw.get("entity") != entity or raw.get("column") != column: + raise PopulationError( + f"Node {node_id!r} mass.partition names " + f"{raw.get('entity')}.{raw.get('column')}; expected {entity}.{column}." + ) + _assert_partition_receipt_mapping( + raw.get("stratum_before"), + before, + label=f"Node {node_id!r} mass.partition.stratum_before", + ) + _assert_partition_receipt_mapping( + raw.get("stratum_after"), + after, + label=f"Node {node_id!r} mass.partition.stratum_after", + ) + + +def _assert_partition_receipt_mapping( + observed: object, + expected: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], + *, + label: str, +) -> None: + if not isinstance(observed, Mapping): + raise PopulationError(f"{label} must be a mapping.") + converted = _partition_receipt_mapping(expected) + if set(observed) != set(converted): + raise PopulationError( + f"{label} changed partitions: expected {list(converted)}, " + f"got {list(observed)}." + ) + for partition, strata in converted.items(): + _assert_receipt_mapping( + observed[partition], strata, f"{label} partition {partition!r}" + ) + + +def _assert_partition_mass_mapping( + expected: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], + observed: tuple[tuple[object, tuple[tuple[object, float], ...]], ...], + *, + label: str, +) -> None: + before = {partition: dict(strata) for partition, strata in expected} + after = {partition: dict(strata) for partition, strata in observed} + partitions = sorted(set(before) | set(after), key=_receipt_key) + for partition in partitions: + _assert_mass_mapping( + before.get(partition, {}), + after.get(partition, {}), + label=f"{label} partition {_receipt_key(partition)!r}", + missing_as_zero=True, + ) def _assert_close(observed: object, expected: float, label: str) -> None: @@ -1551,24 +2586,50 @@ def _assert_receipt_mapping( ) -> None: if not isinstance(observed, Mapping): raise PopulationError(f"{label} must be a mapping.") - converted = {key: float(value) for key, value in observed.items()} - _assert_mass_mapping(expected, converted, label=label) + expected_json = _receipt_mass_mapping(expected, label=label) + observed_json = _receipt_mass_mapping(observed, label=label) + _assert_mass_mapping(expected_json, observed_json, label=label) + + +def _receipt_mass_mapping( + values: Mapping[object, object], *, label: str +) -> dict[str, float]: + """Normalize a mass mapping to its stable JSON-object-key representation.""" + + result: dict[str, float] = {} + for raw_key, value in values.items(): + key = _receipt_key(raw_key) + if key in result: + raise PopulationError(f"{label} has colliding JSON key {key!r}.") + result[key] = float(value) + return result def _assert_mass_mapping( - expected: Mapping[object, float], observed: Mapping[object, float], *, label: str + expected: Mapping[object, float], + observed: Mapping[object, float], + *, + label: str, + missing_as_zero: bool = False, ) -> None: - if set(expected) != set(observed): + expected_keys = set(expected) + observed_keys = set(observed) + if not missing_as_zero and expected_keys != observed_keys: raise PopulationError( f"{label} changed strata: expected {list(expected)}, got {list(observed)}." ) - for stratum in expected: - if not np.isclose( - expected[stratum], observed[stratum], rtol=_MASS_RTOL, atol=0.0 - ): + strata = ( + sorted(expected_keys | observed_keys, key=_receipt_key) + if missing_as_zero + else expected + ) + for stratum in strata: + expected_mass = expected.get(stratum, 0.0) + observed_mass = observed.get(stratum, 0.0) + if not np.isclose(expected_mass, observed_mass, rtol=_MASS_RTOL, atol=0.0): raise PopulationError( - f"{label} changed stratum {stratum!r}: {expected[stratum]!r} -> " - f"{observed[stratum]!r}." + f"{label} changed stratum {stratum!r}: {expected_mass!r} -> " + f"{observed_mass!r}." ) diff --git a/packages/microcosm-graph/src/microcosm/graph/serialize.py b/packages/microcosm-graph/src/microcosm/graph/serialize.py index 32b78f56d..33b5437f3 100644 --- a/packages/microcosm-graph/src/microcosm/graph/serialize.py +++ b/packages/microcosm-graph/src/microcosm/graph/serialize.py @@ -37,6 +37,13 @@ def graph_to_json(graph: Graph) -> str: for source in graph.sources ], "nodes": [_node_payload(node) for node in graph.nodes], + # Amendment 12: present only when declared, so a declaration written + # before the amendment serializes byte for byte as it did. + **( + {} + if graph.mass_partition is None + else {"mass_partition": list(graph.mass_partition)} + ), } return canonical_json(payload).decode("utf-8") @@ -51,7 +58,10 @@ def graph_from_json(text: str) -> Graph: except json.JSONDecodeError as error: raise ValueError("graph JSON is not valid JSON") from error root = _mapping(raw, "graph") - _exact_fields(root, {"country", "sources", "nodes"}, "graph") + fields = {"country", "sources", "nodes"} + if "mass_partition" in root: + fields.add("mass_partition") + _exact_fields(root, fields, "graph") sources_raw = _array(root["sources"], "graph.sources") nodes_raw = _array(root["nodes"], "graph.nodes") return Graph( @@ -63,9 +73,21 @@ def graph_from_json(text: str) -> Graph: nodes=tuple( _node_from_payload(value, index) for index, value in enumerate(nodes_raw) ), + mass_partition=_partition_from_payload( + root.get("mass_partition"), "graph.mass_partition" + ), ) +def _partition_from_payload(value: object, label: str) -> tuple[str, str] | None: + if value is None: + return None + parts = _array(value, label) + if len(parts) != 2: + raise TypeError(f"{label} must be an [entity, column] pair") + return (_string(parts[0], f"{label}[0]"), _string(parts[1], f"{label}[1]")) + + def _node_payload(node: Node) -> dict[str, object]: return { "id": node.id, @@ -104,6 +126,7 @@ def _node_payload(node: Node) -> dict[str, object]: } ), "mass": node.mass, + **({"entrants": True} if node.entrants else {}), "description": node.description, "citation": node.citation, } @@ -138,7 +161,12 @@ def _node_from_payload(value: object, index: int) -> Node: "description", "citation", } + if "entrants" in payload: + fields.add("entrants") _exact_fields(payload, fields, label) + entrants = payload.get("entrants", False) + if not isinstance(entrants, bool): + raise TypeError(f"{label}.entrants must be a boolean") inputs = _array(payload["inputs"], f"{label}.inputs") outputs = _array(payload["outputs"], f"{label}.outputs") sources = _array(payload["sources"], f"{label}.sources") @@ -173,6 +201,7 @@ def _node_from_payload(value: object, index: int) -> Node: ), weights=_weights_from_payload(payload["weights"], f"{label}.weights"), mass=_string(payload["mass"], f"{label}.mass"), + entrants=entrants, description=_string(payload["description"], f"{label}.description"), citation=_string(payload["citation"], f"{label}.citation"), ) diff --git a/packages/microcosm-graph/src/microcosm/graph/view.py b/packages/microcosm-graph/src/microcosm/graph/view.py index 98c3f0a7b..63820048a 100644 --- a/packages/microcosm-graph/src/microcosm/graph/view.py +++ b/packages/microcosm-graph/src/microcosm/graph/view.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Mapping from enum import Enum from .canonical import canonical_json @@ -91,6 +92,30 @@ def describe( 'Seed: int.from_bytes(sha256(b"seed\\0" + node_key)[:8], "little")' ) else: + capabilities = run_receipt.capabilities + if isinstance(capabilities, Mapping): + tolerance_text = "" + determinism = capabilities["determinism"] + numeric = capabilities["numeric"] + seed_source = capabilities["seed_source"] + structural = capabilities["structural"] + consumes_se = capabilities["consumes_se"] + else: + tolerance = capabilities.tolerance + tolerance_text = canonical_json( + None + if tolerance is None + else { + "rtol": tolerance.rtol, + "atol": tolerance.atol, + "ulps": tolerance.ulps, + } + ).decode("utf-8") + determinism = capabilities.determinism + numeric = capabilities.numeric + seed_source = capabilities.seed_source + structural = capabilities.structural + consumes_se = capabilities.consumes_se lines.extend( [ 'Seed: int.from_bytes(sha256(b"seed\\0" + node_key)[:8], ' @@ -98,11 +123,12 @@ def describe( f"Store: {'hit' if run_receipt.hit else 'miss'}; " f"wall_time={run_receipt.wall_time:.6g}s", "Capabilities: " - f"determinism={_value(run_receipt.capabilities.determinism)}, " - f"numeric={_value(run_receipt.capabilities.numeric)}, " - f"seed={_value(run_receipt.capabilities.seed_source)}, " - f"structural={_value(run_receipt.capabilities.structural)}, " - f"consumes_se={run_receipt.capabilities.consumes_se}", + f"determinism={_value(determinism)}, " + f"numeric={_value(numeric)}, " + f"seed={_value(seed_source)}, " + f"structural={_value(structural)}, " + f"consumes_se={consumes_se}, " + f"tolerance={tolerance_text}", "Receipt: " + canonical_json(run_receipt.receipt).decode("utf-8"), ] ) diff --git a/packages/microcosm-graph/tests/_toy.py b/packages/microcosm-graph/tests/_toy.py index f37ab7767..ada684038 100644 --- a/packages/microcosm-graph/tests/_toy.py +++ b/packages/microcosm-graph/tests/_toy.py @@ -69,6 +69,7 @@ Slice, SourceRef, StructuralDelta, + Tolerance, WeightTransition, compile_graph, run_graph, @@ -83,6 +84,7 @@ "PUBLISH_DECISION", "SOURCE", "STRATA", + "TOY_TOLERANCE", "ToyKernel", "ToyRun", "absent_node", @@ -96,6 +98,8 @@ "descendants", "draw", "drop_nodes", + "entrant_expand_node", + "entrant_person_node", "full_graph", "gate_node", "graph_source_files", @@ -130,6 +134,9 @@ #: The one source every toy graph reads, through the ``csv-tables`` codec. SOURCE = SourceRef("survey", "csv-tables", description="the toy country's tables") +#: Cross-machine numeric movement declared by the C5 toy producer. +TOY_TOLERANCE = Tolerance(rtol=1e-6) + def id_column(entity: str) -> str: """The frame convention: ``person_id`` for persons, ``{entity}_id`` elsewhere.""" @@ -432,6 +439,192 @@ def compute(self, context: KernelContext) -> KernelResult: ) +class ExpandEntrants(ToyKernel): + """EXPAND: add one copied person and one materialized entrant household.""" + + def compute(self, context: KernelContext) -> KernelResult: + person = context.tables["person"] + person_ids = pd.Index(person["person_id"], name="person_id") + person_source_id = int(person_ids[0]) + person_copy_id = int(person_ids.max()) + 1 + + household = context.tables["household"] + household_ids = pd.Index(household["household_id"], name="household_id") + household_entrant_id = int(household_ids.max()) + 1 + household_target_ids = household_ids.append( + pd.Index([household_entrant_id], dtype="int64", name="household_id") + ) + person_target_ids = person_ids.append( + pd.Index([person_copy_id], dtype="int64", name="person_id") + ) + + household_size = pd.concat( + [ + household["household_size"].reset_index(drop=True), + pd.Series([1], dtype="int64"), + ], + ignore_index=True, + ) + materialized_size = pd.Series( + household_size.array, index=household_target_ids, dtype="int64" + ) + if context.params.get("missing_entrant_column") == "household_size": + materialized_size = materialized_size.drop(index=household_entrant_id) + memberships = pd.concat( + [ + person["person_household_id"].reset_index(drop=True), + pd.Series([household_entrant_id], dtype="int64"), + ], + ignore_index=True, + ) + materialized_memberships = pd.Series( + memberships.array, index=person_target_ids, dtype="int64" + ) + + empty_releases = pd.Series( + [], + index=pd.Index([], dtype="int64", name="release_id"), + dtype="int64", + ) + household_weights = context.weights["household"] + expanded_weights = np.append( + household_weights.values, float(context.params["entrant_weight"]) + ) + return KernelResult( + expand={ + "person": pd.Series( + [person_source_id], + index=pd.Index([person_copy_id], dtype="int64", name="person_id"), + dtype="int64", + ), + "household": pd.Series( + pd.array([pd.NA], dtype="Int64"), + index=pd.Index( + [household_entrant_id], + dtype="int64", + name="household_id", + ), + ), + "release": empty_releases, + }, + columns={ + ("household", "household_size"): materialized_size, + ("person", "person_household_id"): materialized_memberships, + }, + weights=Weights(expanded_weights, kind=household_weights.kind), + ) + + +class ExpandEntrantPerson(ToyKernel): + """EXPAND: admit one entrant person into an incumbent household. + + The entrant copies nothing: every person column is materialized from a + template row, its memberships name incumbent groups, and its stratum + arrives through ``KernelResult.strata`` (amendment 14). ``strata_mode`` + exercises the refusals: ``missing`` omits the field, ``unknown_id`` + labels an id the node never adds, ``labels_incumbent`` labels an + incumbent person as well, and ``labels_copied`` also copies the + template person to a second new id and labels that copy, which takes + its stratum from lineage and may not be labelled. + ``mutates_copied_income`` instead labels only the entrant but changes the + copied person's carried income to exercise the copy-invariance guard. + """ + + def compute(self, context: KernelContext) -> KernelResult: + person = context.tables["person"] + person_ids = pd.Index(person["person_id"], name="person_id") + template = person.iloc[0] + entrant_id = int(person_ids.max()) + 1 + mode = str(context.params.get("strata_mode", "ok")) + copy_id = entrant_id + 1 + copies_person = mode in {"labels_copied", "mutates_copied_income"} + added_ids = [entrant_id, copy_id] if copies_person else [entrant_id] + target_ids = person_ids.append( + pd.Index(added_ids, dtype="int64", name="person_id") + ) + + def overlay(column: str, dtype: str, value: object) -> pd.Series: + copied_value = template[column] + if mode == "mutates_copied_income" and column == "income": + copied_value = float(copied_value) + 1.0 + added = [value] + ([copied_value] if copies_person else []) + values = pd.concat( + [person[column].reset_index(drop=True), pd.Series(added)], + ignore_index=True, + ) + return pd.Series(pd.array(values, dtype=dtype), index=target_ids) + + columns = { + ("person", "age"): overlay("age", "int64", 30), + ("person", "income"): overlay("income", "float64", 12_500.0), + ("person", "is_adult"): overlay("is_adult", "boolean", True), + ("person", "receives_x"): overlay("receives_x", "boolean", False), + ("person", "person_household_id"): overlay( + "person_household_id", "int64", int(template["person_household_id"]) + ), + ("person", "person_release_id"): overlay( + "person_release_id", "int64", int(template["person_release_id"]) + ), + } + labelled = { + "ok": [entrant_id], + "unknown_id": [entrant_id + 1], + "labels_incumbent": [int(person_ids[0]), entrant_id], + "labels_copied": [entrant_id, copy_id], + "mutates_copied_income": [entrant_id], + } + strata = ( + None + if mode == "missing" + else pd.Series( + ["urban"] * len(labelled[mode]), + index=pd.Index(labelled[mode], dtype="int64", name="person_id"), + dtype=object, + name="stratum", + ) + ) + empty = { + entity: pd.Series( + [], + index=pd.Index([], dtype="int64", name=id_column(entity)), + dtype="int64", + ) + for entity in ("household", "release") + } + household_weights = context.weights["household"] + lineage = [pd.NA] + ([int(person_ids[0])] if copies_person else []) + return KernelResult( + expand={ + "person": pd.Series( + pd.array(lineage, dtype="Int64"), + index=pd.Index(added_ids, dtype="int64", name="person_id"), + ), + **empty, + }, + columns=columns, + weights=Weights( + household_weights.values.copy(), kind=household_weights.kind + ), + strata=strata, + ) + + +class ClaimMaterializedExpand(ToyKernel): + """Claim kernel-supplied EXPAND columns through the ownership surface.""" + + def compute(self, context: KernelContext) -> KernelResult: + columns: dict[tuple[str, str], pd.Series] = {} + for item in context.params["claim_cells"]: + entity, column, dtype = (str(value) for value in item) + table = context.tables[entity] + columns[(entity, column)] = pd.Series( + table[column].array.copy(), + index=pd.Index(table[id_column(entity)], name=id_column(entity)), + dtype=dtype, + ) + return KernelResult(columns=columns) + + class CalibrateToy(ToyKernel): """An ``importance -> calibrated`` transition hitting one target exactly.""" @@ -483,6 +676,55 @@ def compute(self, context: KernelContext) -> KernelResult: ) +class GateReportsTolerance(ToyKernel): + """A gate that reports the input owner's declared numeric tolerance.""" + + def compute(self, context: KernelContext) -> KernelResult: + entity = str(context.params["entity"]) + column = str(context.params["column"]) + observed = float(context.tables[entity][column].astype("float64").mean()) + declared = context.tolerances[(entity, column)] + scope = context.numerics[(entity, column)] + tolerance = ( + None + if declared is None + else { + "rtol": declared.rtol, + "atol": declared.atol, + "ulps": declared.ulps, + } + ) + comparison_platform = context.params.get("comparison_platform") + cross_platform = ( + scope.platform is not None + and comparison_platform is not None + and comparison_platform != scope.platform + ) + outcome = "evidence_absent" if cross_platform else "pass" + evidence = { + "observed": observed, + "tolerance": tolerance, + "numeric": scope.numeric.value, + "platform": scope.platform, + } + if comparison_platform is not None: + evidence["comparison_platform"] = comparison_platform + if cross_platform: + evidence["reason"] = "numeric contract is scoped to a different platform" + verdict_column = str(context.params["verdict_column"]) + return KernelResult( + columns={ + ("release", verdict_column): pd.Series( + [outcome], index=_owned_ids(context, "release"), dtype="string" + ) + }, + receipt={ + "outcome": outcome, + "evidence": evidence, + }, + ) + + class ReleaseTier(ToyKernel): """Derive a release tier from the gate verdicts declared as its inputs.""" @@ -626,6 +868,9 @@ def compute(self, context: KernelContext) -> KernelResult: _FILTER = Capabilities( determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER ) +_EXPAND = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND +) _REWEIGHT = Capabilities( determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT ) @@ -644,12 +889,23 @@ def toy_registry(*, variants: Mapping[str, str] | None = None) -> KernelRegistry kernels = ( SourceCsv("source.csv@1", _CREATE), DeriveAdd("derive.add@1", _DETERMINISTIC), + DeriveAdd( + "derive.tolerant@1", + Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + tolerance=TOY_TOLERANCE, + ), + ), DrawUniform("draw.uniform@1", _SEEDED), ImputeChain("impute.chain@1", _SEEDED), SimulateStub("simulate.stub@1", _DETERMINISTIC), PatchColumn("patch.column@1", _DETERMINISTIC), AbsentColumn("absent.column@1", _DETERMINISTIC), SelectRows("select.rows@1", _FILTER), + ExpandEntrants("expand.entrants@1", _EXPAND), + ExpandEntrantPerson("expand.entrant_person@1", _EXPAND), + ClaimMaterializedExpand("claim.expand@1", _DETERMINISTIC), ReweightScale("reweight.scale@1", _REWEIGHT), CalibrateToy( "calibrate.toy@1", @@ -664,6 +920,10 @@ def toy_registry(*, variants: Mapping[str, str] | None = None) -> KernelRegistry "gate.threshold@1", Capabilities(determinism=Determinism.DETERMINISTIC, role=KernelRole.GATE), ), + GateReportsTolerance( + "gate.tolerance@1", + Capabilities(determinism=Determinism.DETERMINISTIC, role=KernelRole.GATE), + ), ReleaseTier( "release.tier@1", Capabilities( @@ -974,6 +1234,106 @@ def select_node( ) +def entrant_expand_node( + node_id: str = "scheduled_entries", + *, + entrants: bool = True, + missing_entrant_column: str | None = None, +) -> tuple[Node, Node]: + """An EXPAND plus the ownership claim for its materialized person fields.""" + overlays = ( + ("household", "household_size", "int64"), + ("person", "person_household_id", "int64"), + ) + claim_cells = (overlays[0],) + expand = Node( + node_id, + "expand.entrants@1", + structural=StructuralDelta.EXPAND, + base="survey", + inputs=( + Slice("person", ("age",)), + Slice("household", ("household_size",)), + ), + params={ + "expand_cells": overlays, + "expand_weight_entity": "household", + "expand_weight_kind": "design", + "entrant_weight": 125.0, + "missing_entrant_column": missing_entrant_column, + }, + mass="free", + entrants=entrants, + ) + claim = Node( + f"claim_{node_id}", + "claim.expand@1", + outputs=tuple( + Owned(entity, column, dtype) for entity, column, dtype in claim_cells + ), + params={ + "claim_cells": claim_cells, + "materialized_expand_outputs": tuple( + f"{entity}.{column}" for entity, column, _ in claim_cells + ), + }, + population=node_id, + ) + return expand, claim + + +def entrant_person_node( + node_id: str = "immigrant_cohort", + *, + strata_mode: str = "ok", +) -> tuple[Node, Node]: + """An EXPAND admitting one entrant person, plus the claim of its cells.""" + data_cells = ( + ("person", "age", "int64"), + ("person", "income", "float64"), + ("person", "is_adult", "boolean"), + ("person", "receives_x", "boolean"), + ) + overlays = ( + *data_cells, + ("person", "person_household_id", "int64"), + ("person", "person_release_id", "int64"), + ) + expand = Node( + node_id, + "expand.entrant_person@1", + structural=StructuralDelta.EXPAND, + base="survey", + inputs=( + Slice("person", ("age", "income", "is_adult", "receives_x")), + Slice("household", ("household_size",)), + ), + params={ + "expand_cells": overlays, + "expand_weight_entity": "household", + "expand_weight_kind": "design", + "strata_mode": strata_mode, + }, + mass="free", + entrants=True, + ) + claim = Node( + f"claim_{node_id}", + "claim.expand@1", + outputs=tuple( + Owned(entity, column, dtype) for entity, column, dtype in data_cells + ), + params={ + "claim_cells": data_cells, + "materialized_expand_outputs": tuple( + f"{entity}.{column}" for entity, column, _ in data_cells + ), + }, + population=node_id, + ) + return expand, claim + + POOL = Node( "pool", "reweight.scale@1", diff --git a/packages/microcosm-graph/tests/fixtures/manifests/v1_tolerance_bound_without_tolerance.json b/packages/microcosm-graph/tests/fixtures/manifests/v1_tolerance_bound_without_tolerance.json new file mode 100644 index 000000000..4d14e3036 --- /dev/null +++ b/packages/microcosm-graph/tests/fixtures/manifests/v1_tolerance_bound_without_tolerance.json @@ -0,0 +1 @@ +{"content_addressed":{"decisions":[],"node_keys":["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]},"country":"toy","decisions":[],"finished_at":"2026-08-31T00:00:01Z","host":"legacy-runner","key":"822d8ff7a109d4a3fce7f30034ee6fdad1ca8180573f08f1beb4a0dfeaf82367","known_failures":[],"nodes":{"fit_qrf":{"artifacts":[],"capabilities":{"consumes_se":false,"dependencies":["numpy","scikit-learn"],"determinism":"seeded","numeric":"tolerance_bound","role":"compute","seed_source":"param","structural":"none"},"frame_key":null,"hit":true,"kernel_impl_hash":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","kernel_ref":"fit.qrf@1","key":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","opaque_artifacts":{},"receipt":{"rows":3},"seed":42,"wall_time":0.25,"weight_key":null}},"schema_version":1,"started_at":"2026-08-31T00:00:00Z","tier":null} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index cb8a0cc44..8485a3916 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"a7a0330fb7b7a4c80b62e2a44dcc9075aa6da9b27a20a4282ca6969f4570ef18","kernel":"calibrate.adam@1","node":"calibrate","seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"a7a0330fb7b7a4c80b62e2a44dcc9075aa6da9b27a20a4282ca6969f4570ef18","kernel":"calibrate.adam@1","node":"calibrate","node_key":"0168f69547e21ef71c67442a52622de83efd32fd23adc8faf93cc347f6c80b4c","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"0168f69547e21ef71c67442a52622de83efd32fd23adc8faf93cc347f6c80b4c"}},"seed":0} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json index ebfa035a4..1bfbebe22 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","quantile-forest":"1.4.2","scikit-learn":"1.8.0"},"implementation_hash":"8434e3e7e6333a4bb093f5d54a93e38d393553d3b27d811bec7671828688d707","kernel":"fit.qrf@1","node":"fit_qrf","seed":947} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","quantile-forest":"1.4.2","scikit-learn":"1.8.0"},"implementation_hash":"02db8f5c849d876be20a95152b5302a5cacc0a7c77c58d8b436a3a00f57b4c92","kernel":"fit.qrf@1","node":"fit_qrf","node_key":"8878352db3439439f412f26c8762ff5871b94fe8e5fc8d3469dd1c45d7ef7da4","numeric":"platform_bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"8878352db3439439f412f26c8762ff5871b94fe8e5fc8d3469dd1c45d7ef7da4"},"x86_64/linux/py3.13":{"direct":"platforms/x86_64-linux-py3_13/direct.csv","node_key":"f6984280e1ef0f156bd24345f7d3677d573c650ac706f9a3f949627c0d76d2d4"},"x86_64/linux/py3.14":{"direct":"platforms/x86_64-linux-py3_14/direct.csv","node_key":"9e80ee3ac5c30f99725c4dd932535983dabde7913c653e79819b1df3a289720c"}},"seed":947} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/platforms/x86_64-linux-py3_13/direct.csv b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/platforms/x86_64-linux-py3_13/direct.csv new file mode 100644 index 000000000..e4e439508 --- /dev/null +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/platforms/x86_64-linux-py3_13/direct.csv @@ -0,0 +1,13 @@ +person.y +1.5 +1.25 +0.25 +1.75 +2.9171789426187056 +0.75 +2.75 +2.75 +0.75 +3.25 +3.25 +1.255723650771198 diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/platforms/x86_64-linux-py3_14/direct.csv b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/platforms/x86_64-linux-py3_14/direct.csv new file mode 100644 index 000000000..e4e439508 --- /dev/null +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/platforms/x86_64-linux-py3_14/direct.csv @@ -0,0 +1,13 @@ +person.y +1.5 +1.25 +0.25 +1.75 +2.9171789426187056 +0.75 +2.75 +2.75 +0.75 +3.25 +3.25 +1.255723650771198 diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json index a55f81a28..15f5d879b 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json @@ -1 +1 @@ -{"dependencies":{},"implementation_hash":"1d9215fde2dc8bda6225a7d3b7fdba9f1cff74c9478972321c903055aaaaf6d1","kernel":"simulate.rules@1","node":"simulate","seed":null} +{"dependencies":{},"implementation_hash":"eed54eaf27b53faf446068aa833ec16ab398840faf870445dbdcd6a4604c566c","kernel":"simulate.rules@1","node":"simulate","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123"}},"seed":null} diff --git a/packages/microcosm-graph/tests/test_acceptance_b_ownership.py b/packages/microcosm-graph/tests/test_acceptance_b_ownership.py index 72836472c..2ce41a7f9 100644 --- a/packages/microcosm-graph/tests/test_acceptance_b_ownership.py +++ b/packages/microcosm-graph/tests/test_acceptance_b_ownership.py @@ -22,7 +22,16 @@ import pandas as pd import pytest -from microcosm.graph import KernelContext, Node, Owned, Ownership, Slice +from microcosm.graph import ( + GraphError, + KernelContext, + Node, + NodeRejectedError, + Owned, + Ownership, + Slice, + StructuralDelta, +) if "_toy" not in sys.modules: _SPEC = importlib.util.spec_from_file_location( @@ -107,6 +116,8 @@ def test_b2_executor_enforces_ownership(tmp_path: Path) -> None: "params", "rng", "sources", + "tolerances", # amendment 13: declared tolerances of the inputs' owners + "numerics", # amendment 17: per-coordinate numeric class, bound, platform } graph = toy.small_graph( @@ -232,3 +243,115 @@ def test_b5_null_means_absence(tmp_path: Path) -> None: with pytest.raises(NodeRejectedError, match="no_data"): toy.run_toy(liar, tmp_path / "liar") + + +def test_b6_entrants_are_declared(tmp_path: Path) -> None: + """Null lineage is an explicit, complete, and receipted entrant contract. + + The backwards-compatible lineage shape keeps ``(new, source)`` pairs under + ``expand`` and uses a null source for entrants. The test admits a household + entrant because the frozen result interface has no separate output for a + new person's stratum. Its complete carried data surface is materialized by + the EXPAND kernel and passed through ``materialized_expand_outputs``. + """ + with pytest.raises(GraphError, match="conserved_entrants"): + Node( + "conserved_entrants", + "expand.entrants@1", + structural=StructuralDelta.EXPAND, + base="survey", + entrants=True, + mass="conserve", + ) + + expand, claim = toy.entrant_expand_node() + run = toy.run_toy( + toy.small_graph(nodes=(toy.CREATE, expand, claim)), tmp_path / "declared" + ) + before = run.manifest.population("survey") + after = run.manifest.population(expand.id) + person_copy_id = int(before.person["person_id"].max()) + 1 + household_entrant_id = int(before.household["household_id"].max()) + 1 + + assert len(after.person) == len(before.person) + 1 + assert len(after.household) == len(before.household) + 1 + copied = after.person.set_index("person_id").loc[person_copy_id] + source = before.person.set_index("person_id").loc[1] + assert copied["person_household_id"] == household_entrant_id + pd.testing.assert_series_equal( + copied.drop(labels="person_household_id"), + source.drop(labels="person_household_id"), + check_names=False, + ) + entrant = after.household.set_index("household_id").loc[household_entrant_id] + assert entrant["household_size"] == 1 + assert after.household["household_size"].dtype == np.dtype("int64") + assert after.weights_for("household").values[-1] == 125.0 + lineage = run.manifest.nodes[expand.id].receipt + assert lineage["expand"]["person"] == ((person_copy_id, 1),) + assert lineage["expand"]["household"] == ((household_entrant_id, None),) + + undeclared, undeclared_claim = toy.entrant_expand_node( + "undeclared_entrants", entrants=False + ) + with pytest.raises(NodeRejectedError, match="undeclared_entrants"): + toy.run_toy( + toy.small_graph(nodes=(toy.CREATE, undeclared, undeclared_claim)), + tmp_path / "undeclared", + ) + + incomplete, incomplete_claim = toy.entrant_expand_node( + "incomplete_entrant", missing_entrant_column="household_size" + ) + with pytest.raises(NodeRejectedError, match="incomplete_entrant") as error: + toy.run_toy( + toy.small_graph(nodes=(toy.CREATE, incomplete, incomplete_claim)), + tmp_path / "incomplete", + ) + assert "household_size" in str(error.value) + + +def test_b7_entrant_persons_carry_their_stratum(tmp_path: Path) -> None: + """An entrant person's stratum arrives through ``KernelResult.strata``. + + Immigrant cohorts are persons, so an EXPAND admitting entrants must be + able to add a person that copies nobody: every column materialized, + memberships naming incumbent groups, and its stratum declared by id. + The ledger counts the entrant from the node that admits it; a missing + label, a label for an id the node never adds, or a label for an + incumbent person rejects the node by name. + """ + expand, claim = toy.entrant_person_node() + run = toy.run_toy( + toy.small_graph(nodes=(toy.CREATE, expand, claim)), tmp_path / "cohort" + ) + before = run.manifest.population("survey") + after = run.manifest.population(expand.id) + entrant_id = int(before.person["person_id"].max()) + 1 + + assert len(after.person) == len(before.person) + 1 + assert len(after.household) == len(before.household) + entrant = after.person.set_index("person_id").loc[entrant_id] + assert entrant["age"] == 30 and entrant["income"] == 12_500.0 + assert ( + entrant["person_household_id"] == before.person["person_household_id"].iloc[0] + ) + assert after.strata.iloc[-1] == "urban" + assert after.strata.iloc[: len(before.person)].tolist() == before.strata.tolist() + + receipt = run.manifest.nodes[expand.id].receipt + assert receipt["expand"]["person"] == ((entrant_id, None),) + mass = receipt["mass"] + assert mass["after"] > mass["before"] + assert mass["stratum_after"]["urban"] > mass["stratum_before"]["urban"] + assert mass["stratum_after"]["rural"] == mass["stratum_before"]["rural"] + + # A copied person takes its stratum from lineage, so a label for it is + # refused exactly like a label for an incumbent or an unknown id. + for mode in ("missing", "unknown_id", "labels_incumbent", "labels_copied"): + bad, bad_claim = toy.entrant_person_node(f"cohort_{mode}", strata_mode=mode) + with pytest.raises(NodeRejectedError, match=f"cohort_{mode}"): + toy.run_toy( + toy.small_graph(nodes=(toy.CREATE, bad, bad_claim)), + tmp_path / mode, + ) diff --git a/packages/microcosm-graph/tests/test_acceptance_c_seeds.py b/packages/microcosm-graph/tests/test_acceptance_c_seeds.py index 07165e789..54f398fee 100644 --- a/packages/microcosm-graph/tests/test_acceptance_c_seeds.py +++ b/packages/microcosm-graph/tests/test_acceptance_c_seeds.py @@ -18,9 +18,23 @@ import ast import importlib.util import sys +from dataclasses import replace from pathlib import Path -from microcosm.graph import ContentStore, Graph, compile_graph +import pytest + +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + Node, + Numeric, + Owned, + Slice, + Tolerance, + compile_graph, +) if "_toy" not in sys.modules: _SPEC = importlib.util.spec_from_file_location( @@ -167,3 +181,99 @@ def test_c4_seed_from_identity(tmp_path: Path) -> None: assert elsewhere.keys()["draw_z"] != here.keys()["draw_a"] assert elsewhere.seeds()["draw_z"] != here.seeds()["draw_a"] assert len(set(elsewhere.seeds().values())) == len(elsewhere.seeds()) + + +def test_c5_tolerance_is_declared(tmp_path: Path) -> None: + """Receipts and readers carry an owner's exact declared tolerance. + + Capability receipts encode a tolerance as ``rtol``, ``atol``, and ``ulps``; + bitwise owners encode it as ``None``. A gate reads that same owner mapping + by coordinate and reports the JSON-safe value in its evidence. + """ + with pytest.raises(ValueError, match="must declare its Tolerance"): + Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + ) + with pytest.raises(ValueError, match="bitwise kernel declares no Tolerance"): + Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + tolerance=Tolerance(rtol=1e-6), + ) + + bounded = Node( + "bounded", + "derive.tolerant@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "bounded_value", "float64"),), + params={ + "entity": "person", + "columns": ("age",), + "target": "bounded_value", + "scale": 1.0, + }, + population="survey", + ) + bitwise = toy.derive("bitwise", ("age",), "bitwise_value") + + def tolerance_gate(node_id: str, column: str) -> Node: + verdict = f"{node_id}_verdict" + return Node( + node_id, + "gate.tolerance@1", + inputs=(Slice("person", (column,)),), + outputs=(Owned("release", verdict, "string"),), + params={ + "entity": "person", + "column": column, + "verdict_column": verdict, + }, + population="survey", + ) + + bounded_gate = tolerance_gate("bounded_gate", "bounded_value") + bitwise_gate = tolerance_gate("bitwise_gate", "bitwise_value") + graph = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE, bounded, bitwise, bounded_gate, bitwise_gate), + ) + run = toy.run_toy(graph, tmp_path / "run") + bound = {"rtol": 1e-6, "atol": 0.0, "ulps": 0} + + bounded_receipt = run.manifest.nodes[bounded.id].receipt + assert bounded_receipt["capabilities"]["tolerance"] == bound + bounded_evidence = run.manifest.nodes[bounded_gate.id].receipt + assert bounded_evidence["outcome"] == "pass" + assert bounded_evidence["evidence"]["tolerance"] == bound + + bitwise_receipt = run.manifest.nodes[bitwise.id].receipt + assert bitwise_receipt["capabilities"]["tolerance"] is None + bitwise_evidence = run.manifest.nodes[bitwise_gate.id].receipt + assert bitwise_evidence["outcome"] == "pass" + assert bitwise_evidence["evidence"]["tolerance"] is None + + # A carried column keeps its producer's tolerance through a structural + # version: the bitwise FILTER that carries ``bounded_value`` into + # ``adults_view`` neither tightens nor erases the bound a reader sees. + carrier = toy.select_node("adults_view", base="survey", policy="free") + carried_gate = replace( + tolerance_gate("carried_gate", "bounded_value"), population=carrier.id + ) + carried_bitwise_gate = replace( + tolerance_gate("carried_bitwise_gate", "bitwise_value"), + population=carrier.id, + ) + carried = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE, bounded, bitwise, carrier, carried_gate, carried_bitwise_gate), + ) + carried_run = toy.run_toy(carried, tmp_path / "carried") + carried_evidence = carried_run.manifest.nodes[carried_gate.id].receipt + assert carried_evidence["outcome"] == "pass" + assert carried_evidence["evidence"]["tolerance"] == bound + carried_bitwise = carried_run.manifest.nodes[carried_bitwise_gate.id].receipt + assert carried_bitwise["outcome"] == "pass" + assert carried_bitwise["evidence"]["tolerance"] is None diff --git a/packages/microcosm-graph/tests/test_acceptance_d_weights.py b/packages/microcosm-graph/tests/test_acceptance_d_weights.py index 4b254bd78..baf5c45c1 100644 --- a/packages/microcosm-graph/tests/test_acceptance_d_weights.py +++ b/packages/microcosm-graph/tests/test_acceptance_d_weights.py @@ -15,15 +15,21 @@ from __future__ import annotations import importlib.util +import json import sys +from dataclasses import replace from pathlib import Path +import numpy as np +import pandas as pd import pytest from microcosm.frame import WeightKind from microcosm.graph import ( + Graph, GraphError, Node, + NodeRejectedError, Owned, Slice, StructuralDelta, @@ -215,6 +221,135 @@ def test_d5_uncertainty_travels(tmp_path: Path) -> None: assert toy.calibrated_node(kernel="calibrate.blind@1").params["target_se"] == 2500.0 +def test_d6_mass_is_partitioned(tmp_path: Path) -> None: + """Mass is conserved and receipted inside every partition value. + + The receipt retains D2's flat totals and stratum maps and adds + ``mass["partition"]`` with ``entity``, ``column``, and nested + ``stratum_before``/``stratum_after`` maps keyed first by the JSON-string + partition value and then by stratum. This test fixes that public shape for + the implementation lane. + """ + missing = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE,), + mass_partition=("person", "period"), + ) + with pytest.raises(GraphError, match=r"person\.period.*survey"): + compile_graph(missing) + + source_path = toy.copy_source(tmp_path / "period-source") + person_path = source_path / "person.csv" + person = pd.read_csv(person_path) + is_adult = person["is_adult"].astype("boolean").fillna(False).to_numpy(bool) + person["period"] = np.where( + ~is_adult | person["person_id"].mod(2).eq(0), 2024, 2025 + ).astype("int64") + person.to_csv(person_path, index=False) + + schema_path = source_path / "schema.json" + schema = json.loads(schema_path.read_text()) + schema["data_columns"]["person"].append("period") + schema["dtypes"]["period"] = "int64" + schema_path.write_text(json.dumps(schema, indent=2, sort_keys=True) + "\n") + + create = replace( + toy.CREATE, + outputs=(*toy.CREATE.outputs, Owned("person", "period", "int64")), + ) + + # A partition value is fixed when the row is created: a later node that + # writes or rewrites the column could move mass between partitions with + # the total unchanged, which no policy can see, so compilation refuses it. + reassigning = Graph( + "toy", + (toy.SOURCE,), + ( + create, + toy.select_node("period_view", base="survey", policy="free"), + Node( + "reassign_period", + "derive.rewrite@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "period", "int64", rewrite=True),), + population="period_view", + ), + ), + mass_partition=("person", "period"), + ) + with pytest.raises(GraphError, match=r"reassign_period.*mass partition"): + compile_graph(reassigning) + + sources = {"survey": source_path} + conserving = Graph( + "toy", + (toy.SOURCE,), + ( + create, + toy.select_node("period_conserve", base="survey", policy="conserve"), + ), + mass_partition=("person", "period"), + ) + with pytest.raises(NodeRejectedError, match="period_conserve") as rejected: + toy.run_toy(conserving, tmp_path / "conserve", sources=sources) + assert "2024" in str(rejected.value) + + free = Graph( + "toy", + (toy.SOURCE,), + (create, toy.select_node("period_free", base="survey", policy="free")), + mass_partition=("person", "period"), + ) + run = toy.run_toy(free, tmp_path / "free", sources=sources) + mass = run.manifest.nodes["period_free"].receipt["mass"] + partition = mass["partition"] + assert set(partition) == { + "entity", + "column", + "stratum_before", + "stratum_after", + } + assert partition["entity"] == "person" + assert partition["column"] == "period" + + weights = pd.read_csv(source_path / "weights.csv").set_index("household_id")[ + "design_weight" + ] + weighted = person.assign( + _mass=person["person_household_id"].map(weights).astype("float64") + ) + + def expected(frame: pd.DataFrame) -> dict[str, dict[str, float]]: + grouped = frame.groupby(["period", "stratum"], observed=True)["_mass"].sum() + return { + str(period): { + str(stratum): float(grouped.loc[(period, stratum)]) + for stratum in grouped.loc[period].index + } + for period in grouped.index.get_level_values("period").unique() + } + + before = expected(weighted) + after = expected(weighted.loc[is_adult]) + for key, wanted in (("stratum_before", before), ("stratum_after", after)): + actual = partition[key] + assert set(actual) == set(wanted) == {"2024", "2025"} + for period, strata in wanted.items(): + assert actual[period] == pytest.approx(strata) + + before_total = sum(sum(strata.values()) for strata in before.values()) + after_total = sum(sum(strata.values()) for strata in after.values()) + assert mass["before"] == pytest.approx(before_total) + assert mass["after"] == pytest.approx(after_total) + assert partition["stratum_after"]["2025"] == pytest.approx( + partition["stratum_before"]["2025"] + ) + assert sum(partition["stratum_after"]["2024"].values()) < sum( + partition["stratum_before"]["2024"].values() + ) + + def test_the_toy_country_declares_the_weight_lineage_the_charter_names() -> None: """The toy graph really does compose the three kinds, in order. diff --git a/packages/microcosm-graph/tests/test_acceptance_h_parity.py b/packages/microcosm-graph/tests/test_acceptance_h_parity.py index 7045973e0..44698f765 100644 --- a/packages/microcosm-graph/tests/test_acceptance_h_parity.py +++ b/packages/microcosm-graph/tests/test_acceptance_h_parity.py @@ -23,6 +23,8 @@ import numpy as np import pytest +from microcosm.graph import platform_fingerprint + if "_toy" not in sys.modules: _SPEC = importlib.util.spec_from_file_location( "_toy", Path(__file__).with_name("_toy.py") @@ -37,7 +39,8 @@ #: H1: one directory per wrapped legacy kernel, each holding ``graph.json`` #: (the node declaration), ``inputs.csv``, ``direct.csv`` (the direct call's #: output at the pinned seed), and ``pins.json`` (seed, kernel ref, kernel -#: implementation hash, and the dependency versions the pin was taken under). +#: implementation hash, target node key, and the dependency versions the pin +#: was taken under). KERNEL_PARITY = PARITY / "kernels" #: H2: ``uk_spine.json`` — the 28-stage FRS spine expressed as a graph — plus @@ -60,7 +63,7 @@ #: not promise cross-platform bit stability, so ``fit.qrf@1`` says so; parity #: in the locked environment is still asserted byte for byte below. NUMERIC_CLAIMS = { - "fit.qrf": "tolerance_bound", + "fit.qrf": "platform_bitwise", "calibrate": "bitwise", "simulate": "bitwise", } @@ -161,7 +164,15 @@ def test_h1_kernel_parity(tmp_path: Path) -> None: for name in WRAPPED_KERNELS: case = _require(KERNEL_PARITY / name, "the kernel-wrapper lane") pins = json.loads((case / "pins.json").read_text()) - assert set(pins) >= {"seed", "kernel", "implementation_hash", "dependencies"} + assert set(pins) >= { + "seed", + "kernel", + "implementation_hash", + "dependencies", + "node_key", + "numeric", + "platform", + } kernel = registry.get(pins["kernel"]) assert kernel.implementation_hash() == pins["implementation_hash"] assert set(pins["dependencies"]) == set(kernel.capabilities.dependencies) @@ -169,7 +180,7 @@ def test_h1_kernel_parity(tmp_path: Path) -> None: store = ContentStore(tmp_path / name) manifest = run_graph( compile_graph(graph_from_json((case / "graph.json").read_text())), - sources={"fixture": case}, + sources={"fixture": case / "inputs.csv"}, store=store, kernels=registry, resume="forbid", @@ -181,30 +192,54 @@ def test_h1_kernel_parity(tmp_path: Path) -> None: # own; the direct call produced only what direct.csv holds, so those # are the cells compared. A weight transition is compared through the # weight artifact under the ``.weights`` column. - direct = _direct_table(case) - compared = 0 + # A platform-bitwise kernel's bytes are asserted on every platform + # that carries a pin: the authoring platform and each CI platform + # (amendment 16). Elsewhere the property that holds is identity + # partitioning: the node key carries the platform, so the local key + # differs from every pinned platform's key and a shared store can + # never serve another platform's artifact. + platforms = dict(pins.get("platforms", {})) + platforms.setdefault( + pins["platform"], {"node_key": pins["node_key"], "direct": "direct.csv"} + ) + local = platforms.get(platform_fingerprint()) + off_platform = pins["numeric"] == "platform_bitwise" and local is None + if off_platform: + assert node.key not in {entry["node_key"] for entry in platforms.values()} + direct = _direct_table(case) + else: + pinned = ( + local + if pins["numeric"] == "platform_bitwise" + else platforms[pins["platform"]] + ) + assert node.key == pinned["node_key"] + direct = _direct_table(case, pinned["direct"]) + exposed = 0 for cell, key in node.artifacts.items(): label = f"{cell[0]}.{cell[1]}" if label in direct.columns: - _assert_same_bytes(store.load_column(key), direct[label]) - compared += 1 + exposed += 1 + if not off_platform: + _assert_same_bytes(store.load_column(key), direct[label]) if node.weight_key is not None: + exposed += 1 entity = ( graph_from_json((case / "graph.json").read_text()) .node(pins["node"]) .weights.entity ) - _assert_same_bytes( - store.load_column(node.weight_key), direct[f"{entity}.weights"] - ) - compared += 1 - assert compared, f"{name}: the fixture exposed nothing to compare" + if not off_platform: + _assert_same_bytes( + store.load_column(node.weight_key), direct[f"{entity}.weights"] + ) + assert exposed, f"{name}: the fixture exposed nothing to compare" -def _direct_table(case: Path): +def _direct_table(case: Path, name: str = "direct.csv"): import pandas as pd - return pd.read_csv(case / "direct.csv", float_precision="round_trip") + return pd.read_csv(case / name, float_precision="round_trip") @pytest.mark.requires_uk diff --git a/packages/microcosm-graph/tests/test_graph_acceptance_burndown.py b/packages/microcosm-graph/tests/test_graph_acceptance_burndown.py index dc1cc330d..5da99cbbb 100644 --- a/packages/microcosm-graph/tests/test_graph_acceptance_burndown.py +++ b/packages/microcosm-graph/tests/test_graph_acceptance_burndown.py @@ -273,7 +273,9 @@ def test_the_real_suite_is_all_strict_and_all_accounted_for() -> None: root = burndown.ROOT assert data["total"] == 0 assert not [entry for entry in data["properties"] if entry["state"] == "missing"] - assert all(entry["state"] == "green" for entry in data["properties"]) + states = {entry["id"]: entry["state"] for entry in data["properties"]} + assert not {identifier for identifier, state in states.items() if state == "red"} + assert all(state == "green" for state in states.values()) for entry in data["files"]: source = (root / entry["file"]).read_text() for marker in markers_in(source, entry["file"]): @@ -291,3 +293,348 @@ def test_the_tool_runs_from_the_command_line(flag: str) -> None: ) assert completed.returncode == 0, completed.stderr assert completed.stdout + + +NEW_PROPERTY_STARTS_RED = ( + ONE_RED_PROPERTY + + """ + +@pytest.mark.xfail(strict=True, reason="charter A9: pending") +def test_a9_nine() -> None: + assert False +""" +) + + +A1_GREEN_A3_RED = """ +import pytest + + +def test_a1_one() -> None: + assert True + + +@pytest.mark.xfail(strict=True, reason="charter A3: pending") +def test_a3_three() -> None: + assert False +""" + + +def test_verify_ratchets_on_property_identity_not_on_counts(tmp_path: Path) -> None: + """A1 going green cannot pay for A3 going red: the count is 1 -> 1, and + the ratchet still refuses, naming the property and the file.""" + root = _repository(tmp_path, {"test_acceptance_a.py": ONE_RED_PROPERTY}) + target = root / "packages" / "microcosm-graph" / "tests" / "test_acceptance_a.py" + target.write_text(A1_GREEN_A3_RED) + swapped = _run(root, "--verify") + assert swapped.returncode == 1 + assert "1 -> 1" in swapped.stdout + assert "re-reds 1 property: A3" in swapped.stdout + assert "test_acceptance_a.py" in swapped.stdout + assert "verification=failed" in swapped.stdout + + +def test_verify_counts_markers_in_new_files(tmp_path: Path) -> None: + """A file the baseline lacks constrains nothing by itself, but a marker + inside it on a property that was green is still a re-red.""" + root = _repository(tmp_path, {"test_acceptance_a.py": ONE_RED_PROPERTY}) + tests = root / "packages" / "microcosm-graph" / "tests" + (tests / "test_acceptance_b.py").write_text( + 'import pytest\n\n\n@pytest.mark.xfail(strict=True, reason="charter A3: moved")\n' + "def test_a3_three_again() -> None:\n assert False\n" + ) + moved = _run(root, "--verify") + assert moved.returncode == 1 + assert "[new] packages/microcosm-graph/tests/test_acceptance_b.py" in moved.stdout + assert "re-reds 1 property: A3" in moved.stdout + assert "test_acceptance_b.py" in moved.stdout + + +def test_verify_refuses_a_marker_on_an_id_the_charter_does_not_list( + tmp_path: Path, +) -> None: + """An invented id is not a new property; it is a marker on nothing.""" + root = _repository(tmp_path, {"test_acceptance_a.py": ONE_RED_PROPERTY}) + target = root / "packages" / "microcosm-graph" / "tests" / "test_acceptance_a.py" + target.write_text(NEW_PROPERTY_STARTS_RED) # A9 is not in the charter here + invented = _run(root, "--verify") + assert invented.returncode == 1 + assert "names A9, which docs/graph-acceptance.md does not list" in invented.stdout + + +MISLABELLED_MARKER = """ +import pytest + + +@pytest.mark.xfail(strict=True, reason="charter A3: pending") +def test_a1_one() -> None: + assert False + + +@pytest.mark.xfail(strict=True, reason="charter A3: pending") +def test_a3_three() -> None: + assert False +""" + + +def test_verify_binds_a_marker_to_the_test_it_decorates(tmp_path: Path) -> None: + """The reason is free text; the test name is the binding. + + A marker on ``test_a1_*`` that claims A3 would let A1 go red while the + set of red ids stays {A3}; the tool refuses the mismatch and the + duplicate, so the ratchet compares one marker per property. + """ + root = _repository(tmp_path, {"test_acceptance_a.py": ONE_RED_PROPERTY}) + target = root / "packages" / "microcosm-graph" / "tests" / "test_acceptance_a.py" + target.write_text(MISLABELLED_MARKER) + mislabelled = _run(root, "--verify") + assert mislabelled.returncode == 1 + assert "test_a1_one claims charter A3 but is not that property's test" in ( + mislabelled.stdout + ) + assert "A3 carries two markers" in mislabelled.stdout + + +def test_verify_refuses_removing_a_charter_property(tmp_path: Path) -> None: + """Dropping a baseline row (and its test) is not a way to go green.""" + root = _repository(tmp_path, {"test_acceptance_a.py": ONE_RED_PROPERTY}) + (root / "docs" / "graph-acceptance.md").write_text( + "| Id | Property |\n|---|---|\n| A3 | three |\n" + ) + target = root / "packages" / "microcosm-graph" / "tests" / "test_acceptance_a.py" + target.write_text("def test_a3_three() -> None:\n assert True\n") + removed = _run(root, "--verify") + assert removed.returncode == 1 + assert "charter A1 was listed on origin/main but is gone" in removed.stdout + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + "import pytest\n\npytestmark = pytest.mark.skip\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import pytest\n\npytestmark: list = [pytest.mark.skip]\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import pytest\n\nif True:\n pytestmark: list = [pytest.mark.skip]\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import pytest\n\n(pytestmark,) = (pytest.mark.skip,)\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import pytest\n\nif (pytestmark := pytest.mark.skip):\n pass\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import pytest\n\nglobals()['pytestmark'] = pytest.mark.skip\n\n\ndef test_a1_one() -> None:\n assert False\n", + "dynamic module namespace", + ), + ( + "import pytest\nimport sys\n\nsys.modules[__name__].pytestmark = pytest.mark.skip\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import pytest\nimport sys\n\nsys.modules[__name__].__dict__['pytestmark'] = pytest.mark.skip\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + "import sys\nfrom helper import SKIP_MARK\n\nsys.modules[__name__].__dict__.update({'pytestmark': SKIP_MARK})\n\n\ndef test_a1_one() -> None:\n assert False\n", + "dynamic module namespace", + ), + ( + "import sys\nfrom helper import SKIP_MARK\n\nsys.modules[__name__].__dict__.setdefault('pytestmark', SKIP_MARK)\n\n\ndef test_a1_one() -> None:\n assert False\n", + "dynamic module namespace", + ), + ( + "import pytest\n\n\n@pytest.mark.skip\ndef test_a1_one() -> None:\n assert False\n", + "carries mark 'skip'", + ), + ( + "import pytest\n\nskip = pytest.mark.skip\n\n\n@skip\ndef test_a1_one() -> None:\n assert False\n", + "unrecognized decorator 'skip'", + ), + ( + "import pytest\n\nsuppress = pytest.skip\n\n\ndef test_a1_one() -> None:\n suppress('x')\n", + "references pytest.skip", + ), + ( + "import pytest\n\nsuppress = getattr(pytest, 'skip')\n\n\ndef test_a1_one() -> None:\n suppress('x')\n", + "dynamic module namespace", + ), + ( + "import pytest\n\np = pytest\n\n\ndef test_a1_one() -> None:\n p.skip('x')\n", + "outside a direct pytest.mark decorator", + ), + ( + "import pytest\n\n\n@pytest.mark.parametrize('p', [pytest])\ndef test_a1_one(p) -> None:\n p.skip('x')\n", + "outside a direct pytest.mark decorator", + ), + ( + "import pytest\n\n\ndef test_a1_one() -> None:\n p = pytest\n p.skip('x')\n", + "outside a direct pytest.mark decorator", + ), + ( + "import pytest\n\nnamespace = globals\nnamespace()['pytestmark'] = pytest.mark.skip\n\n\ndef test_a1_one() -> None:\n assert False\n", + "dynamic module namespace", + ), + ( + "import types\nimport unittest\n\ndisguised = types.SimpleNamespace(mark=types.SimpleNamespace(parametrize=lambda *args, **kwargs: unittest.skip('hidden')))\n\n\n@disguised.mark.parametrize('x', [1])\ndef test_a1_one(x) -> None:\n assert False\n", + "unrecognized decorator 'disguised.mark.parametrize'", + ), + ( + "import pytest\nimport types\n\npytest = types.SimpleNamespace(mark=types.SimpleNamespace(parametrize=lambda *args, **kwargs: (lambda function: function)))\n\n\n@pytest.mark.parametrize('x', [1])\ndef test_a1_one(x) -> None:\n assert False\n", + "rebinds pytest", + ), + ( + "import pytest\nimport unittest\n\npytest.mark.parametrize = lambda *args, **kwargs: unittest.skip('hidden')\n\n\n@pytest.mark.parametrize('x', [1])\ndef test_a1_one(x) -> None:\n assert False\n", + "rebinds pytest", + ), + ( + "import pytest\nimport unittest\n\nmarks = pytest.mark\nmarks.parametrize = lambda *args, **kwargs: unittest.skip('hidden')\n\n\n@pytest.mark.parametrize('x', [1])\ndef test_a1_one(x) -> None:\n assert False\n", + "outside a direct pytest.mark decorator", + ), + ( + "from counterfeit import pytest\n\n\n@pytest.mark.parametrize('x', [1])\ndef test_a1_one(x) -> None:\n assert False\n", + "rebinds pytest", + ), + ( + "import pytest\n\n\ndef install_marker() -> None:\n global pytestmark\n pytestmark = pytest.mark.skip\n\n\ninstall_marker()\n\n\ndef test_a1_one() -> None:\n assert False\n", + "module-level pytestmark", + ), + ( + 'import pytest\n\n\n@pytest.mark.skipif(True, reason="x")\ndef test_a1_one() -> None:\n assert False\n', + "carries mark 'skipif'", + ), + ( + "import unittest\n\n\n@unittest.skip('x')\ndef test_a1_one() -> None:\n assert False\n", + "unrecognized decorator 'unittest.skip'", + ), + ( + "import unittest\n\n\n@unittest.skipIf(True, 'x')\ndef test_a1_one() -> None:\n assert False\n", + "unrecognized decorator 'unittest.skipIf'", + ), + ( + "import unittest\n\n\n@unittest.skipUnless(False, 'x')\ndef test_a1_one() -> None:\n assert False\n", + "unrecognized decorator 'unittest.skipUnless'", + ), + ( + "import unittest\n\n\ndef test_a1_one() -> None:\n raise unittest.SkipTest('hidden')\n", + "runtime unittest.SkipTest() suppresses", + ), + ( + "import unittest\n\n\ndef test_a1_one() -> None:\n raise unittest.SkipTest\n", + "references unittest.SkipTest", + ), + ( + "from unittest import SkipTest\n\n\ndef test_a1_one() -> None:\n raise SkipTest('hidden')\n", + "importing unittest SkipTest", + ), + ( + "import pytest\n\n\nclass TestA:\n def test_a1_one(self) -> None:\n assert False\n", + "tests must be module-level functions", + ), + ( + 'import pytest\n\n\ndef test_a1_one() -> None:\n pytest.xfail("later")\n', + "runtime pytest.xfail() suppresses", + ), + ( + 'import pytest\n\n\n@pytest.mark.parametrize("x", [pytest.param(1, marks=pytest.mark.xfail)])\ndef test_a1_one(x) -> None:\n assert False\n', + "smuggles marks through pytest.param", + ), + ( + "def test_a1_outer() -> None:\n def test_a1_one() -> None:\n assert False\n", + "nests test_a1_one", + ), + ], +) +def test_suppression_forms_the_marker_scan_cannot_model_are_refused( + source: str, expected: str +) -> None: + problems = burndown.suppressions_in(source, "sample.py") + assert any(expected in problem for problem in problems), problems + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + 'import pytest as hidden\n\n\n@hidden.mark.xfail(strict=True, reason="charter A1: x")\ndef test_a1_one() -> None:\n assert False\n', + "imported under an alias", + ), + ( + 'from pytest import mark as hidden\n\n\n@hidden.xfail(strict=True, reason="charter A1: x")\ndef test_a1_one() -> None:\n assert False\n', + "hides marker spellings", + ), + ( + 'import pytest\n\nCASES = [pytest.param(1, marks=pytest.mark.xfail)]\n\n\n@pytest.mark.parametrize("x", CASES)\ndef test_a1_one(x) -> None:\n assert False\n', + "non-literal", + ), + ( + "import pytest\n\nCASES = [pytest.param(1, marks=pytest.mark.xfail)]\n\n\ndef test_a1_one() -> None:\n assert False\n", + "pytest.param(..., marks=...) is not allowed", + ), + ], +) +def test_aliases_and_indirect_parameters_are_refused( + source: str, expected: str +) -> None: + problems = burndown.suppressions_in(source, "sample.py") + assert any(expected in problem for problem in problems), problems + + +def test_the_allowed_marks_are_not_suppressions() -> None: + source = ( + "import pytest\n\n\n" + '@pytest.mark.requires_uk\n@pytest.mark.parametrize("x", [1, 2])\n' + '@pytest.mark.xfail(strict=True, reason="charter A1: pending")\n' + "def test_a1_one(x) -> None:\n assert False\n" + ) + assert burndown.suppressions_in(source, "sample.py") == () + + +def test_the_literal_toy_module_loader_is_not_a_suppression() -> None: + source = ( + "import sys\n\n" + "if '_toy' not in sys.modules:\n" + " sys.modules['_toy'] = object()\n" + "toy = sys.modules['_toy']\n\n\n" + "def test_a1_one() -> None:\n assert toy is not None\n" + ) + assert burndown.suppressions_in(source, "sample.py") == () + + +def test_verify_lets_a_property_new_to_the_charter_start_red(tmp_path: Path) -> None: + """The charter's meta-TDD rule: a new property is committed red first. + + A marker on an id the baseline charter never listed is not a re-red; a + marker on an id it did list still is. + """ + root = _repository(tmp_path, {"test_acceptance_a.py": ONE_RED_PROPERTY}) + (root / "docs" / "graph-acceptance.md").write_text( + "| Id | Property |\n|---|---|\n| A1 | one |\n| A3 | three |\n| A9 | nine |\n" + ) + target = root / "packages" / "microcosm-graph" / "tests" / "test_acceptance_a.py" + target.write_text(NEW_PROPERTY_STARTS_RED) + admitted = _run(root, "--verify") + assert admitted.returncode == 0, admitted.stdout + assert "1 -> 1 (+1 new: A9)" in admitted.stdout + assert "verification=ok" in admitted.stdout + + # The same marker on a property the baseline charter already listed + # (A3, green there) is a re-red and still fails. + target.write_text( + NEW_PROPERTY_STARTS_RED.replace( + "def test_a3_three() -> None:\n assert True", + '@pytest.mark.xfail(strict=True, reason="charter A3: pending")\n' + "def test_a3_three() -> None:\n assert False", + ) + ) + refused = _run(root, "--verify") + assert refused.returncode == 1 + assert "re-reds 1 property" in refused.stdout diff --git a/packages/microcosm-graph/tests/test_graph_decl.py b/packages/microcosm-graph/tests/test_graph_decl.py index c5065ae6f..9ee8426ac 100644 --- a/packages/microcosm-graph/tests/test_graph_decl.py +++ b/packages/microcosm-graph/tests/test_graph_decl.py @@ -18,6 +18,7 @@ Slice, SourceRef, StructuralDelta, + WeightTransition, compile_graph, ) @@ -270,3 +271,81 @@ def test_a_filter_cannot_read_a_column_nobody_defines() -> None: ) with pytest.raises(GraphError, match="no node owns"): compile_graph(Graph("toy", (SRC,), (CREATE, subset))) + + +def _expand(node_id: str, **overrides: object) -> Node: + settings: dict[str, object] = {"mass": "free", **overrides} + return Node( + node_id, + "clone.persons@1", + base="survey", + structural=StructuralDelta.EXPAND, + **settings, # type: ignore[arg-type] + ) + + +def test_entrants_are_an_expand_declaration_that_cannot_conserve_mass() -> None: + """Amendment 11: entrants add rows without lineage, so they add mass.""" + admitted = _expand("cohort", entrants=True) + assert admitted.entrants is True + assert "entrants" in admitted.normative() + with pytest.raises(GraphError, match="cannot declare mass='conserve'"): + _expand("cohort", entrants=True, mass="conserve") + with pytest.raises(GraphError, match="only an EXPAND node"): + Node("fit", "fit.qrf@1", entrants=True) + with pytest.raises(GraphError, match="must be a boolean"): + _expand("cohort", entrants="yes") + + +def test_mass_partition_must_be_declared_by_every_create_node() -> None: + """Amendment 12: partitions exist from the first version, with a partition dtype.""" + periodic = Node( + "survey", + "source.frame@1", + sources=("survey",), + structural=StructuralDelta.CREATE, + outputs=(Owned("person", "age", "int64"), Owned("person", "period", "int64")), + ) + graph = Graph("toy", (SRC,), (periodic,), mass_partition=("person", "period")) + compiled = compile_graph(graph) + assert compiled.graph.normative() == {"mass_partition": ("person", "period")} + assert Graph("toy", (SRC,), (periodic,)).normative() == {"mass_partition": None} + with pytest.raises(GraphError, match="does not declare"): + compile_graph( + Graph("toy", (SRC,), (CREATE,), mass_partition=("person", "period")) + ) + floating = Node( + "survey", + "source.frame@1", + sources=("survey",), + structural=StructuralDelta.CREATE, + outputs=(Owned("person", "period", "float64"),), + ) + with pytest.raises(GraphError, match="partition column must be one of"): + compile_graph( + Graph("toy", (SRC,), (floating,), mass_partition=("person", "period")) + ) + with pytest.raises(GraphError, match="pair of strings"): + Graph("toy", (SRC,), (periodic,), mass_partition=("person",)) # type: ignore[arg-type] + + +def test_declared_names_may_not_contain_dots() -> None: + """Amendment 15: ``entity.column`` spellings must be unambiguous.""" + with pytest.raises(GraphError, match="may not contain '.'"): + Slice("a.b", ("c",)) + with pytest.raises(GraphError, match="may not contain '.'"): + Slice("a", ("b.c",)) + with pytest.raises(GraphError, match="may not contain '.'"): + Owned("a", "b.c", "int64") + with pytest.raises(GraphError, match="may not contain '.'"): + Owned("a.b", "c", "int64") + with pytest.raises(GraphError, match="may not contain '.'"): + Slice("a", ("c",), rows="m.k") + + +def test_every_declared_name_channel_refuses_dots() -> None: + """Amendment 15 covers weight transitions and the mass partition too.""" + with pytest.raises(GraphError, match="may not contain '.'"): + WeightTransition("house.hold", "design", "importance") + with pytest.raises(GraphError, match="may not contain '.'"): + Graph("toy", (), (), mass_partition=("person", "per.iod")) diff --git a/packages/microcosm-graph/tests/test_graph_executor.py b/packages/microcosm-graph/tests/test_graph_executor.py index 8da32936b..be379eeb7 100644 --- a/packages/microcosm-graph/tests/test_graph_executor.py +++ b/packages/microcosm-graph/tests/test_graph_executor.py @@ -3,6 +3,9 @@ from __future__ import annotations import hashlib +import importlib.util +import json +import sys from collections.abc import Callable, Mapping from dataclasses import replace from pathlib import Path @@ -11,9 +14,11 @@ import pandas as pd import pytest +import microcosm.graph.executor as graph_executor from microcosm.frame import EntitySchema, Frame, WeightKind, Weights from microcosm.graph.decl import ( Graph, + GraphError, Node, Owned, Ownership, @@ -31,8 +36,12 @@ KernelRegistry, KernelResult, KernelRole, + Numeric, + NumericScope, + Tolerance, ) -from microcosm.graph.manifest import Decision +from microcosm.graph.keys import platform_fingerprint +from microcosm.graph.manifest import Decision, RunManifest from microcosm.graph.store import ( ContentStore, StoreCorrupt, @@ -40,6 +49,15 @@ StoreUnavailable, ) +if "_toy" not in sys.modules: + _TOY_SPEC = importlib.util.spec_from_file_location( + "_toy", Path(__file__).with_name("_toy.py") + ) + assert _TOY_SPEC is not None and _TOY_SPEC.loader is not None + sys.modules["_toy"] = importlib.util.module_from_spec(_TOY_SPEC) + _TOY_SPEC.loader.exec_module(sys.modules["_toy"]) +toy = sys.modules["_toy"] + SOURCE = SourceRef("survey", "csv-tables", description="toy bytes") CREATE = Node( "survey", @@ -345,8 +363,8 @@ def test_determinism_across_stores_and_zero_kernel_memoization( node: item.key for node, item in second.nodes.items() } assert _object_bytes(first_store) == _object_bytes(second_store) - assert all(not item.hit for item in first.nodes.values()) assert first.key == second.key + assert all(not item.hit for item in first.nodes.values()) assert first.nodes["survey"].frame_key is not None assert first.nodes["a"].frame_key is None assert first.nodes["a"].receipt["capabilities"]["determinism"] == ( # type: ignore[index] @@ -370,7 +388,7 @@ def test_determinism_across_stores_and_zero_kernel_memoization( assert warm.population("survey").table("person")["b"].tolist() == [60, 120, 180] -def test_exact_param_kernel_source_and_decision_invalidation( +def test_exact_param_kernel_source_invalidation_and_decision_node_invariance( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: source = _source_path(tmp_path / "source") @@ -416,10 +434,33 @@ def test_exact_param_kernel_source_and_decision_invalidation( assert {node: item.key for node, item in decided.nodes.items()} == { node: item.key for node, item in changed_source.nodes.items() } - assert decided.key != changed_source.key + assert decided.key == replace(decided, decisions=()).key assert [dict(record) for record in decided.decisions] == [decision] +def test_decisions_supplied_to_a_run_do_not_move_the_manifest_key( + tmp_path: Path, +) -> None: + """F5: a release node's decision-derived outcome is provenance, not identity.""" + + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + graph = _release_graph( + gate_outcome="pass", tier_answer="certified", requires=("publish",) + ) + undecided = _run(graph, source, store, _release_registry()) + decision = { + "name": "publish", + "owner": "reviewer", + "signature": "toy-signature-0001", + } + decided = _run(graph, source, store, _release_registry(), decisions=(decision,)) + assert undecided.nodes["release"].receipt["outcome"] == "unreached" + assert decided.nodes["release"].receipt["outcome"] == "pass" + assert undecided.key == decided.key + assert undecided.to_json() != decided.to_json() + + def test_inert_fields_order_and_leaf_removal_do_not_move_survivors( tmp_path: Path, ) -> None: @@ -667,10 +708,13 @@ def cross_entity(context: KernelContext) -> KernelResult: ) +@pytest.mark.parametrize("explicit_rewrite_input", [False, True]) def test_rewrite_incumbent_is_projected_from_its_owned_declaration( tmp_path: Path, + explicit_rewrite_input: bool, ) -> None: source = _source_path(tmp_path / "source") + boundary_tolerance = Tolerance(atol=1e-6) def keep_all(context: KernelContext) -> KernelResult: person = context.tables["person"] @@ -686,6 +730,15 @@ def rewrite(context: KernelContext) -> KernelResult: "age", "income", } + # The FILTER that opens this version carries ``age`` and ``income`` + # from CREATE without writing them, so the tolerances a reader (and + # a rewrite, for its incumbent) sees are the producer's — bitwise + # here — not the tolerance-bound carrier's (charter C5). + assert context.tolerances == { + ("person", "age"): None, + ("person", "income"): None, + } + return KernelResult( columns={ ("person", "income"): pd.Series( @@ -706,7 +759,9 @@ def rewrite(context: KernelContext) -> KernelResult: rewriter = Node( "rewrite_income", "rewrite.income@1", - inputs=(Slice("person", ("age",)),), + inputs=( + Slice("person", ("age", "income") if explicit_rewrite_input else ("age",)), + ), outputs=(Owned("person", "income", "float64", rewrite=True),), population=boundary.id, ) @@ -714,7 +769,12 @@ def rewrite(context: KernelContext) -> KernelResult: registry.register( _Kernel( boundary.kernel, - Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.FILTER), + Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + structural=StructuralDelta.FILTER, + tolerance=boundary_tolerance, + ), keep_all, ) ) @@ -729,7 +789,7 @@ def rewrite(context: KernelContext) -> KernelResult: manifest = _run( Graph("toy", (SOURCE,), (CREATE, boundary, rewriter)), source, - ContentStore(tmp_path / "store"), + ContentStore(tmp_path / f"store-{explicit_rewrite_input}"), registry, ) @@ -740,6 +800,335 @@ def rewrite(context: KernelContext) -> KernelResult: ] +def test_masked_writer_preserves_upstream_tolerance_provenance( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + producer_tolerance = Tolerance(rtol=1e-5) + + def keep_all(context: KernelContext) -> KernelResult: + person = context.tables["person"] + return KernelResult( + keep=pd.Series(True, index=person["person_id"], dtype="bool") + ) + + def patch_selected(context: KernelContext) -> KernelResult: + person = context.tables["person"] + selected = person["selected"].to_numpy(dtype=np.bool_) + ids = person.loc[selected, "person_id"] + return KernelResult( + columns={ + ("person", "income"): pd.Series( + np.array([10.0, 30.0], dtype=np.float64), + index=pd.Index(ids, name="person_id"), + dtype="float64", + ) + } + ) + + def audit_income(context: KernelContext) -> KernelResult: + assert context.tolerances == {("person", "income"): producer_tolerance} + return KernelResult( + receipt={ + "outcome": "pass", + "evidence": {"tolerance": {"rtol": 1e-5, "atol": 0.0, "ulps": 0}}, + } + ) + + create = replace(CREATE, kernel="masked.source@1") + boundary = Node( + "masked_boundary", + "masked.filter@1", + inputs=(Slice("person", ("selected",)),), + structural=StructuralDelta.FILTER, + base=create.id, + mass="free", + ) + masked = Node( + "masked_income", + "masked.writer@1", + inputs=(Slice("person", ("selected",)),), + outputs=(Owned("person", "income", "float64", rows="selected"),), + population=boundary.id, + ) + reader = Node( + "audit_masked_income", + "masked.gate@1", + inputs=(Slice("person", ("income",)),), + population=boundary.id, + ) + registry = _registry() + for kernel in ( + _Kernel( + create.kernel, + Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + structural=StructuralDelta.CREATE, + tolerance=producer_tolerance, + ), + _source, + ), + _Kernel( + boundary.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + ), + keep_all, + ), + _Kernel( + masked.kernel, + Capabilities(Determinism.DETERMINISTIC), + patch_selected, + ), + _Kernel( + reader.kernel, + Capabilities( + Determinism.DETERMINISTIC, + role=KernelRole.GATE, + ), + audit_income, + ), + ): + registry.register(kernel) + + manifest = _run( + Graph("toy", (SOURCE,), (create, boundary, masked, reader)), + source, + ContentStore(tmp_path / "store"), + registry, + ) + receipt = json.loads(manifest.to_json())["nodes"][reader.id]["receipt"] + assert receipt["outcome"] == "pass" + assert receipt["evidence"]["tolerance_writers"] == { + "person.income": [create.id, masked.id] + } + + +@pytest.mark.parametrize( + ("first_numeric", "second_numeric"), + [ + (Numeric.TOLERANCE_BOUND, Numeric.PLATFORM_BITWISE), + (Numeric.PLATFORM_BITWISE, Numeric.TOLERANCE_BOUND), + ], + ids=["bounded-then-platform", "platform-then-bounded"], +) +def test_input_numeric_scopes_preserve_platform_across_masked_writer_order( + tmp_path: Path, + first_numeric: Numeric, + second_numeric: Numeric, +) -> None: + source = _source_path(tmp_path / "source") + bound = Tolerance(rtol=1e-6, atol=2e-6, ulps=3) + current_platform = platform_fingerprint() + + def write_full(target: str) -> Callable[[KernelContext], KernelResult]: + def compute(context: KernelContext) -> KernelResult: + person = context.tables["person"] + return KernelResult( + columns={ + ("person", target): pd.Series( + person["age"].to_numpy(dtype=np.float64), + index=pd.Index(person["person_id"], name="person_id"), + dtype="float64", + ) + } + ) + + return compute + + def rewrite_selected(context: KernelContext) -> KernelResult: + person = context.tables["person"] + selected = person["selected"].to_numpy(dtype=np.bool_) + return KernelResult( + columns={ + ("person", "mixed_value"): pd.Series( + person.loc[selected, "mixed_value"].to_numpy(dtype=np.float64), + index=pd.Index(person.loc[selected, "person_id"], name="person_id"), + dtype="float64", + ) + } + ) + + def keep_all(context: KernelContext) -> KernelResult: + person = context.tables["person"] + return KernelResult( + keep=pd.Series(True, index=person["person_id"], dtype="bool") + ) + + expected = { + ("person", "age"): NumericScope(), + ("person", "mixed_value"): NumericScope( + numeric=Numeric.TOLERANCE_BOUND, + tolerance=bound, + platform=current_platform, + ), + ("person", "platform_value"): NumericScope( + numeric=Numeric.PLATFORM_BITWISE, + platform=current_platform, + ), + } + + def audit(context: KernelContext) -> KernelResult: + assert context.numerics == expected + assert set(context.tolerances) == set(expected) + for coordinate, scope in context.numerics.items(): + assert context.tolerances[coordinate] == scope.tolerance + return KernelResult(receipt={"outcome": "pass", "evidence": {"scopes": 3}}) + + def numeric_capabilities(numeric: Numeric) -> Capabilities: + return Capabilities( + Determinism.DETERMINISTIC, + numeric=numeric, + tolerance=bound if numeric is Numeric.TOLERANCE_BOUND else None, + ) + + create = replace(CREATE, kernel="numeric.source@1") + boundary = Node( + "numeric_boundary", + "numeric.filter@1", + inputs=(Slice("person", ("selected", "mixed_value", "platform_value")),), + structural=StructuralDelta.FILTER, + base=create.id, + mass="free", + ) + platform_only = Node( + "platform_only", + "numeric.platform-only@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "platform_value", "float64"),), + population=create.id, + ) + first = Node( + "mixed_first", + "numeric.mixed-first@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "mixed_value", "float64"),), + population=create.id, + ) + second = Node( + "mixed_second", + "numeric.mixed-second@1", + inputs=(Slice("person", ("selected",)),), + outputs=( + Owned("person", "mixed_value", "float64", rows="selected", rewrite=True), + ), + population=boundary.id, + ) + gate = Node( + "numeric_gate", + "numeric.gate@1", + inputs=(Slice("person", ("age", "mixed_value", "platform_value")),), + population=boundary.id, + ) + registry = _registry() + for kernel in ( + _Kernel( + create.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.CREATE, + ), + _source, + ), + _Kernel( + platform_only.kernel, + Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + ), + write_full("platform_value"), + ), + _Kernel( + boundary.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + ), + keep_all, + ), + _Kernel( + first.kernel, numeric_capabilities(first_numeric), write_full("mixed_value") + ), + _Kernel(second.kernel, numeric_capabilities(second_numeric), rewrite_selected), + _Kernel( + gate.kernel, + Capabilities(Determinism.DETERMINISTIC, role=KernelRole.GATE), + audit, + ), + ): + registry.register(kernel) + + manifest = _run( + Graph( + "numeric-scopes", + (SOURCE,), + (create, boundary, platform_only, first, second, gate), + ), + source, + ContentStore(tmp_path / "store"), + registry, + ) + assert manifest.nodes[gate.id].receipt["outcome"] == "pass" + + +def test_tolerance_gate_refuses_cross_platform_numeric_scope() -> None: + coordinate = ("person", "income") + current_platform = platform_fingerprint() + gate = toy.GateReportsTolerance( + "gate.platform-scope@1", + Capabilities(Determinism.DETERMINISTIC, role=KernelRole.GATE), + ) + node = Node( + "platform_scope_gate", + gate.ref, + inputs=(Slice("person", ("income",)),), + outputs=(Owned("release", "platform_verdict", "string"),), + params={ + "entity": "person", + "column": "income", + "verdict_column": "platform_verdict", + "comparison_platform": "different/platform", + }, + ) + context = KernelContext( + node=node, + tables={ + "person": pd.DataFrame( + {"person_id": [1], "income": np.array([1.0], dtype=np.float64)} + ), + "release": pd.DataFrame({"release_id": [1]}), + }, + weights={}, + strata=pd.Series(["a"], name="stratum"), + params=node.params, + rng=np.random.default_rng(0), + tolerances={coordinate: None}, + numerics={ + coordinate: NumericScope( + numeric=Numeric.PLATFORM_BITWISE, + platform=current_platform, + ) + }, + ) + + result = gate.run(context) + + assert result.receipt["outcome"] == "evidence_absent" + assert result.receipt["evidence"] == { + "observed": 1.0, + "tolerance": None, + "numeric": "platform_bitwise", + "platform": current_platform, + "comparison_platform": "different/platform", + "reason": "numeric contract is scoped to a different platform", + } + assert result.columns[("release", "platform_verdict")].tolist() == [ + "evidence_absent" + ] + + def test_filter_mask_result_is_applied_to_the_base_frame(tmp_path: Path) -> None: source = _source_path(tmp_path / "source") @@ -779,91 +1168,317 @@ def select_rows(context: KernelContext) -> KernelResult: assert manifest.mass_ledger("selected")[-1].operation == "filter" -def test_expand_lineage_receipt_and_materialized_cell_survive_cache( +@pytest.mark.parametrize( + ( + "producer_tolerance", + "expand_tolerance", + "claim_tolerance", + "claim_expected", + "reader_expected", + ), + [ + ( + Tolerance(rtol=1e-6), + None, + None, + Tolerance(rtol=1e-6), + Tolerance(rtol=1e-6), + ), + ( + None, + Tolerance(atol=2e-6), + Tolerance(ulps=3), + Tolerance(atol=2e-6), + Tolerance(atol=2e-6, ulps=3), + ), + ( + Tolerance(rtol=1e-6), + Tolerance(atol=2e-6), + Tolerance(ulps=3), + Tolerance(rtol=1e-6, atol=2e-6), + Tolerance(rtol=1e-6, atol=2e-6, ulps=3), + ), + ], + ids=("producer-bound", "expand-and-claim-bound", "componentwise-maximum"), +) +def test_entrant_expand_aggregates_all_coordinate_writer_tolerances( tmp_path: Path, + producer_tolerance: Tolerance | None, + expand_tolerance: Tolerance | None, + claim_tolerance: Tolerance | None, + claim_expected: Tolerance, + reader_expected: Tolerance, ) -> None: source = _source_path(tmp_path / "source") - def expand(context: KernelContext) -> KernelResult: + def capabilities( + *, + structural: StructuralDelta = StructuralDelta.NONE, + tolerance: Tolerance | None = None, + role: KernelRole = KernelRole.COMPUTE, + ) -> Capabilities: + if tolerance is None: + return Capabilities( + Determinism.DETERMINISTIC, + structural=structural, + role=role, + ) + return Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + structural=structural, + role=role, + tolerance=tolerance, + ) + + def expand_household(context: KernelContext) -> KernelResult: return KernelResult( expand={ "person": pd.Series( - [1, 2], - index=pd.Index([4, 5], name="person_id"), - dtype="int64", + [1], index=pd.Index([4], name="person_id"), dtype="int64" ), "household": pd.Series( - [10], + [pd.NA], index=pd.Index([30], name="household_id"), - dtype="int64", + dtype="Int64", ), }, columns={ - ("household", "is_clone"): pd.Series( - [False, False, True], + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 30], + index=pd.Index([1, 2, 3, 4], name="person_id"), + dtype="int64", + ), + ("household", "size"): pd.Series( + [2, 1, 1], index=pd.Index([10, 20, 30], name="household_id"), - dtype="bool", - ) + dtype="int64", + ), }, weights=Weights( - np.array([0.5, 2.0, 0.5], dtype=np.float64), - WeightKind.IMPORTANCE, + np.array([1.0, 2.0, 1.0], dtype=np.float64), WeightKind.DESIGN ), ) - def claim(context: KernelContext) -> KernelResult: + def claim_size(context: KernelContext) -> KernelResult: + assert context.tolerances == {("household", "size"): claim_expected} household = context.tables["household"] - assert set(household) == {"household_id", "is_clone"} return KernelResult( columns={ - ("household", "is_clone"): pd.Series( - household["is_clone"].array.copy(), + ("household", "size"): pd.Series( + household["size"].array.copy(), index=pd.Index(household["household_id"], name="household_id"), - dtype="bool", + dtype="int64", ) } ) - clone = Node( - "clone", - "expand@1", + def report_tolerance(context: KernelContext) -> KernelResult: + assert context.tolerances == {("household", "size"): reader_expected} + tolerance_payload = { + "rtol": reader_expected.rtol, + "atol": reader_expected.atol, + "ulps": reader_expected.ulps, + } + household = context.tables["household"] + return KernelResult( + columns={ + ("household", "tolerance_verdict"): pd.Series( + ["pass"] * len(household), + index=pd.Index(household["household_id"], name="household_id"), + dtype="string", + ) + }, + receipt={ + "outcome": "pass", + "evidence": {"tolerance": tolerance_payload}, + }, + ) + + create = replace(CREATE, kernel="writer.source@1") + expand = Node( + "admit_household", + "writer.expand@1", structural=StructuralDelta.EXPAND, - base="survey", + base=create.id, params={ - "expand_cells": (("household", "is_clone", "bool"),), + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("household", "size", "int64"), + ), "expand_weight_entity": "household", - "expand_weight_kind": "importance", + "expand_weight_kind": "design", }, - mass="conserve", - ) - claim_clone = Node( - "claim_clone", - "claim@1", - outputs=(Owned("household", "is_clone", "bool"),), - params={"materialized_expand_outputs": ("household.is_clone",)}, - population=clone.id, + mass="free", + entrants=True, ) - expand_kernel = _Kernel( - clone.kernel, - Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND), - expand, + claim = Node( + "claim_size", + "writer.claim@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=expand.id, ) - claim_kernel = _Kernel( - claim_clone.kernel, - Capabilities(Determinism.DETERMINISTIC), - claim, + reader = Node( + "report_tolerance", + "writer.gate@1", + inputs=(Slice("household", ("size",)),), + outputs=(Owned("household", "tolerance_verdict", "string"),), + population=expand.id, ) registry = _registry() - registry.register(expand_kernel) - registry.register(claim_kernel) - graph = Graph("toy", (SOURCE,), (CREATE, clone, claim_clone)) - store = ContentStore(tmp_path / "store") - + for kernel in ( + _Kernel( + create.kernel, + capabilities( + structural=StructuralDelta.CREATE, + tolerance=producer_tolerance, + ), + _source, + ), + _Kernel( + expand.kernel, + capabilities( + structural=StructuralDelta.EXPAND, + tolerance=expand_tolerance, + ), + expand_household, + ), + _Kernel( + claim.kernel, + capabilities(tolerance=claim_tolerance), + claim_size, + ), + _Kernel( + reader.kernel, + capabilities(role=KernelRole.GATE), + report_tolerance, + ), + ): + registry.register(kernel) + + manifest = _run( + Graph("toy", (SOURCE,), (create, expand, claim, reader)), + source, + ContentStore(tmp_path / "store"), + registry, + ) + document = json.loads(manifest.to_json()) + expand_receipt = document["nodes"][expand.id]["receipt"] + assert expand_receipt["expand_writes"]["household.size"] == ["entrant"] + claim_receipt = document["nodes"][claim.id]["receipt"] + assert claim_receipt["capabilities"]["tolerance_writers"] == { + "household.size": [create.id, expand.id] + } + reader_receipt = document["nodes"][reader.id]["receipt"] + assert reader_receipt["evidence"]["tolerance"] == { + "rtol": reader_expected.rtol, + "atol": reader_expected.atol, + "ulps": reader_expected.ulps, + } + assert reader_receipt["evidence"]["tolerance_writers"] == { + "household.size": [create.id, expand.id, claim.id] + } + + +def test_expand_lineage_receipt_and_materialized_cell_survive_cache( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + + def expand(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1, 2], + index=pd.Index([4, 5], name="person_id"), + dtype="int64", + ), + "household": pd.Series( + [10], + index=pd.Index([30], name="household_id"), + dtype="int64", + ), + }, + columns={ + ("household", "is_clone"): pd.Series( + [False, False, True], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="bool", + ) + }, + weights=Weights( + np.array([0.5, 2.0, 0.5], dtype=np.float64), + WeightKind.IMPORTANCE, + ), + ) + + def claim(context: KernelContext) -> KernelResult: + assert context.tolerances == {("household", "is_clone"): Tolerance(atol=9e-6)} + household = context.tables["household"] + assert set(household) == {"household_id", "is_clone"} + return KernelResult( + columns={ + ("household", "is_clone"): pd.Series( + household["is_clone"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="bool", + ) + } + ) + + clone = Node( + "clone", + "expand@1", + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": (("household", "is_clone", "bool"),), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + mass="free", + entrants=True, + ) + claim_clone = Node( + "claim_clone", + "claim@1", + outputs=(Owned("household", "is_clone", "bool"),), + params={"materialized_expand_outputs": ("household.is_clone",)}, + population=clone.id, + ) + expand_kernel = _Kernel( + clone.kernel, + Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + structural=StructuralDelta.EXPAND, + tolerance=Tolerance(atol=9e-6), + ), + expand, + ) + claim_kernel = _Kernel( + claim_clone.kernel, + Capabilities(Determinism.DETERMINISTIC), + claim, + ) + registry = _registry() + registry.register(expand_kernel) + registry.register(claim_kernel) + graph = Graph("toy", (SOURCE,), (CREATE, clone, claim_clone)) + store = ContentStore(tmp_path / "store") + cold = _run(graph, source, store, registry, resume="forbid") + assert cold.nodes[clone.id].receipt["expand_writes"] == { + "household.is_clone": ("new-column",) + } assert dict(cold.nodes[clone.id].receipt["expand"]) == { "household": ((30, 10),), "person": ((4, 1), (5, 2)), } + assert cold.nodes[claim_clone.id].receipt["capabilities"]["tolerance_writers"] == { + "household.is_clone": (clone.id,) + } assert cold.population(clone.id).table("person")[ "person_household_id" ].tolist() == [10, 10, 20, 30, 30] @@ -874,19 +1489,809 @@ def claim(context: KernelContext) -> KernelResult: ] assert cold.mass_ledger(clone.id)[-1].operation == "expand" - warm = _run(graph, source, store, registry) - assert warm.nodes[clone.id].hit - assert warm.nodes[claim_clone.id].hit - assert ( - warm.nodes[clone.id].receipt["expand"] == cold.nodes[clone.id].receipt["expand"] + warm = _run(graph, source, store, registry) + assert warm.nodes[clone.id].hit + assert warm.nodes[claim_clone.id].hit + assert ( + warm.nodes[clone.id].receipt["expand"] == cold.nodes[clone.id].receipt["expand"] + ) + assert ( + warm.population(clone.id) + .table("person") + .equals(cold.population(clone.id).table("person")) + ) + assert expand_kernel.calls == 1 + assert claim_kernel.calls == 1 + + +@pytest.mark.parametrize( + "coordinate", + [("house.hold", "new_value"), ("household", "new.value")], + ids=["entity", "column"], +) +def test_dotted_expand_cell_is_rejected_before_cold_or_warm_execution( + tmp_path: Path, + coordinate: tuple[str, str], +) -> None: + source = _source_path(tmp_path / "source") + entity, column = coordinate + + def expand(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1, 2], + index=pd.Index([4, 5], name="person_id"), + dtype="int64", + ), + "household": pd.Series( + [10], + index=pd.Index([30], name="household_id"), + dtype="int64", + ), + }, + columns={ + coordinate: pd.Series( + [False, False, True], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="bool", + ) + }, + weights=Weights( + np.array([0.5, 2.0, 0.5], dtype=np.float64), + WeightKind.IMPORTANCE, + ), + ) + + clone = Node( + "dotted_clone", + "expand.dotted@1", + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": ((entity, column, "bool"),), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + mass="free", + entrants=True, + ) + expand_kernel = _Kernel( + clone.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.EXPAND, + ), + expand, + ) + registry = _registry() + registry.register(expand_kernel) + graph = Graph("toy", (SOURCE,), (CREATE, clone)) + store = ContentStore(tmp_path / "store") + + for resume in ("forbid", "auto"): + with pytest.raises(NodeRejected, match=r"expand_cells.*dot-free"): + _run(graph, source, store, registry, resume=resume) + assert expand_kernel.calls == 0 + assert _object_bytes(store) == {} + + +def test_materialized_expand_claim_rejects_filter_population(tmp_path: Path) -> None: + source = _source_path(tmp_path / "source") + + def keep_all(context: KernelContext) -> KernelResult: + person = context.tables["person"] + return KernelResult( + keep=pd.Series(True, index=person["person_id"], dtype="bool") + ) + + def claim_size(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + columns={ + ("household", "size"): pd.Series( + household["size"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + } + ) + + filtered = Node( + "filtered", + "filter.claim-boundary@1", + inputs=(Slice("person", ("selected",)),), + structural=StructuralDelta.FILTER, + base="survey", + mass="free", + ) + claimant = Node( + "claim_filtered_size", + "claim.filtered-size@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=filtered.id, + ) + filter_kernel = _Kernel( + filtered.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + ), + keep_all, + ) + claim_kernel = _Kernel( + claimant.kernel, + Capabilities(Determinism.DETERMINISTIC), + claim_size, + ) + registry = _registry() + registry.register(filter_kernel) + registry.register(claim_kernel) + graph = Graph("toy", (SOURCE,), (CREATE, filtered, claimant)) + store = ContentStore(tmp_path / "store") + + for _ in range(2): + with pytest.raises( + NodeRejected, + match="claim_filtered_size.*filtered.*FILTER", + ): + _run(graph, source, store, registry) + assert filter_kernel.calls == 1 + assert claim_kernel.calls == 0 + + +def test_materialized_expand_claim_rejects_non_materialized_coordinate( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + + def copy_household(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1, 2], + index=pd.Index([4, 5], name="person_id"), + dtype="int64", + ), + "household": pd.Series( + [10], + index=pd.Index([30], name="household_id"), + dtype="int64", + ), + }, + weights=Weights( + np.array([0.5, 2.0, 0.5], dtype=np.float64), + WeightKind.IMPORTANCE, + ), + ) + + def claim_size(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + columns={ + ("household", "size"): pd.Series( + household["size"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + } + ) + + clone = Node( + "clone", + "expand.unmaterialized@1", + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": (), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + mass="free", + ) + claimant = Node( + "claim_unmaterialized_size", + "claim.unmaterialized-size@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=clone.id, + ) + expand_kernel = _Kernel( + clone.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.EXPAND, + ), + copy_household, + ) + claim_kernel = _Kernel( + claimant.kernel, + Capabilities(Determinism.DETERMINISTIC), + claim_size, + ) + registry = _registry() + registry.register(expand_kernel) + registry.register(claim_kernel) + graph = Graph("toy", (SOURCE,), (CREATE, clone, claimant)) + store = ContentStore(tmp_path / "store") + + for _ in range(2): + with pytest.raises( + NodeRejected, + match=r"claim_unmaterialized_size.*household\.size.*clone", + ): + _run(graph, source, store, registry) + assert expand_kernel.calls == 1 + assert claim_kernel.calls == 0 + + +@pytest.mark.parametrize("lineage_dtype", ["int64", "Int64"]) +def test_entrant_expand_with_zero_entrants_allows_declared_materialized_claim( + tmp_path: Path, lineage_dtype: str +) -> None: + """A declared-entrants node may carry a nullable lineage dtype even when it + admits no entrant on this run.""" + + source = _source_path(tmp_path / "source") + + def admit_no_entrants(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + expand={ + "person": pd.Series( + [], + index=pd.Index([], name="person_id", dtype="int64"), + dtype=lineage_dtype, + ), + "household": pd.Series( + [], + index=pd.Index([], name="household_id", dtype="int64"), + dtype=lineage_dtype, + ), + }, + columns={ + ("household", "size"): pd.Series( + household["size"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + }, + weights=context.weights["household"], + ) + + def claim_size(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + columns={ + ("household", "size"): pd.Series( + household["size"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + } + ) + + expand = Node( + "zero_entrant_expand", + "expand.zero-entrants@1", + inputs=(Slice("household", ("size",)),), + structural=StructuralDelta.EXPAND, + base=CREATE.id, + params={ + "expand_cells": (("household", "size", "int64"),), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + claimant = Node( + "claim_zero_entrant_size", + "claim.zero-entrant-size@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=expand.id, + ) + expand_kernel = _Kernel( + expand.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.EXPAND, + ), + admit_no_entrants, + ) + claim_kernel = _Kernel( + claimant.kernel, + Capabilities(Determinism.DETERMINISTIC), + claim_size, + ) + registry = _registry() + registry.register(expand_kernel) + registry.register(claim_kernel) + graph = Graph("toy", (SOURCE,), (CREATE, expand, claimant)) + store = ContentStore(tmp_path / "store") + + cold = _run(graph, source, store, registry) + warm = _run(graph, source, store, registry) + + for manifest in (cold, warm): + assert manifest.nodes[expand.id].receipt["expand_declared"] == ( + "household.size", + ) + assert manifest.nodes[expand.id].receipt["expand_writes"] == {} + assert manifest.nodes[claimant.id].receipt["capabilities"][ + "tolerance_writers" + ] == {"household.size": (CREATE.id,)} + assert not cold.nodes[expand.id].hit + assert not cold.nodes[claimant.id].hit + assert warm.nodes[expand.id].hit + assert warm.nodes[claimant.id].hit + assert expand_kernel.calls == 1 + assert claim_kernel.calls == 1 + + +def test_entrant_expand_rejects_mutated_copied_carried_values( + tmp_path: Path, +) -> None: + expand, claim = toy.entrant_person_node( + "mutated_copy", strata_mode="mutates_copied_income" + ) + graph = toy.small_graph(nodes=(toy.CREATE, expand, claim)) + sources = {"survey": toy.copy_source(tmp_path / "source")} + registry = toy.toy_registry() + store = ContentStore(tmp_path / "store") + source_person = toy.read_toy_frame(sources["survey"]).table("person") + source_id = int(source_person["person_id"].iloc[0]) + copied_id = int(source_person["person_id"].max()) + 2 + + for attempt in range(2): + with pytest.raises( + NodeRejected, match=r"mutated_copy.*person\.income" + ) as error: + toy.run_toy( + graph, + tmp_path / f"attempt-{attempt}", + sources=sources, + registry=registry, + store=store, + ) + assert f"({copied_id}, {source_id})" in str(error.value) + + calls = toy.calls_by_ref(registry) + assert calls["source.csv@1"] == 1 + assert calls[expand.kernel] == 2 + assert calls[claim.kernel] == 0 + + +def test_expand_allows_copied_value_declared_as_same_version_rewrite( + tmp_path: Path, +) -> None: + expand_tolerance = Tolerance(rtol=7e-6) + + def copy_with_rewritten_income(context: KernelContext) -> KernelResult: + person = context.tables["person"] + person_ids = pd.Index(person["person_id"], name="person_id") + source_id = int(person_ids[0]) + copy_id = int(person_ids.max()) + 1 + target_ids = person_ids.append( + pd.Index([copy_id], dtype="int64", name="person_id") + ) + income = pd.concat( + [ + person["income"].reset_index(drop=True), + pd.Series([float(person["income"].iloc[0]) + 1.0]), + ], + ignore_index=True, + ) + return KernelResult( + expand={ + "person": pd.Series( + [source_id], + index=pd.Index([copy_id], dtype="int64", name="person_id"), + dtype="int64", + ), + "household": pd.Series( + [], + index=pd.Index([], dtype="int64", name="household_id"), + dtype="int64", + ), + }, + columns={ + ("person", "income"): pd.Series( + income.array, index=target_ids, dtype="float64" + ) + }, + weights=context.weights["household"], + ) + + def claim_rewritten_income(context: KernelContext) -> KernelResult: + assert context.tolerances == {("person", "income"): expand_tolerance} + person = context.tables["person"] + return KernelResult( + columns={ + ("person", "income"): pd.Series( + person["income"].array.copy(), + index=pd.Index(person["person_id"], name="person_id"), + dtype="float64", + ) + } + ) + + expand = Node( + "copy_rewritten_income", + "copy.rewritten_income@1", + inputs=( + Slice("person", ("income",)), + Slice("household", ("size",)), + ), + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": (("person", "income", "float64"),), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + ) + claim = Node( + "claim_rewritten_income", + "claim.rewritten_income@1", + outputs=(Owned("person", "income", "float64", rewrite=True),), + population=expand.id, + ) + registry = _registry() + registry.register( + _Kernel( + expand.kernel, + Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + structural=StructuralDelta.EXPAND, + tolerance=expand_tolerance, + ), + copy_with_rewritten_income, + ) + ) + registry.register( + _Kernel( + claim.kernel, + Capabilities(Determinism.DETERMINISTIC), + claim_rewritten_income, + ) + ) + graph = Graph("toy", (SOURCE,), (CREATE, expand, claim)) + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + + cold = _run(graph, source, store, registry, resume="forbid") + warm = _run(graph, source, store, registry) + + for run in (cold, warm): + assert run.population(expand.id).table("person")["income"].iloc[-1] == 1.0 + assert run.nodes[expand.id].receipt["expand_writes"] == { + "person.income": ("copied-rewrite",) + } + assert run.nodes[claim.id].receipt["capabilities"]["tolerance_writers"] == { + "person.income": (CREATE.id, expand.id) + } + assert not cold.nodes[expand.id].hit + assert warm.nodes[expand.id].hit + assert warm.nodes[claim.id].hit + + +def test_group_entrant_expand_manifest_json_carries_mass_record( + tmp_path: Path, +) -> None: + expand, claim = toy.entrant_expand_node() + graph = toy.small_graph(nodes=(toy.CREATE, expand, claim)) + sources = {"survey": toy.copy_source(tmp_path / "source")} + registry = toy.toy_registry() + store = ContentStore(tmp_path / "store") + cold = toy.run_toy( + graph, + tmp_path / "cold", + sources=sources, + registry=registry, + store=store, + ) + warm = toy.run_toy( + graph, + tmp_path / "warm", + sources=sources, + registry=registry, + store=store, + ) + + for run in (cold, warm): + ledger = run.manifest.mass_ledger(expand.id)[-1] + document = json.loads(run.manifest.to_json()) + mass = document["nodes"][expand.id]["receipt"]["mass"] + assert mass == { + "policy": ledger.policy, + "before": ledger.before_total, + "after": ledger.after_total, + "stratum_before": { + str(key): value for key, value in ledger.before_by_stratum + }, + "stratum_after": { + str(key): value for key, value in ledger.after_by_stratum + }, + } + assert mass["after"] - mass["before"] == 125.0 + assert warm.manifest.nodes[expand.id].hit + + +def test_expand_id_overlay_is_rejected_without_committing_cache(tmp_path: Path) -> None: + source = _source_path(tmp_path / "source") + + def replace_lineage_id(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1], index=pd.Index([4], name="person_id"), dtype="int64" + ), + "household": pd.Series( + [10], index=pd.Index([30], name="household_id"), dtype="int64" + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 40], + index=pd.Index([1, 2, 3, 4], name="person_id"), + dtype="int64", + ), + ("household", "household_id"): pd.Series( + [10, 20, 40], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="int64", + ), + }, + weights=Weights( + np.array([1.0, 2.0, 1.0], dtype=np.float64), WeightKind.DESIGN + ), + ) + + expand = Node( + "replace_lineage_id", + "bad.expand@1", + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("household", "household_id", "int64"), + ), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", ) - assert ( - warm.population(clone.id) - .table("person") - .equals(cold.population(clone.id).table("person")) + kernel = _Kernel( + expand.kernel, + Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND), + replace_lineage_id, ) - assert expand_kernel.calls == 1 - assert claim_kernel.calls == 1 + registry = _registry(extra=kernel) + graph = Graph("toy", (SOURCE,), (CREATE, expand)) + store = ContentStore(tmp_path / "store") + + for _ in range(2): + with pytest.raises(NodeRejected, match="cannot overlay entity id column"): + _run(graph, source, store, registry) + if kernel.calls == 1: + first_store_bytes = _object_bytes(store) + else: + assert _object_bytes(store) == first_store_bytes + assert kernel.calls == 2 + + +def test_partitioned_graph_accepts_structural_entrant_partition_values( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + + def create_partitioned(context: KernelContext) -> KernelResult: + original = _source_frame(context.sources["survey"]) + tables = {entity: original.table(entity).copy() for entity in original.entities} + tables["household"]["period"] = np.array([2024, 2025], dtype=np.int64) + return KernelResult( + frame=Frame( + tables, + original.schema, + {"household": original.weights_for("household")}, + original.strata, + ) + ) + + def admit_household(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1], index=pd.Index([4], name="person_id"), dtype="int64" + ), + "household": pd.Series( + [pd.NA], + index=pd.Index([30], name="household_id"), + dtype="Int64", + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 30], + index=pd.Index([1, 2, 3, 4], name="person_id"), + dtype="int64", + ), + ("household", "size"): pd.Series( + [2, 1, 1], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="int64", + ), + ("household", "period"): pd.Series( + [2024, 2025, 2026], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="int64", + ), + }, + weights=Weights( + np.array([1.0, 2.0, 1.0], dtype=np.float64), WeightKind.DESIGN + ), + ) + + def pass_through(column: str) -> Callable[[KernelContext], KernelResult]: + def run(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + columns={ + ("household", column): pd.Series( + household[column].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + } + ) + + return run + + partition_tolerance = Tolerance(atol=4e-6) + + def audit_period(context: KernelContext) -> KernelResult: + assert context.tolerances == {("household", "period"): partition_tolerance} + return KernelResult( + receipt={ + "outcome": "pass", + "evidence": {"tolerance": {"rtol": 0.0, "atol": 4e-6, "ulps": 0}}, + } + ) + + create = replace( + CREATE, + kernel="partition.source@1", + outputs=(*CREATE.outputs, Owned("household", "period", "int64")), + ) + expand = Node( + "admit_household", + "partition.expand@1", + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("household", "size", "int64"), + ("household", "period", "int64"), + ), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + claim_size = Node( + "claim_size", + "claim.size@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=expand.id, + ) + period_gate = Node( + "audit_period", + "gate.partition-tolerance@1", + inputs=(Slice("household", ("period",)),), + population=expand.id, + ) + kernels = ( + _Kernel( + create.kernel, + Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE), + create_partitioned, + ), + _Kernel( + expand.kernel, + Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + structural=StructuralDelta.EXPAND, + tolerance=partition_tolerance, + ), + admit_household, + ), + _Kernel( + claim_size.kernel, + Capabilities(Determinism.DETERMINISTIC), + pass_through("size"), + ), + _Kernel( + period_gate.kernel, + Capabilities( + Determinism.DETERMINISTIC, + role=KernelRole.GATE, + ), + audit_period, + ), + ) + + def registry(*extra: _Kernel) -> KernelRegistry: + result = _registry() + for kernel in (*kernels, *extra): + result.register(kernel) + return result + + graph = Graph( + "toy", + (SOURCE,), + (create, expand, claim_size, period_gate), + mass_partition=("household", "period"), + ) + store = ContentStore(tmp_path / "store") + first_registry = registry() + cold = _run(graph, source, store, first_registry) + warm = _run(graph, source, store, first_registry) + + for manifest in (cold, warm): + assert manifest.population(expand.id).table("household")["period"].tolist() == [ + 2024, + 2025, + 2026, + ] + partition = manifest.nodes[expand.id].receipt["mass"]["partition"] # type: ignore[index] + assert (partition["entity"], partition["column"]) == ("household", "period") + document = json.loads(manifest.to_json()) + period_receipt = document["nodes"][period_gate.id]["receipt"] + assert period_receipt["capabilities"]["tolerance_writers"] == { + "household.period": [create.id, expand.id] + } + assert period_receipt["evidence"]["tolerance_writers"] == { + "household.period": [create.id, expand.id] + } + assert warm.nodes[expand.id].hit + assert warm.nodes[claim_size.id].hit + assert warm.nodes[period_gate.id].hit + + claim_period = Node( + "claim_period", + "claim.period@1", + outputs=(Owned("household", "period", "int64"),), + params={"materialized_expand_outputs": ("household.period",)}, + population=expand.id, + ) + period_kernel = _Kernel( + claim_period.kernel, + Capabilities(Determinism.DETERMINISTIC), + pass_through("period"), + ) + # Compilation refuses the owner before the executor's own guard can. + with pytest.raises(GraphError, match="owns mass partition"): + _run( + Graph( + "toy", + (SOURCE,), + (create, expand, claim_size, claim_period), + mass_partition=("household", "period"), + ), + source, + ContentStore(tmp_path / "ordinary-owner"), + registry(period_kernel), + ) def test_create_rejects_undeclared_frame_columns(tmp_path: Path) -> None: @@ -937,51 +2342,294 @@ def malformed(context: KernelContext) -> KernelResult: Graph("toy", (SOURCE,), (CREATE, node)), source, ContentStore(tmp_path / "store"), - _registry(extra=kernel), + _registry(extra=kernel), + ) + + +def test_corrupt_cache_and_unavailable_codec_abort_before_recompute( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + manifest = _run(_graph(), source, store, _registry()) + artifact = manifest.nodes["a"].artifacts[("person", "a")] + payload = store.object_path(artifact) / "values.npy" + damaged = bytearray(payload.read_bytes()) + damaged[-1] ^= 1 + payload.write_bytes(damaged) + + registry = _registry() + with pytest.raises(StoreCorrupt): + _run(_graph(), source, store, registry) + assert sum(_calls(registry).values()) == 0 + + missing_codec_graph = Graph( + "toy", + (SourceRef("survey", "missing-codec"),), + (CREATE,), + ) + missing_registry = _registry() + with pytest.raises(StoreUnavailable): + _run( + missing_codec_graph, + source, + ContentStore(tmp_path / "unavailable"), + missing_registry, + ) + assert sum(_calls(missing_registry).values()) == 0 + + isolated_registry = _registry() + with pytest.raises(StoreUnavailable, match="csv-tables"): + _run( + _graph(), + source, + ContentStore(tmp_path / "isolated", codecs={}), + isolated_registry, + ) + assert sum(_calls(isolated_registry).values()) == 0 + + +def test_entrants_need_a_design_anchor_after_a_reweight(tmp_path: Path) -> None: + """B6 / amendment 11: an entrant is admitted only while weights are design.""" + + source = _source_path(tmp_path / "source") + + def reweight(context: KernelContext) -> KernelResult: + before = context.weights["household"].values + return KernelResult( + weights=Weights(before * 2, WeightKind.IMPORTANCE), + receipt={ + "mass": { + "policy": "free", + "before": 4.0, + "after": 8.0, + "stratum_before": {"a": 2.0, "b": 2.0}, + "stratum_after": {"a": 4.0, "b": 4.0}, + } + }, + ) + + def admit(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1], index=pd.Index([4], name="person_id"), dtype="int64" + ), + "household": pd.Series( + [pd.NA], index=pd.Index([30], name="household_id"), dtype="Int64" + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 30], + index=pd.Index([1, 2, 3, 4], name="person_id"), + dtype="int64", + ), + ("household", "size"): pd.Series( + [2, 1, 1], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="int64", + ), + }, + weights=Weights( + np.array([2.0, 4.0, 1.0], dtype=np.float64), WeightKind.IMPORTANCE + ), + ) + + pool = Node( + "pool", + "reweight@1", + structural=StructuralDelta.REWEIGHT, + base="survey", + inputs=(Slice("household", ("size",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + ) + entrant = Node( + "admit_household", + "admit@1", + structural=StructuralDelta.EXPAND, + base=pool.id, + params={ + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("household", "size", "int64"), + ), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + mass="free", + entrants=True, + ) + registry = _registry( + extra=_Kernel( + pool.kernel, + Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ), + reweight, + ) + ) + registry.register( + _Kernel( + entrant.kernel, + Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND), + admit, + ) + ) + + def claim_size(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + columns={ + ("household", "size"): pd.Series( + household["size"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + } + ) + + claim = Node( + "claim_size", + "claim@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=entrant.id, + ) + registry.register( + _Kernel(claim.kernel, Capabilities(Determinism.DETERMINISTIC), claim_size) + ) + with pytest.raises(NodeRejected, match=r"cannot anchor new 'household' ids"): + _run( + Graph("toy", (SOURCE,), (CREATE, pool, entrant, claim)), + source, + ContentStore(tmp_path / "store"), + registry, ) -def test_corrupt_cache_and_unavailable_codec_abort_before_recompute( - tmp_path: Path, -) -> None: +def test_expand_cannot_regress_the_base_weight_kind(tmp_path: Path) -> None: + """B6 / D1: weight kinds only move forward; declaring design on a reweighted base is refused.""" + source = _source_path(tmp_path / "source") - store = ContentStore(tmp_path / "store") - manifest = _run(_graph(), source, store, _registry()) - artifact = manifest.nodes["a"].artifacts[("person", "a")] - payload = store.object_path(artifact) / "values.npy" - damaged = bytearray(payload.read_bytes()) - damaged[-1] ^= 1 - payload.write_bytes(damaged) - registry = _registry() - with pytest.raises(StoreCorrupt): - _run(_graph(), source, store, registry) - assert sum(_calls(registry).values()) == 0 + def reweight(context: KernelContext) -> KernelResult: + before = context.weights["household"].values + return KernelResult( + weights=Weights(before * 2, WeightKind.IMPORTANCE), + receipt={ + "mass": { + "policy": "free", + "before": 4.0, + "after": 8.0, + "stratum_before": {"a": 2.0, "b": 2.0}, + "stratum_after": {"a": 4.0, "b": 4.0}, + } + }, + ) - missing_codec_graph = Graph( - "toy", - (SourceRef("survey", "missing-codec"),), - (CREATE,), + def admit(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1], index=pd.Index([4], name="person_id"), dtype="int64" + ), + "household": pd.Series( + [pd.NA], index=pd.Index([30], name="household_id"), dtype="Int64" + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 30], + index=pd.Index([1, 2, 3, 4], name="person_id"), + dtype="int64", + ), + ("household", "size"): pd.Series( + [2, 1, 1], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="int64", + ), + }, + weights=Weights( + np.array([1.0, 2.0, 1.0], dtype=np.float64), WeightKind.DESIGN + ), + ) + + pool = Node( + "pool", + "reweight@1", + structural=StructuralDelta.REWEIGHT, + base="survey", + inputs=(Slice("household", ("size",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", ) - missing_registry = _registry() - with pytest.raises(StoreUnavailable): - _run( - missing_codec_graph, - source, - ContentStore(tmp_path / "unavailable"), - missing_registry, + entrant = Node( + "admit_household", + "admit@1", + structural=StructuralDelta.EXPAND, + base=pool.id, + params={ + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("household", "size", "int64"), + ), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + registry = _registry( + extra=_Kernel( + pool.kernel, + Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ), + reweight, ) - assert sum(_calls(missing_registry).values()) == 0 + ) + registry.register( + _Kernel( + entrant.kernel, + Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND), + admit, + ) + ) - isolated_registry = _registry() - with pytest.raises(StoreUnavailable, match="csv-tables"): + def claim_size(context: KernelContext) -> KernelResult: + household = context.tables["household"] + return KernelResult( + columns={ + ("household", "size"): pd.Series( + household["size"].array.copy(), + index=pd.Index(household["household_id"], name="household_id"), + dtype="int64", + ) + } + ) + + claim = Node( + "claim_size", + "claim@1", + outputs=(Owned("household", "size", "int64"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=entrant.id, + ) + registry.register( + _Kernel(claim.kernel, Capabilities(Determinism.DETERMINISTIC), claim_size) + ) + with pytest.raises( + NodeRejected, + match=r"cannot regress 'household' weights from 'importance' to 'design'", + ): _run( - _graph(), + Graph("toy", (SOURCE,), (CREATE, pool, entrant, claim)), source, - ContentStore(tmp_path / "isolated", codecs={}), - isolated_registry, + ContentStore(tmp_path / "store"), + registry, ) - assert sum(_calls(isolated_registry).values()) == 0 def test_structural_reweight_uses_explicit_kind_and_mass_receipt( @@ -1226,6 +2874,102 @@ def test_release_decision_changes_only_the_manifest_outcome(tmp_path: Path) -> N assert signed.nodes["release"].key == missing.nodes["release"].key assert signed.nodes["release"].artifacts == missing.nodes["release"].artifacts assert signed.nodes["release"].receipt["tier"] == "certified" + assert signed.nodes["release"].receipt["requires_decisions"] == ("publish",) + + +def test_certified_loader_revalidates_authenticated_required_decisions( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + graph = _release_graph( + gate_outcome="pass", + tier_answer="certified", + requires=("publish",), + ) + signed = _run( + graph, + source, + store, + _release_registry(), + decisions=({"name": "publish", "owner": "reviewer", "signature": "signed"},), + ) + path = tmp_path / "certified.json" + signed.save(path) + document = json.loads(path.read_text()) + authenticated_body = document["content_addressed"] + authenticated_key = document["key"] + document["decisions"] = [] + assert document["content_addressed"] == authenticated_body + assert document["key"] == authenticated_key + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(NodeRejected, match=r"unreached.*publish"): + RunManifest.load_certified(path, store) + + +@pytest.mark.parametrize( + "replacement", + [ + {"name": "publish", "owner": "", "signature": "signed"}, + {"name": "publish", "owner": "reviewer", "signature": ""}, + { + "owner": "", + "kind": "publish", + "text": "approved", + "signed_at": "2026-09-03", + }, + { + "owner": "reviewer", + "kind": "publish", + "text": "", + "signed_at": "2026-09-03", + }, + { + "owner": "reviewer", + "kind": "publish", + "text": "approved", + "signed_at": "", + }, + ], + ids=[ + "legacy-owner", + "legacy-signature", + "current-owner", + "current-text", + "current-signed-at", + ], +) +def test_certified_loader_revalidates_signed_decision_fields( + tmp_path: Path, + replacement: dict[str, str], +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + graph = _release_graph( + gate_outcome="pass", + tier_answer="certified", + requires=("publish",), + ) + signed = _run( + graph, + source, + store, + _release_registry(), + decisions=({"name": "publish", "owner": "reviewer", "signature": "signed"},), + ) + path = tmp_path / "invalid-signed-record.json" + signed.save(path) + document = json.loads(path.read_text()) + authenticated_body = document["content_addressed"] + authenticated_key = document["key"] + document["decisions"] = [replacement] + assert document["content_addressed"] == authenticated_body + assert document["key"] == authenticated_key + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(NodeRejected, match="non-empty signed decision"): + RunManifest.load_certified(path, store) def test_gate_outcome_is_closed_and_gate_exceptions_become_failures( @@ -1287,3 +3031,336 @@ def explode(context: KernelContext) -> KernelResult: ContentStore(tmp_path / "compute"), compute_registry, ) + + +def test_cache_misses_receipt_without_tolerance_writer_provenance( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + graph = _graph(leaf=False) + registry = _registry() + cold = _run(graph, source, store, registry) + + node = graph.node("a") + kernel = registry.get(node.kernel) + assert isinstance(kernel, _Kernel) + assert kernel.calls == 1 + key = cold.nodes[node.id].key + record_key = graph_executor._cache_record_key(key) + record = store.load_json(record_key) + raw_receipt = record["receipt"] + assert isinstance(raw_receipt, dict) + receipt_capabilities = raw_receipt["capabilities"] + assert isinstance(receipt_capabilities, dict) + assert receipt_capabilities.pop("tolerance_writers") == {"person.age": ["survey"]} + store.put_json(record_key, record, node_key=key, verify_existing=False) + + with pytest.raises(StoreMiss, match=r"cache misses.*'a'"): + _run(graph, source, store, registry, resume="require") + assert kernel.calls == 1 + + warm = _run(graph, source, store, registry) + assert not warm.nodes[node.id].hit + assert kernel.calls == 2 + repaired = store.load_json(record_key) + repaired_receipt = repaired["receipt"] + assert isinstance(repaired_receipt, dict) + repaired_capabilities = repaired_receipt["capabilities"] + assert isinstance(repaired_capabilities, dict) + assert repaired_capabilities["tolerance_writers"] == {"person.age": ["survey"]} + + +def test_cache_load_misses_when_stored_capabilities_disagree( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + graph = _graph() + registry = _registry() + manifest = _run(graph, source, store, registry) + + node = graph.node("a") + key = manifest.nodes["a"].key + record_key = graph_executor._cache_record_key(key) + record = store.load_json(record_key) + stored_capabilities = record["capabilities"] + assert isinstance(stored_capabilities, dict) + record["capabilities"] = { + **stored_capabilities, + "seed_source": "param", + } + store.put_json(record_key, record, node_key=key, verify_existing=False) + + kernel = registry.get(node.kernel) + with pytest.raises(StoreMiss, match="capabilities"): + graph_executor._load_record( + store, + node, + key=key, + kernel_impl_hash=kernel.implementation_hash(), + capabilities=kernel.capabilities, + ) + + +def test_cache_load_names_legacy_capabilities_without_tolerance( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + graph = _graph() + registry = _registry() + manifest = _run(graph, source, store, registry) + + node = graph.node("a") + key = manifest.nodes["a"].key + record_key = graph_executor._cache_record_key(key) + record = store.load_json(record_key) + stored_capabilities = record["capabilities"] + assert isinstance(stored_capabilities, dict) + del stored_capabilities["tolerance"] + store.put_json(record_key, record, node_key=key, verify_existing=False) + + kernel = registry.get(node.kernel) + with pytest.raises(StoreMiss, match=r"legacy_capabilities.*tolerance"): + graph_executor._load_record( + store, + node, + key=key, + kernel_impl_hash=kernel.implementation_hash(), + capabilities=kernel.capabilities, + ) + + +def test_fit_qrf_seed_source_change_misses_a_shared_store( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from microcosm.fit.kernels import QRF_EXECUTOR_KERNEL, QRF_PARAM_KERNEL + from microcosm.graph import graph_from_json + from tools.graph_parity_fixtures import FIXTURES, ParityCsvSource + + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + case = FIXTURES / "fit.qrf" + compiled = compile_graph(graph_from_json((case / "graph.json").read_text())) + store = ContentStore(tmp_path / "store") + + param_registry = KernelRegistry() + param_registry.register(ParityCsvSource()) + param_registry.register(QRF_PARAM_KERNEL) + cold = run_graph( + compiled, + sources={"fixture": case / "inputs.csv"}, + store=store, + kernels=param_registry, + ) + assert not cold.nodes["fit_qrf"].hit + + assert QRF_PARAM_KERNEL.ref == QRF_EXECUTOR_KERNEL.ref + assert ( + QRF_PARAM_KERNEL.implementation_hash() + == QRF_EXECUTOR_KERNEL.implementation_hash() + ) + assert ( + QRF_PARAM_KERNEL.capabilities.seed_source + is not QRF_EXECUTOR_KERNEL.capabilities.seed_source + ) + + executor_registry = KernelRegistry() + executor_registry.register(ParityCsvSource()) + executor_registry.register(QRF_EXECUTOR_KERNEL) + with pytest.raises(NodeRejected, match="EXECUTOR-seeded.*must omit"): + run_graph( + compiled, + sources={"fixture": case / "inputs.csv"}, + store=store, + kernels=executor_registry, + ) + + +def test_fit_qrf_model_artifact_is_canonical_across_runtime_worker_settings( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import pickle + + from microcosm.fit.kernels import QRF_PARAM_KERNEL + from microcosm.graph import graph_from_json + from tools.graph_parity_fixtures import FIXTURES, ParityCsvSource + + monkeypatch.setenv("POPULACE_FIT_PREDICT_WORKERS", "1") + case = FIXTURES / "fit.qrf" + compiled = compile_graph(graph_from_json((case / "graph.json").read_text())) + registry = KernelRegistry() + registry.register(ParityCsvSource()) + registry.register(QRF_PARAM_KERNEL) + + manifests = [] + stores = [] + for n_jobs in ("1", "2"): + monkeypatch.setenv("POPULACE_FIT_N_JOBS", n_jobs) + store = ContentStore(tmp_path / f"store-{n_jobs}") + stores.append(store) + manifests.append( + run_graph( + compiled, + sources={"fixture": case / "inputs.csv"}, + store=store, + kernels=registry, + resume="forbid", + ) + ) + + first_node = manifests[0].nodes["fit_qrf"] + second_node = manifests[1].nodes["fit_qrf"] + assert first_node.key == second_node.key + first_artifact = first_node.opaque_artifacts["model"] + second_artifact = second_node.opaque_artifacts["model"] + assert first_artifact == second_artifact + first_bytes = stores[0].load_bytes(first_artifact) + assert first_bytes == stores[1].load_bytes(second_artifact) + + monkeypatch.setenv("POPULACE_FIT_N_JOBS", "2") + loaded = pickle.loads(first_bytes) # noqa: S301 - trusted graph-store artifact + forests = [ + forest + for target in loaded._target_models.values() + for forest in (target.positive, target.negative) + if forest is not None + ] + assert forests + assert {forest.model.n_jobs for forest in forests} == {2} + + +def test_fit_qrf_tolerance_source_hash_pin_is_current() -> None: + import json + + from microcosm.fit.kernels import QRF_PARAM_KERNEL, QRFKernel + from tools.graph_parity_fixtures import FIXTURES + + pins = json.loads((FIXTURES / "fit.qrf" / "pins.json").read_text()) + # fit.qrf@1 is platform-bitwise (amendment 16): no tolerance, and the + # implementation identity must not depend on the capability declaration. + assert QRF_PARAM_KERNEL.capabilities.numeric is Numeric.PLATFORM_BITWISE + assert QRF_PARAM_KERNEL.capabilities.tolerance is None + assert pins["implementation_hash"] == QRF_PARAM_KERNEL.implementation_hash() + changed_tolerance = QRFKernel(QRF_PARAM_KERNEL.capabilities.seed_source) + changed_tolerance.capabilities = replace( + changed_tolerance.capabilities, + numeric=Numeric.TOLERANCE_BOUND, + tolerance=Tolerance(ulps=2), + ) + assert changed_tolerance.implementation_hash() == pins["implementation_hash"] + + +def test_entrant_materialization_rejects_a_masked_claimant( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + + def admit_household(context: KernelContext) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1], + index=pd.Index([4], name="person_id"), + dtype="int64", + ), + "household": pd.Series( + [pd.NA], + index=pd.Index([30], name="household_id"), + dtype="Int64", + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 30], + index=pd.Index([1, 2, 3, 4], name="person_id"), + dtype="int64", + ), + ("household", "size"): pd.Series( + [2, 1, 1], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="int64", + ), + ("household", "claim_mask"): pd.Series( + [True, True, False], + index=pd.Index([10, 20, 30], name="household_id"), + dtype="boolean", + ), + }, + weights=Weights( + np.array([1.0, 2.0, 1.0], dtype=np.float64), + WeightKind.DESIGN, + ), + ) + + expand = Node( + "admit_household", + "masked.expand@1", + structural=StructuralDelta.EXPAND, + base="survey", + params={ + "expand_cells": ( + ("person", "person_household_id", "int64"), + ("household", "size", "int64"), + ("household", "claim_mask", "boolean"), + ), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + claim_mask = Node( + "claim_mask", + "claim.mask@1", + outputs=(Owned("household", "claim_mask", "boolean"),), + params={"materialized_expand_outputs": ("household.claim_mask",)}, + population=expand.id, + ) + claim_size = Node( + "claim_size", + "claim.masked-size@1", + inputs=(Slice("household", ("claim_mask",)),), + outputs=(Owned("household", "size", "int64", rows="claim_mask"),), + params={"materialized_expand_outputs": ("household.size",)}, + population=expand.id, + ) + + def must_not_run(context: KernelContext) -> KernelResult: + raise AssertionError(f"claimant {context.node.id} should not run") + + registry = _registry() + registry.register( + _Kernel( + expand.kernel, + Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.EXPAND, + ), + admit_household, + ) + ) + mask_kernel = _Kernel( + claim_mask.kernel, + Capabilities(Determinism.DETERMINISTIC), + must_not_run, + ) + size_kernel = _Kernel( + claim_size.kernel, + Capabilities(Determinism.DETERMINISTIC), + must_not_run, + ) + registry.register(mask_kernel) + registry.register(size_kernel) + + with pytest.raises(NodeRejected, match="household.size.*rows='all'"): + _run( + Graph("toy", (SOURCE,), (CREATE, expand, claim_mask, claim_size)), + source, + ContentStore(tmp_path / "store"), + registry, + ) + assert mask_kernel.calls == size_kernel.calls == 0 diff --git a/packages/microcosm-graph/tests/test_graph_explain.py b/packages/microcosm-graph/tests/test_graph_explain.py index 62d9b6aa8..da74a5385 100644 --- a/packages/microcosm-graph/tests/test_graph_explain.py +++ b/packages/microcosm-graph/tests/test_graph_explain.py @@ -158,6 +158,36 @@ def test_explain_html_is_the_public_export() -> None: assert graph_api.explain_html is explain_html +def test_entrant_person_strata_survive_cache_and_are_explained(tmp_path: Path) -> None: + expand, claim = toy.entrant_person_node() + graph = toy.small_graph(nodes=(toy.CREATE, expand, claim)) + cold = toy.run_toy(graph, tmp_path / "cold") + warm = toy.run_toy( + graph, + tmp_path / "warm", + sources=cold.sources, + registry=cold.registry, + store=cold.store, + ) + entrant_id = int(cold.manifest.population("survey").person["person_id"].max()) + 1 + + assert warm.manifest.nodes[expand.id].hit + assert warm.manifest.population(expand.id).strata.equals( + cold.manifest.population(expand.id).strata + ) + assert cold.manifest.nodes[expand.id].receipt["entrant_strata"] == ( + (entrant_id, "urban"), + ) + assert ( + warm.manifest.nodes[expand.id].receipt["entrant_strata"] + == cold.manifest.nodes[expand.id].receipt["entrant_strata"] + ) + detail = describe(cold.compiled, expand.id, cold.manifest) + rendered = explain_html(cold.compiled, cold.manifest) + assert "entrant_strata" in detail and "urban" in detail + assert "entrant_strata" in rendered and "urban" in rendered + + def test_page_contains_every_node_and_its_click_detail(explanation) -> None: run, _charter, rendered = explanation for node_id in run.compiled.order: @@ -172,11 +202,13 @@ def test_page_contains_every_node_and_its_click_detail(explanation) -> None: def test_page_contains_every_charter_property(explanation) -> None: _run, charter, rendered = explanation identifiers = re.findall(r"^\|\s*([A-Z]\d+)\s*\|", charter, re.MULTILINE) - assert len(dict.fromkeys(identifiers)) == 41 + assert ( + len(dict.fromkeys(identifiers)) == 45 + ) # 41 + B6, C5, D6, B7 (amendments 11-14) for identifier in identifiers: assert f"{identifier}" in rendered assert "35 green" not in rendered # V1-V4 are also represented. - assert "41 green" in rendered + assert "45 green" in rendered assert "0 red" in rendered assert "Flip PR" in rendered assert "Not recorded" in rendered @@ -208,6 +240,79 @@ def test_calibration_view_uses_targets_ratios_and_mass(explanation) -> None: assert "urban" in rendered +def test_calibration_view_renders_partition_mass_with_deltas(tmp_path: Path) -> None: + run = toy.run_toy(toy.full_graph(), tmp_path / "run") + original = run.manifest.nodes["calibrated"] + mass = dict(original.receipt["mass"]) + mass["stratum_before"] = {} + mass["stratum_after"] = {} + mass["partition"] = { + "entity": "household", + "column": "period", + "stratum_before": { + "2024": {"rural": 10.0, "urban": 20.0}, + "2025": {"rural": 5.0}, + }, + "stratum_after": { + "2024": {"rural": 8.0, "urban": 23.0}, + "2026": {"urban": 10.0}, + }, + } + changed = replace( + original, + receipt={**dict(original.receipt), "mass": mass}, + ) + manifest = replace( + run.manifest, + nodes={**dict(run.manifest.nodes), "calibrated": changed}, + ) + + rendered = explain_html(run.compiled, manifest) + + assert "Mass by household.period partition" in rendered + assert ( + "Partition valueStratumBeforeAfter" + "Change" in rendered + ) + assert ">2024rural108-2" in rendered + assert ">2024urban20233" in rendered + assert ">2025rural50-5" in rendered + assert ">2026urban01010" in rendered + assert "http://" not in rendered and "https://" not in rendered + + +def test_calibration_view_fallback_renders_partitioned_mass_record( + tmp_path: Path, +) -> None: + run = toy.run_toy(toy.full_graph(), tmp_path / "run") + original_receipt = run.manifest.nodes["calibrated"] + receipt_without_mass = dict(original_receipt.receipt) + receipt_without_mass.pop("mass") + changed_receipt = replace(original_receipt, receipt=receipt_without_mass) + original_record = next( + record + for record in run.manifest.mass_ledgers["calibrated"] + if record.node_id == "calibrated" + ) + partitioned_record = replace( + original_record, + partition_entity="household", + partition_column="period", + before_by_partition_stratum=((2024, (("rural", 10.0),)),), + after_by_partition_stratum=((2024, (("rural", 8.0),)),), + ) + manifest = replace( + run.manifest, + nodes={**dict(run.manifest.nodes), "calibrated": changed_receipt}, + mass_ledgers={"calibrated": (partitioned_record,)}, + ) + + rendered = explain_html(run.compiled, manifest) + + assert "Mass by household.period partition" in rendered + assert ">2024rural108-2" in rendered + + def test_calibration_view_reads_adam_diagnostics(tmp_path: Path) -> None: run = toy.run_toy(toy.full_graph(), tmp_path / "run") targets = ( diff --git a/packages/microcosm-graph/tests/test_graph_kernel_contract.py b/packages/microcosm-graph/tests/test_graph_kernel_contract.py new file mode 100644 index 000000000..057bec79f --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_kernel_contract.py @@ -0,0 +1,211 @@ +"""Kernel-protocol contracts of the frozen interface (amendment 13). + +A kernel that claims bounded numeric movement declares the bound; a bitwise +kernel declares none; the context hands readers their inputs' declared +tolerances; and the two new declaration fields round-trip through JSON. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pandas as pd +import pytest + +from microcosm.graph import ( + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + Node, + Numeric, + NumericScope, + Owned, + SourceRef, + StructuralDelta, + Tolerance, + graph_from_json, + graph_to_json, +) + + +def test_tolerance_must_allow_some_movement_and_stay_finite() -> None: + assert Tolerance(rtol=1e-9).ulps == 0 + assert Tolerance(ulps=2) == Tolerance(0.0, 0.0, 2) + with pytest.raises(ValueError, match="allow some movement"): + Tolerance() + with pytest.raises(ValueError, match="non-negative and finite"): + Tolerance(rtol=-1e-9) + with pytest.raises(ValueError, match="non-negative and finite"): + Tolerance(atol=float("inf")) + with pytest.raises(ValueError, match="non-negative and finite"): + Tolerance(rtol=float("nan")) + with pytest.raises(ValueError, match="must be an integer"): + Tolerance(ulps=1.5) # type: ignore[arg-type] + + +def test_tolerance_bound_kernels_declare_a_bound_and_bitwise_kernels_none() -> None: + bounded = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.TOLERANCE_BOUND, + tolerance=Tolerance(rtol=1e-6), + ) + assert bounded.tolerance == Tolerance(rtol=1e-6) + with pytest.raises(ValueError, match="must declare its Tolerance"): + Capabilities(determinism=Determinism.SEEDED, numeric=Numeric.TOLERANCE_BOUND) + with pytest.raises(ValueError, match="bitwise kernel declares no Tolerance"): + Capabilities(determinism=Determinism.DETERMINISTIC, tolerance=Tolerance(ulps=1)) + with pytest.raises(TypeError, match="must be a Tolerance or None"): + Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.TOLERANCE_BOUND, + tolerance=1e-6, # type: ignore[arg-type] + ) + + +def test_context_carries_declared_tolerances_and_defaults_to_none() -> None: + node = Node("gate", "gate.check@1") + context = KernelContext( + node=node, + tables={}, + weights={}, + strata=pd.Series([], dtype=object, name="stratum"), + params={}, + rng=np.random.default_rng(0), + ) + assert dict(context.tolerances) == {} + carried = KernelContext( + node=node, + tables={}, + weights={}, + strata=pd.Series([], dtype=object, name="stratum"), + params={}, + rng=np.random.default_rng(0), + tolerances={ + ("person", "income"): Tolerance(rtol=1e-6), + ("person", "age"): None, + }, + ) + assert carried.tolerances[("person", "income")] == Tolerance(rtol=1e-6) + assert carried.tolerances[("person", "age")] is None + + +def test_entrants_and_mass_partition_round_trip_through_canonical_json() -> None: + source = SourceRef("survey", "frame-h5") + create = Node( + "survey", + "source.frame@1", + sources=("survey",), + structural=StructuralDelta.CREATE, + outputs=(Owned("person", "age", "int64"), Owned("person", "period", "int64")), + ) + cohort = Node( + "cohort", + "enter.immigrants@1", + base="survey", + structural=StructuralDelta.EXPAND, + mass="declared", + entrants=True, + ) + graph = Graph( + "toy", (source,), (create, cohort), mass_partition=("person", "period") + ) + text = graph_to_json(graph) + assert '"entrants":true' in text and '"mass_partition":["person","period"]' in text + assert graph_from_json(text) == graph + # A declaration without either field serializes exactly as it did before + # amendments 11 and 12, so every pinned graph JSON still matches. + plain = Graph("toy", (source,), (create,)) + plain_text = graph_to_json(plain) + assert "entrants" not in plain_text and "mass_partition" not in plain_text + assert graph_from_json(plain_text) == plain + + +def test_capabilities_reject_look_alike_fields_and_registration_needs_the_real_thing() -> ( + None +): + """A string spelling an enum member is not the member (review of #851, finding 5).""" + with pytest.raises(TypeError, match="Capabilities.numeric must be a Numeric"): + Capabilities(determinism=Determinism.DETERMINISTIC, numeric="tolerance_bound") # type: ignore[arg-type] + with pytest.raises( + TypeError, match="Capabilities.determinism must be a Determinism" + ): + Capabilities(determinism="deterministic") # type: ignore[arg-type] + with pytest.raises(TypeError, match="consumes_se must be a boolean"): + Capabilities(determinism=Determinism.DETERMINISTIC, consumes_se=1) # type: ignore[arg-type] + with pytest.raises(TypeError, match="dependencies must be a tuple"): + Capabilities(determinism=Determinism.DETERMINISTIC, dependencies=["numpy"]) # type: ignore[arg-type] + + class LookAlike: + determinism = Determinism.DETERMINISTIC + numeric = "tolerance_bound" + seed_source = "none" + structural = "none" + role = "compute" + consumes_se = False + dependencies = () + tolerance = None + + class Impostor(KernelBase): + ref = "impostor@1" + capabilities = LookAlike() # type: ignore[assignment] + + def run(self, context): # pragma: no cover - never reached + raise AssertionError + + with pytest.raises(TypeError, match="must carry a Capabilities instance"): + KernelRegistry().register(Impostor()) + + +def test_platform_bitwise_forbids_a_tolerance_like_bitwise() -> None: + """Amendment 16: platform-bitwise kernels declare no per-cell tolerance.""" + Capabilities(Determinism.SEEDED, numeric=Numeric.PLATFORM_BITWISE) + with pytest.raises(ValueError, match="bitwise kernel declares no Tolerance"): + Capabilities( + Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + tolerance=Tolerance(rtol=1e-6), + ) + + +def test_numeric_scope_validates_class_tolerance_and_platform() -> None: + """Amendment 17: a scope carries exactly the fields its class permits.""" + assert NumericScope().numeric is Numeric.BITWISE + bound = Tolerance(rtol=1e-6) + NumericScope(numeric=Numeric.TOLERANCE_BOUND, tolerance=bound) + NumericScope( + numeric=Numeric.TOLERANCE_BOUND, tolerance=bound, platform="arm64/darwin/py3.13" + ) + NumericScope(numeric=Numeric.PLATFORM_BITWISE, platform="arm64/darwin/py3.13") + with pytest.raises(ValueError, match="must carry its Tolerance"): + NumericScope(numeric=Numeric.TOLERANCE_BOUND) + with pytest.raises(ValueError, match="carries no Tolerance"): + NumericScope(numeric=Numeric.PLATFORM_BITWISE, platform="x", tolerance=bound) + with pytest.raises(ValueError, match="must name its platform"): + NumericScope(numeric=Numeric.PLATFORM_BITWISE) + with pytest.raises(ValueError, match="every platform"): + NumericScope(platform="arm64/darwin/py3.13") + + +def test_context_numerics_default_empty_and_carry_scopes() -> None: + """Amendment 17: ``numerics`` defaults empty and rides at the end of the context.""" + fields = [f.name for f in dataclasses.fields(KernelContext)] + assert fields[-2:] == ["tolerances", "numerics"] + scope = NumericScope( + numeric=Numeric.PLATFORM_BITWISE, platform="arm64/darwin/py3.13" + ) + context = KernelContext( + node=Node("gate", "gate.check@1"), + tables={}, + weights={}, + strata=pd.Series(dtype="int64"), + params={}, + rng=np.random.default_rng(0), + tolerances={("person", "income"): None}, + numerics={("person", "income"): scope}, + ) + assert context.numerics[("person", "income")] is scope + assert context.tolerances[("person", "income")] is None diff --git a/packages/microcosm-graph/tests/test_graph_keys.py b/packages/microcosm-graph/tests/test_graph_keys.py index 4b1acdd25..3b0fc89f5 100644 --- a/packages/microcosm-graph/tests/test_graph_keys.py +++ b/packages/microcosm-graph/tests/test_graph_keys.py @@ -6,6 +6,8 @@ from dataclasses import replace from pathlib import Path +import pytest + from microcosm.graph.canonical import canonical_json, sha256_domain from microcosm.graph.decl import ( CompiledGraph, @@ -17,6 +19,14 @@ StructuralDelta, compile_graph, ) +from microcosm.graph.kernel import ( + Capabilities, + Determinism, + KernelRole, + Numeric, + SeedSource, + Tolerance, +) from microcosm.graph.keys import ( artifact_key, frame_key, @@ -39,6 +49,12 @@ ) +def _capabilities( + structural: StructuralDelta = StructuralDelta.NONE, +) -> Capabilities: + return Capabilities(Determinism.DETERMINISTIC, structural=structural) + + def _ordinary( node_id: str, inputs: tuple[str, ...], @@ -87,6 +103,7 @@ def _all_keys( keys, implementation_hashes[node.kernel], {"survey": source_key}, + kernel_capabilities=_capabilities(node.structural), ) return compiled, keys @@ -199,13 +216,21 @@ def test_carried_columns_resolve_to_the_structural_version() -> None: graph = Graph("toy", (SOURCE,), (CREATE, subset, model)) compiled = compile_graph(graph) keys = {"survey": "a" * 64, "adults": "b" * 64} - baseline = node_key(compiled, "model", keys, "c" * 64, {}) + baseline = node_key( + compiled, + "model", + keys, + "c" * 64, + {}, + kernel_capabilities=_capabilities(), + ) changed_unreachable_base = node_key( compiled, "model", {"survey": "d" * 64, "adults": "b" * 64}, "c" * 64, {}, + kernel_capabilities=_capabilities(), ) assert baseline == changed_unreachable_base @@ -227,6 +252,7 @@ def test_structural_key_binds_every_patch_in_its_base_version() -> None: {"survey": "a" * 64, "patched": "b" * 64}, "c" * 64, {}, + kernel_capabilities=_capabilities(StructuralDelta.FILTER), ) changed_patch = node_key( compiled, @@ -234,6 +260,7 @@ def test_structural_key_binds_every_patch_in_its_base_version() -> None: {"survey": "a" * 64, "patched": "d" * 64}, "c" * 64, {}, + kernel_capabilities=_capabilities(StructuralDelta.FILTER), ) assert baseline != changed_patch @@ -248,6 +275,7 @@ def test_non_create_source_consumers_bind_their_declared_source_bytes() -> None: {"survey": "a" * 64}, "b" * 64, {"survey": "c" * 64}, + kernel_capabilities=_capabilities(), ) changed = node_key( compiled, @@ -255,5 +283,75 @@ def test_non_create_source_consumers_bind_their_declared_source_bytes() -> None: {"survey": "a" * 64}, "b" * 64, {"survey": "d" * 64}, + kernel_capabilities=_capabilities(), ) assert baseline != changed + + +def test_every_capability_field_changes_the_node_key() -> None: + compiled = compile_graph(_graph()) + base = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.TOLERANCE_BOUND, + seed_source=SeedSource.EXECUTOR, + role=KernelRole.COMPUTE, + consumes_se=False, + dependencies=("numpy",), + tolerance=Tolerance(rtol=1e-6, atol=2e-6, ulps=1), + ) + + def key(capabilities: Capabilities) -> str: + return node_key( + compiled, + "a", + {"survey": "a" * 64}, + "b" * 64, + {}, + kernel_capabilities=capabilities, + ) + + baseline = key(base) + variants = ( + replace(base, determinism=Determinism.DETERMINISTIC), + replace(base, numeric=Numeric.BITWISE, tolerance=None), + replace(base, seed_source=SeedSource.PARAM), + replace(base, structural=StructuralDelta.FILTER), + replace(base, role=KernelRole.GATE), + replace(base, consumes_se=True), + replace(base, dependencies=("numpy", "pandas")), + replace(base, tolerance=Tolerance(rtol=3e-6, atol=2e-6, ulps=1)), + ) + assert all(key(capabilities) != baseline for capabilities in variants) + + positive_zero = replace(base, tolerance=Tolerance(rtol=0.0, atol=2e-6, ulps=1)) + negative_zero = replace(base, tolerance=Tolerance(rtol=-0.0, atol=2e-6, ulps=1)) + assert key(positive_zero) == key(negative_zero) + + +def test_platform_bitwise_keys_carry_the_platform( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Amendment 16: a platform-bitwise kernel's key differs across platforms; + other kernels' keys do not depend on the platform at all.""" + from microcosm.graph import keys as keys_module + + fingerprints = iter(["arm64/darwin/py3.14", "x86_64/linux/py3.14"]) + monkeypatch.setattr(keys_module, "platform_fingerprint", lambda: next(fingerprints)) + compiled = compile_graph(_graph()) + bound = Capabilities(Determinism.SEEDED, numeric=Numeric.PLATFORM_BITWISE) + first = node_key( + compiled, "a", {"survey": "s" * 64}, "impl", {}, kernel_capabilities=bound + ) + second = node_key( + compiled, "a", {"survey": "s" * 64}, "impl", {}, kernel_capabilities=bound + ) + assert first != second + plain = Capabilities(Determinism.DETERMINISTIC) + monkeypatch.setattr( + keys_module, "platform_fingerprint", lambda: "never/called/py0.0" + ) + assert node_key( + compiled, "a", {"survey": "s" * 64}, "impl", {}, kernel_capabilities=plain + ) == node_key( + compiled, "a", {"survey": "s" * 64}, "impl", {}, kernel_capabilities=plain + ) diff --git a/packages/microcosm-graph/tests/test_graph_manifest.py b/packages/microcosm-graph/tests/test_graph_manifest.py index 8d6a50621..587f9a0d4 100644 --- a/packages/microcosm-graph/tests/test_graph_manifest.py +++ b/packages/microcosm-graph/tests/test_graph_manifest.py @@ -6,15 +6,20 @@ from dataclasses import FrozenInstanceError, replace from pathlib import Path +import numpy as np import pandas as pd import pytest import microcosm.graph as graph_api +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.canonical import canonical_json, sha256_domain from microcosm.graph.decl import StructuralDelta from microcosm.graph.kernel import Capabilities, Determinism, KernelRole, SeedSource -from microcosm.graph.manifest import Decision, NodeReceipt, RunManifest +from microcosm.graph.manifest import Decision, NodeReceipt, PopulationView, RunManifest from microcosm.graph.population import MassRecord +MANIFEST_FIXTURES = Path(__file__).parent / "fixtures" / "manifests" + def _capabilities(role: KernelRole = KernelRole.COMPUTE) -> Capabilities: return Capabilities( @@ -26,6 +31,59 @@ def _capabilities(role: KernelRole = KernelRole.COMPUTE) -> Capabilities: ) +def _frame() -> Frame: + person = pd.DataFrame( + { + "person_id": np.asarray([1, 2], dtype=np.int64), + "person_household_id": np.asarray([10, 20], dtype=np.int64), + } + ) + household = pd.DataFrame( + { + "household_id": np.asarray([10, 20], dtype=np.int64), + "size": np.asarray([1, 1], dtype=np.int64), + } + ) + return Frame( + {"person": person, "household": household}, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + np.asarray([1.0, 2.0], dtype=np.float64), + WeightKind.DESIGN, + ) + }, + pd.Series(["a", "b"], name="stratum"), + ) + + +def _frame_with_colliding_entity(name: str) -> Frame: + person = pd.DataFrame( + { + "person_id": np.asarray([1, 2], dtype=np.int64), + f"person_{name}_id": np.asarray([10, 20], dtype=np.int64), + } + ) + group = pd.DataFrame( + { + f"{name}_id": np.asarray([10, 20], dtype=np.int64), + f"{name}_value": np.asarray([100, 200], dtype=np.int64), + } + ) + return Frame( + {"person": person, name: group}, + EntitySchema(group_entities=(name,)), + { + name: Weights( + np.asarray([1.0, 2.0], dtype=np.float64), + WeightKind.DESIGN, + ) + }, + pd.Series(["a", "b"], name="stratum"), + metadata={"source": "collision fixture"}, + ) + + def _receipt(key: str, *, hit: bool = False, wall_time: float = 0.2) -> NodeReceipt: return NodeReceipt( key=key, @@ -82,14 +140,15 @@ def _persisted_manifest( "tier": tier, "outcome": release_outcome or ("pass" if tier == "certified" else "fail"), "gate_ancestry": ["gate"], + "requires_decisions": [], }, artifacts={("release", "tier"): release_artifact}, ) return RunManifest("toy", {"release": release, "gate": gate}) -def test_manifest_json_round_trip_and_convenient_lookup() -> None: - population = object() +def test_manifest_json_round_trip_and_population_view() -> None: + raw = _frame() manifest = RunManifest( country="toy", nodes={"b": _receipt("b" * 64), "a": _receipt("a" * 64)}, @@ -97,7 +156,7 @@ def test_manifest_json_round_trip_and_convenient_lookup() -> None: started_at="2026-09-01T12:00:00Z", finished_at="2026-09-01T12:00:01Z", host="runner-1", - populations={"survey": population}, # type: ignore[dict-item] + populations={"survey": raw, "filtered": raw}, ) restored = RunManifest.from_json(manifest.to_json()) assert restored == manifest @@ -105,12 +164,52 @@ def test_manifest_json_round_trip_and_convenient_lookup() -> None: assert manifest.nodes["a"] is manifest.node("a") assert manifest.receipts["a"] is manifest.receipt("a") assert manifest["a"].artifacts[("person", "x")] == "d" * 64 - assert manifest.population("survey") is population + + survey = manifest.population("survey") + filtered = manifest.population("filtered") + assert type(survey) is type(filtered) is PopulationView + assert isinstance(survey, Frame) + assert manifest.population("survey") is survey + assert type(raw) is Frame + assert not hasattr(raw, "household") + assert survey.person is raw.person + assert survey.household is raw.table("household") + assert survey.table("household") is raw.table("household") + assert survey.weights_for("household") is raw.weights_for("household") + assert survey.strata is raw.strata + with pytest.raises(AttributeError, match="PopulationView.*missing"): + _ = survey.missing + with pytest.raises(KeyError, match="not attached"): restored.population("survey") + with pytest.raises(TypeError, match="values must be Frame"): + RunManifest( + "toy", + {"a": _receipt("a" * 64)}, + populations={"survey": object()}, # type: ignore[dict-item] + ) + + +@pytest.mark.parametrize("entity_name", ["metadata", "schema", "table", "entities"]) +def test_population_view_entity_accessor_handles_frame_attribute_collisions( + entity_name: str, +) -> None: + raw = _frame_with_colliding_entity(entity_name) + view = RunManifest( + country="toy", + nodes={"a": _receipt("a" * 64)}, + populations={"survey": raw}, + ).population("survey") + + assert view.entity(entity_name) is raw.table(entity_name) + # The colliding name resolves to the Frame member, exactly as on a Frame, + # and inherited Frame operations keep working on the view. + assert getattr(view, entity_name) is not raw.table(entity_name) + assert view.n("person") == raw.n("person") + assert list(view.entities) == list(raw.entities) -def test_manifest_key_excludes_every_operational_field() -> None: +def test_manifest_key_authenticates_receipts_and_excludes_run_metadata() -> None: cold = RunManifest( country="toy", nodes={ @@ -135,11 +234,29 @@ def test_manifest_key_excludes_every_operational_field() -> None: finished_at="soon", host="host-b", ) - assert cold.key == warm.key - assert cold.to_json() != warm.to_json() + metadata_only = RunManifest( + country="renamed descriptive label", + nodes=cold.nodes, + started_at="second", + finished_at="soon", + host="host-b", + ) + assert cold.key == metadata_only.key + # A warm run that changed nothing but the run-level fields shares the key. + run_level_only = replace( + warm, + nodes={ + node_id: replace(receipt, receipt=cold.nodes[node_id].receipt) + for node_id, receipt in warm.nodes.items() + }, + ) + assert cold.key == run_level_only.key + assert cold.to_json() != run_level_only.to_json() + # A changed kernel receipt is content, so it moves the key. + assert cold.key != warm.key -def test_decisions_change_manifest_identity_but_not_node_identity() -> None: +def test_decisions_are_provenance_outside_manifest_and_node_identity() -> None: receipt = _receipt("a" * 64) bare = RunManifest("toy", {"a": receipt}) decided = RunManifest( @@ -148,7 +265,7 @@ def test_decisions_change_manifest_identity_but_not_node_identity() -> None: decisions=(Decision("owner", "release", "yes", "2026-09-01"),), ) assert bare.nodes["a"].key == decided.nodes["a"].key - assert bare.key != decided.key + assert bare.key == decided.key def test_original_signed_decision_records_remain_mapping_compatible() -> None: @@ -185,7 +302,7 @@ def test_decision_and_node_mapping_order_are_not_identity() -> None: assert forward.key == reversed_.key -def test_serialized_content_key_detects_provenance_tampering() -> None: +def test_serialized_decisions_are_provenance_outside_content_key() -> None: manifest = RunManifest( "toy", {"a": _receipt("a" * 64)}, @@ -193,8 +310,9 @@ def test_serialized_content_key_detects_provenance_tampering() -> None: ) payload = json.loads(manifest.to_json()) payload["decisions"][0]["text"] = "no" - with pytest.raises(ValueError, match="content key mismatch"): - RunManifest.from_json(json.dumps(payload)) + restored = RunManifest.from_json(json.dumps(payload)) + assert restored.key == manifest.key + assert restored.decisions[0].text == "no" def test_receipts_and_nested_payloads_are_immutable() -> None: @@ -213,7 +331,7 @@ def test_receipts_and_nested_payloads_are_immutable() -> None: assert receipt.artifact_keys == receipt.artifacts -def test_optional_artifact_identities_round_trip_and_allow_legacy_absence() -> None: +def test_optional_artifact_identities_round_trip_and_missing_fields_fail_key() -> None: manifest = RunManifest("toy", {"a": _receipt("a" * 64)}) restored = RunManifest.from_json(manifest.to_json()) receipt = restored.nodes["a"] @@ -225,10 +343,8 @@ def test_optional_artifact_identities_round_trip_and_allow_legacy_absence() -> N del legacy_payload["nodes"]["a"]["frame_key"] del legacy_payload["nodes"]["a"]["weight_key"] del legacy_payload["nodes"]["a"]["opaque_artifacts"] - legacy = RunManifest.from_json(json.dumps(legacy_payload)) - assert legacy.nodes["a"].frame_key is None - assert legacy.nodes["a"].weight_key is None - assert legacy.nodes["a"].opaque_artifacts == {} + with pytest.raises(ValueError, match="node 'a'"): + RunManifest.from_json(json.dumps(legacy_payload)) def test_transient_mass_ledgers_are_immutable_and_not_portable_identity() -> None: @@ -265,14 +381,23 @@ def test_saved_manifest_persists_and_rederives_release_fields(tmp_path: Path) -> manifest.save(path) document = json.loads(path.read_text()) - assert document["schema_version"] == 1 + assert document["schema_version"] == 2 assert document["key"] == manifest.key assert document["tier"] == "evidence" assert document["known_failures"] == ["gate"] - assert document["content_addressed"] == { - "node_keys": ["a" * 64, "c" * 64], - "decisions": [], - } + body = document["content_addressed"] + assert body["tier"] == "evidence" + # The body is the serialized receipts less their run-level fields, and a + # release node's decision-derived outcome is run-level too. + for node_id, receipt in document["nodes"].items(): + serialized = {k: v for k, v in receipt.items() if k not in ("hit", "wall_time")} + if receipt["capabilities"]["role"] == "release": + assert "outcome" in receipt["receipt"] + serialized["receipt"] = { + k: v for k, v in receipt["receipt"].items() if k != "outcome" + } + assert body["nodes"][node_id] == serialized + assert "decisions" not in body restored = RunManifest.load(path, store) assert restored.key == manifest.key @@ -282,6 +407,115 @@ def test_saved_manifest_persists_and_rederives_release_fields(tmp_path: Path) -> RunManifest.load_certified(path, store) +def test_from_json_rejects_coordinated_gate_and_tier_tampering( + tmp_path: Path, +) -> None: + store = graph_api.ContentStore(tmp_path / "store") + manifest = _persisted_manifest(store) + document = json.loads(manifest.to_json()) + document["nodes"]["gate"]["receipt"]["outcome"] = "pass" + document["nodes"]["release"]["receipt"].update( + {"tier": "certified", "outcome": "pass"} + ) + document["tier"] = "certified" + document["known_failures"] = [] + tampered = json.dumps(document) + + with pytest.raises(ValueError, match="gate"): + RunManifest.from_json(tampered) + + path = tmp_path / "tampered.json" + path.write_text(tampered, encoding="utf-8") + with pytest.raises(graph_api.StoreCorruptError, match="gate"): + RunManifest.load_certified(path, store) + + +def test_schema_v2_body_rejects_canonically_distinct_numeric_value( + tmp_path: Path, +) -> None: + store = graph_api.ContentStore(tmp_path / "store") + manifest = _persisted_manifest(store) + document = json.loads(manifest.to_json()) + document["content_addressed"]["nodes"]["gate"]["seed"] = 1.0 + document["key"] = sha256_domain( + "manifest", canonical_json(document["content_addressed"]) + ) + + with pytest.raises(ValueError, match="node 'gate'"): + RunManifest.from_json(json.dumps(document)) + + +def test_v1_manifest_relabel_with_fabricated_tolerance_fails_key() -> None: + path = MANIFEST_FIXTURES / "v1_tolerance_bound_without_tolerance.json" + document = json.loads(path.read_text()) + document["schema_version"] = 2 + receipt = document["nodes"]["fit_qrf"] + receipt["legacy_capabilities"] = False + receipt["capabilities"]["tolerance"] = { + "rtol": 1e-6, + "atol": 1e-9, + "ulps": 2, + } + + with pytest.raises(ValueError, match="fit_qrf"): + RunManifest.from_json(json.dumps(document)) + + +def test_untouched_certified_manifest_round_trips_and_loads_certified( + tmp_path: Path, +) -> None: + store = graph_api.ContentStore(tmp_path / "store") + manifest = _persisted_manifest( + store, + tier="certified", + gate_outcome="pass", + ) + + restored = RunManifest.from_json(manifest.to_json()) + assert restored.to_json() == manifest.to_json() + path = tmp_path / "certified.json" + manifest.save(path) + assert RunManifest.load_certified(path, store).key == manifest.key + + +def test_v1_tolerance_bound_manifest_loads_as_legacy_cache_miss( + tmp_path: Path, +) -> None: + path = MANIFEST_FIXTURES / "v1_tolerance_bound_without_tolerance.json" + raw_capabilities = json.loads(path.read_text())["nodes"]["fit_qrf"]["capabilities"] + store = graph_api.ContentStore(tmp_path / "store") + + manifest = RunManifest.load(path, store) + node = manifest.nodes["fit_qrf"] + + assert node.hit is False + assert node.legacy_capabilities is True + assert set(node.capabilities) == set(raw_capabilities) + assert node.capabilities["numeric"] == "tolerance_bound" + assert tuple(node.capabilities["dependencies"]) == tuple( + raw_capabilities["dependencies"] + ) + with pytest.raises(ValueError, match=r"legacy capabilities.*omit.*tolerance"): + replace( + node, + capabilities={**raw_capabilities, "tolerance": None}, + legacy_capabilities=True, + ) + with pytest.raises( + graph_api.NodeRejectedError, + match=r"unreached.*legacy_capabilities.*tolerance", + ): + RunManifest.load_certified(path, store) + + emitted = json.loads(manifest.to_json()) + emitted_node = emitted["nodes"]["fit_qrf"] + assert emitted["schema_version"] == 2 + assert emitted_node["legacy_capabilities"] is True + assert emitted_node["capabilities"] == raw_capabilities + assert "tolerance" not in emitted_node["capabilities"] + assert RunManifest.from_json(manifest.to_json()).to_json() == manifest.to_json() + + def test_certified_loader_checks_unreached_before_tier(tmp_path: Path) -> None: store = graph_api.ContentStore(tmp_path / "store") certified = _persisted_manifest( @@ -303,6 +537,29 @@ def test_certified_loader_checks_unreached_before_tier(tmp_path: Path) -> None: RunManifest.load_certified(unreached_path, store) +def test_certified_loader_requires_authenticated_decision_requirements( + tmp_path: Path, +) -> None: + store = graph_api.ContentStore(tmp_path / "store") + manifest = _persisted_manifest(store, tier="certified", gate_outcome="pass") + path = tmp_path / "missing-requirements.json" + document = json.loads(manifest.to_json()) + del document["nodes"]["release"]["receipt"]["requires_decisions"] + del document["content_addressed"]["nodes"]["release"]["receipt"][ + "requires_decisions" + ] + document["key"] = sha256_domain( + "manifest", canonical_json(document["content_addressed"]) + ) + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises( + graph_api.NodeRejectedError, + match="authenticated requires_decisions", + ): + RunManifest.load_certified(path, store) + + def test_loader_rederives_tier_from_gate_receipts(tmp_path: Path) -> None: store = graph_api.ContentStore(tmp_path / "store") manifest = _persisted_manifest(store, tier="certified", gate_outcome="pass") @@ -323,13 +580,43 @@ def test_loader_wraps_noncanonical_body_with_manifest_key(tmp_path: Path) -> Non path = tmp_path / "manifest.json" manifest.save(path) document = json.loads(path.read_text()) - document["content_addressed"]["node_keys"][0] = float("nan") + document["content_addressed"]["nodes"]["gate"]["seed"] = float("nan") path.write_text(json.dumps(document)) with pytest.raises(graph_api.StoreCorruptError, match=manifest.key): RunManifest.load(path, store) +def test_load_wraps_oversized_tolerance_integer_as_store_corrupt( + tmp_path: Path, +) -> None: + store = graph_api.ContentStore(tmp_path / "store") + manifest = _persisted_manifest(store) + path = tmp_path / "manifest.json" + document = json.loads(manifest.to_json()) + oversized = int("9" * 400) + for gate in ( + document["nodes"]["gate"], + document["content_addressed"]["nodes"]["gate"], + ): + gate["capabilities"]["numeric"] = "tolerance_bound" + gate["capabilities"]["tolerance"] = { + "rtol": oversized, + "atol": 0, + "ulps": 0, + } + document["key"] = sha256_domain( + "manifest", canonical_json(document["content_addressed"]) + ) + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises( + graph_api.StoreCorruptError, + match="representable as a finite float", + ): + RunManifest.load(path, store) + + def test_known_failures_includes_explicitly_rejected_nodes() -> None: rejected = replace( _receipt("a" * 64), @@ -347,7 +634,7 @@ def test_known_failures_includes_explicitly_rejected_nodes() -> None: ("schema_version", True), ("known_failures", []), ("key", "0" * 64), - ("content_addressed", {"node_keys": [], "decisions": []}), + ("content_addressed", {"nodes": {}, "tier": None}), ], ) def test_load_rejects_every_persisted_projection_mismatch( @@ -397,6 +684,7 @@ def test_load_requires_every_manifest_artifact( def test_package_exports_runtime_implementations_and_failures() -> None: assert graph_api.ContentStore.__module__.endswith(".store") assert graph_api.RunManifest is RunManifest + assert graph_api.PopulationView is PopulationView assert graph_api.NodeReceipt is NodeReceipt assert graph_api.Decision is Decision assert graph_api.run_graph.__module__.endswith(".executor") diff --git a/packages/microcosm-graph/tests/test_graph_population.py b/packages/microcosm-graph/tests/test_graph_population.py index 13bb682c9..a96562d88 100644 --- a/packages/microcosm-graph/tests/test_graph_population.py +++ b/packages/microcosm-graph/tests/test_graph_population.py @@ -21,6 +21,7 @@ PopulationError, dtype_for_token, dtype_matches, + entrant_strata_receipt, expand_lineage_receipt, owned_ids, patch, @@ -393,6 +394,35 @@ def test_declared_mass_validates_the_kernel_receipt() -> None: patch(population, node, result) +def test_mass_receipt_rejects_partition_when_graph_has_none() -> None: + population = _population() + node = Node( + "importance", + "test@1", + structural=StructuralDelta.REWEIGHT, + base="source", + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + ) + receipt = _mass_receipt( + policy="declared", + before=7.0, + after=14.0, + stratum_before={"a": 2.0, "b": 5.0}, + stratum_after={"a": 4.0, "b": 10.0}, + ) + mass = receipt["mass"] + assert isinstance(mass, dict) + mass["partition"] = {} + result = KernelResult( + weights=Weights(np.array([2.0, 4.0, 6.0]), WeightKind.IMPORTANCE), + receipt=receipt, + ) + + with pytest.raises(PopulationError, match="declares no mass partition"): + patch(population, node, result) + + def test_filter_requires_subset_ids_and_records_free_mass() -> None: population = _population() filtered = population.frame.select( @@ -423,6 +453,49 @@ def test_filter_requires_subset_ids_and_records_free_mass() -> None: patch(population, conserve, KernelResult(frame=filtered)) +def test_filter_conserve_allows_removed_zero_mass_partition_support() -> None: + base = _frame() + person = base.table("person").copy() + person["period"] = pd.Series(["zero", "zero", "kept", "kept"], dtype="string") + frame = Frame( + {"person": person, "household": base.table("household").copy()}, + base.schema, + { + "household": Weights( + np.array([0.0, 2.0, 3.0], dtype=np.float64), WeightKind.DESIGN + ) + }, + base.strata, + ) + population = Population.from_frame(frame, "source") + filtered = frame.select(np.array([False, False, True, True], dtype=np.bool_)) + node = Node( + "filter_zero_support", + "test@1", + structural=StructuralDelta.FILTER, + base="source", + mass="conserve", + ) + + updated = patch( + population, + node, + KernelResult(frame=filtered), + mass_partition=("person", "period"), + ) + + record = updated.mass_ledger[-1] + assert dict(record.before_by_stratum) == {"a": 0.0, "b": 5.0} + assert dict(record.after_by_stratum) == {"b": 5.0} + assert { + partition: dict(strata) + for partition, strata in record.before_partitions.items() + } == {"kept": {"b": 5.0}, "zero": {"a": 0.0}} + assert { + partition: dict(strata) for partition, strata in record.after_partitions.items() + } == {"kept": {"b": 5.0}} + + def test_expand_must_retain_every_original_id() -> None: population = _population() filtered = population.frame.select( @@ -475,6 +548,260 @@ def _lineage_expand_result(*, bad_source: bool = False) -> KernelResult: ) +def _membership_overlay_result(target: int) -> KernelResult: + return KernelResult( + expand={ + "person": pd.Series( + [1, 2, 3], + index=pd.Index([5, 6, 7], name="person_id"), + dtype="int64", + ), + "household": pd.Series( + [10, 20], + index=pd.Index([40, 50], name="household_id"), + dtype="int64", + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 10, 20, 30, target, 40, 50], + index=pd.Index([1, 2, 3, 4, 5, 6, 7], name="person_id"), + dtype="int64", + ) + }, + weights=Weights( + np.array([0.5, 1.0, 3.0, 0.5, 1.0], dtype=np.float64), + WeightKind.IMPORTANCE, + ), + ) + + +@pytest.mark.parametrize("cached", [False, True], ids=("cold", "cached")) +@pytest.mark.parametrize("target", [20, 50], ids=("incumbent", "copied-group")) +def test_expand_rejects_repointed_copied_membership(target: int, cached: bool) -> None: + population = _population() + node = Node( + "membership_overlay", + "test@1", + structural=StructuralDelta.EXPAND, + base="source", + params={ + "expand_cells": (("person", "person_household_id", "int64"),), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + mass="free", + ) + result = _membership_overlay_result(target) + if cached: + legal = _membership_overlay_result(40) + expanded = patch(population, node, legal) + person = expanded.frame.table("person").copy() + person.loc[person["person_id"] == 5, "person_household_id"] = target + result = KernelResult( + frame=_replace_person_table(expanded.frame, person, expanded.frame.strata), + weights=legal.weights, + receipt={"expand": expand_lineage_receipt(legal.expand)}, + ) + + with pytest.raises(PopulationError) as error: + if cached: + restore_cached_expand(population, node, result) + else: + patch(population, node, result) + + message = str(error.value) + assert "membership_overlay" in message + assert "person.person_household_id" in message + assert "person id 5" in message + assert f"household id {target}" in message + + +def _entrant_to_copied_group_result(entrant_group: int) -> KernelResult: + frame = _frame() + person = frame.table("person") + person_ids = pd.Index([1, 2, 3, 4, 5, 6, 7], name="person_id", dtype="int64") + additions = { + "person_household_id": (40, 40, entrant_group), + "keep": (True, True, True), + "owned": (False, True, False), + "nullable": (True, False, pd.NA), + "amount": (-0.0, 1.0, 3.0), + } + tokens = {column: token_for_dtype(person[column].dtype) for column in additions} + return KernelResult( + expand={ + "person": pd.Series( + [1, 2, pd.NA], + index=pd.Index([5, 6, 7], name="person_id", dtype="int64"), + dtype="Int64", + ), + "household": pd.Series( + [10], + index=pd.Index([40], name="household_id", dtype="int64"), + dtype="int64", + ), + }, + columns={ + ("person", column): pd.Series( + pd.array([*person[column], *values], dtype=tokens[column]), + index=person_ids, + ) + for column, values in additions.items() + }, + weights=Weights( + np.array([1.0, 2.0, 3.0, 1.0], dtype=np.float64), + WeightKind.DESIGN, + ), + strata=pd.Series( + ["entrant"], + index=pd.Index([7], name="person_id", dtype="int64"), + dtype=object, + ), + ) + + +@pytest.mark.parametrize("cached", [False, True], ids=("cold", "cached")) +def test_expand_rejects_entrant_membership_to_copied_group(cached: bool) -> None: + population = _population() + node = Node( + "entrant_membership", + "test@1", + structural=StructuralDelta.EXPAND, + base="source", + params={ + "expand_cells": tuple( + ( + "person", + column, + token_for_dtype(_frame().table("person")[column].dtype), + ) + for column in ( + "person_household_id", + "keep", + "owned", + "nullable", + "amount", + ) + ), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + result = _entrant_to_copied_group_result(40) + if cached: + legal = _entrant_to_copied_group_result(10) + expanded = patch(population, node, legal) + person = expanded.frame.table("person").copy() + person.loc[person["person_id"] == 7, "person_household_id"] = 40 + assert legal.expand is not None + result = KernelResult( + frame=_replace_person_table(expanded.frame, person, expanded.frame.strata), + weights=legal.weights, + receipt={ + "expand": expand_lineage_receipt(legal.expand), + "entrant_strata": entrant_strata_receipt( + population.frame, + node, + legal.expand, + legal.strata, + ), + }, + ) + + with pytest.raises(PopulationError) as error: + if cached: + restore_cached_expand(population, node, result) + else: + patch(population, node, result) + + message = str(error.value) + assert "entrant_membership" in message + assert "person.person_household_id" in message + assert "entrant person id 7" in message + assert "copied household id 40" in message + + +def _entrant_person_expand_node(*, membership_dtype: str = "int64") -> Node: + return Node( + "entrant_person", + "test@1", + structural=StructuralDelta.EXPAND, + base="source", + params={ + "expand_cells": ( + ("person", "person_household_id", membership_dtype), + ("person", "keep", "bool"), + ("person", "owned", "boolean"), + ("person", "nullable", "boolean"), + ("person", "amount", "float64"), + ), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + entrants=True, + ) + + +def _entrant_person_expand_result( + strata: object, *, frame: Frame | None = None +) -> KernelResult: + frame = _frame() if frame is None else frame + person = frame.table("person") + person_id_dtype = person["person_id"].dtype + household_id_dtype = frame.table("household")["household_id"].dtype + entrant_id = int(person["person_id"].max()) + 1 + ids = pd.Index( + pd.Series([*person["person_id"], entrant_id], dtype=person_id_dtype).array, + name="person_id", + ) + additions = { + "person_household_id": 10, + "keep": True, + "owned": False, + "nullable": pd.NA, + "amount": 3.0, + } + tokens = { + "person_household_id": token_for_dtype(person["person_household_id"].dtype), + "keep": "bool", + "owned": "boolean", + "nullable": "boolean", + "amount": "float64", + } + columns = { + ("person", column): pd.Series( + pd.array([*person[column], value], dtype=tokens[column]), index=ids + ) + for column, value in additions.items() + } + return KernelResult( + expand={ + "person": pd.Series( + pd.array( + [pd.NA], + dtype=f"Int{np.dtype(person_id_dtype).itemsize * 8}", + ), + index=pd.Index( + pd.Series([entrant_id], dtype=person_id_dtype).array, + name="person_id", + ), + ), + "household": pd.Series( + [], + index=pd.Index([], dtype=household_id_dtype, name="household_id"), + dtype=household_id_dtype, + ), + }, + columns=columns, + weights=frame.weights_for("household"), + strata=strata, # type: ignore[arg-type] + ) + + def test_expand_lineage_carries_rows_remaps_memberships_and_restores_cache() -> None: population = _population() node = _lineage_expand_node() @@ -492,13 +819,14 @@ def test_expand_lineage_carries_rows_remaps_memberships_and_restores_cache() -> assert expanded.mass_ledger[-1].operation == "expand" assert result.expand is not None + receipt = {"expand": expand_lineage_receipt(result.expand)} cached = restore_cached_expand( population, node, KernelResult( frame=expanded.frame, weights=result.weights, - receipt={"expand": expand_lineage_receipt(result.expand)}, + receipt=receipt, ), ) np.testing.assert_array_equal( @@ -506,6 +834,354 @@ def test_expand_lineage_carries_rows_remaps_memberships_and_restores_cache() -> ) assert cached.mass_ledger == expanded.mass_ledger + mutated_person = expanded.frame.table("person").copy() + mutated_person.loc[mutated_person["person_id"] == 5, "amount"] = 99.0 + mutated_frame = _replace_person_table( + expanded.frame, mutated_person, expanded.frame.strata + ) + with pytest.raises( + PopulationError, + match=r"person\.amount.*copied target/source ids.*\(5, 1\)", + ): + restore_cached_expand( + population, + node, + KernelResult( + frame=mutated_frame, + weights=result.weights, + receipt=receipt, + ), + ) + + +def test_cached_expand_rejects_changed_incumbent_storage() -> None: + population = _population() + node = _lineage_expand_node() + result = _lineage_expand_result() + expanded = patch(population, node, result) + person = expanded.frame.table("person").copy() + person.loc[person["person_id"] == 1, "amount"] = 99.0 + mutated_frame = _replace_person_table(expanded.frame, person, expanded.frame.strata) + assert result.expand is not None + + with pytest.raises(PopulationError, match="incumbent storage"): + restore_cached_expand( + population, + node, + KernelResult( + frame=mutated_frame, + weights=result.weights, + receipt={"expand": expand_lineage_receipt(result.expand)}, + ), + ) + + +@pytest.mark.parametrize("mutation", ["extra", "missing"], ids=["extra", "missing"]) +def test_cached_expand_requires_exact_declared_column_set(mutation: str) -> None: + population = _population() + node = Node( + "column_set_expand", + "test@1", + structural=StructuralDelta.EXPAND, + base="source", + params={ + "expand_cells": (("person", "new_value", "float64"),), + "expand_weight_entity": "household", + "expand_weight_kind": "importance", + }, + mass="conserve", + ) + lineage_result = _lineage_expand_result() + result = KernelResult( + expand=lineage_result.expand, + columns={ + ("person", "new_value"): pd.Series( + [10.0, 20.0, 30.0, 40.0, 50.0, 60.0], + index=pd.Index([1, 2, 3, 4, 5, 6], name="person_id"), + dtype="float64", + ) + }, + weights=lineage_result.weights, + ) + expanded = patch(population, node, result) + person = expanded.frame.table("person").copy() + if mutation == "extra": + person["unexpected"] = np.arange(len(person), dtype=np.int64) + else: + person = person.drop(columns="new_value") + mutated_frame = _replace_person_table(expanded.frame, person, expanded.frame.strata) + assert result.expand is not None + + with pytest.raises(PopulationError, match="column set"): + restore_cached_expand( + population, + node, + KernelResult( + frame=mutated_frame, + weights=result.weights, + receipt={"expand": expand_lineage_receipt(result.expand)}, + ), + ) + + +def test_cached_expand_requires_exact_lineage_id_sequence() -> None: + population = _population() + node = Node( + "cached_midpoint", + "test@1", + structural=StructuralDelta.EXPAND, + base="source", + params={ + "expand_cells": (), + "expand_weight_entity": "household", + "expand_weight_kind": "design", + }, + mass="free", + ) + before = population.frame + person = before.table("person") + household = before.table("household") + added_person = person.iloc[[0]].copy() + added_person["person_id"] = np.array([5], dtype=np.int64) + added_person["person_household_id"] = np.array([15], dtype=np.int64) + final_person = pd.concat([person, added_person], ignore_index=True) + added_household = household.iloc[[0]].copy() + added_household["household_id"] = np.array([15], dtype=np.int64) + final_household = ( + pd.concat([household, added_household], ignore_index=True) + .sort_values("household_id") + .reset_index(drop=True) + ) + final_weights = Weights( + np.array([1.0, 1.0, 2.0, 3.0], dtype=np.float64), WeightKind.DESIGN + ) + cached_frame = Frame( + {"person": final_person, "household": final_household}, + before.schema, + {"household": final_weights}, + pd.concat([before.strata, before.strata.iloc[[0]]], ignore_index=True), + ) + lineage = { + "person": pd.Series([1], index=pd.Index([5], name="person_id"), dtype="int64"), + "household": pd.Series( + [10], index=pd.Index([15], name="household_id"), dtype="int64" + ), + } + + with pytest.raises(PopulationError, match="final 'household' ids"): + restore_cached_expand( + population, + node, + KernelResult( + frame=cached_frame, + weights=final_weights, + receipt={"expand": expand_lineage_receipt(lineage)}, + ), + ) + + +def test_entrant_person_strata_materialize_and_attest_cached_replay() -> None: + population = _population() + node = _entrant_person_expand_node() + result = _entrant_person_expand_result( + pd.Series( + ["new"], + index=pd.Index([5], dtype="int64", name="ignored"), + dtype=object, + name="ignored", + ) + ) + + expanded = patch(population, node, result) + + assert expanded.frame.table("person")["person_household_id"].tolist()[-1] == 10 + assert expanded.frame.strata.tolist() == ["a", "a", "b", "b", "new"] + assert expanded.mass_ledger[-1].before_total == 7.0 + assert expanded.mass_ledger[-1].after_total == 8.0 + assert result.expand is not None + entrant_receipt = entrant_strata_receipt( + population.frame, node, result.expand, result.strata + ) + receipt = { + "expand": expand_lineage_receipt(result.expand), + "entrant_strata": entrant_receipt, + } + cached = restore_cached_expand( + population, + node, + KernelResult( + frame=expanded.frame, + weights=result.weights, + receipt=receipt, + ), + ) + pd.testing.assert_series_equal(cached.frame.strata, expanded.frame.strata) + assert cached.mass_ledger == expanded.mass_ledger + + with pytest.raises(PopulationError, match="entrant-strata receipt"): + restore_cached_expand( + population, + node, + KernelResult( + frame=expanded.frame, + weights=result.weights, + receipt={"expand": expand_lineage_receipt(result.expand)}, + ), + ) + + +def test_cached_entrant_strata_rehydrate_the_base_id_dtype() -> None: + source = _frame() + person = source.table("person").copy() + household = source.table("household").copy() + for column in ("person_id", "person_household_id"): + person[column] = person[column].astype("int32") + household["household_id"] = household["household_id"].astype("int32") + frame = Frame( + {"person": person, "household": household}, + source.schema, + {"household": source.weights_for("household")}, + source.strata.copy(), + ) + population = Population.from_frame(frame, "source") + node = _entrant_person_expand_node(membership_dtype="int32") + result = _entrant_person_expand_result( + pd.Series(["new"], index=pd.Index([5], dtype="int32"), dtype=object), + frame=frame, + ) + + expanded = patch(population, node, result) + assert result.expand is not None + receipt = { + "expand": expand_lineage_receipt(result.expand), + "entrant_strata": entrant_strata_receipt( + frame, node, result.expand, result.strata + ), + } + cached = restore_cached_expand( + population, + node, + KernelResult( + frame=expanded.frame, + weights=result.weights, + receipt=receipt, + ), + ) + + assert cached.frame.table("person")["person_id"].dtype == np.dtype("int32") + pd.testing.assert_series_equal(cached.frame.strata, expanded.frame.strata) + + +@pytest.mark.parametrize( + ("receipt_label", "changed_label"), + [(1, True), (1, 1.0), (-0.0, 0.0)], + ids=["bool", "float", "signed-zero"], +) +def test_cached_entrant_strata_preserve_label_scalar( + receipt_label: object, changed_label: object +) -> None: + population = _population() + node = _entrant_person_expand_node() + result = _entrant_person_expand_result( + pd.Series([receipt_label], index=pd.Index([5], dtype="int64"), dtype=object) + ) + expanded = patch(population, node, result) + changed_strata = expanded.frame.strata.copy() + changed_strata.iloc[-1] = changed_label + changed_frame = _replace_person_table( + expanded.frame, + expanded.frame.table("person").copy(), + changed_strata, + ) + assert result.expand is not None + receipt = { + "expand": expand_lineage_receipt(result.expand), + "entrant_strata": entrant_strata_receipt( + population.frame, node, result.expand, result.strata + ), + } + + with pytest.raises(PopulationError, match="label"): + restore_cached_expand( + population, + node, + KernelResult( + frame=changed_frame, + weights=result.weights, + receipt=receipt, + ), + ) + + +def test_cached_entrant_strata_encode_bytes_labels() -> None: + population = _population() + node = _entrant_person_expand_node() + result = _entrant_person_expand_result( + pd.Series([b"new\x00stratum"], index=pd.Index([5], dtype="int64"), dtype=object) + ) + expanded = patch(population, node, result) + assert result.expand is not None + receipt = { + "expand": expand_lineage_receipt(result.expand), + "entrant_strata": entrant_strata_receipt( + population.frame, node, result.expand, result.strata + ), + } + + assert receipt["entrant_strata"] == [[5, {"bytes_hex": "6e6577007374726174756d"}]] + cached = restore_cached_expand( + population, + node, + KernelResult( + frame=expanded.frame, + weights=result.weights, + receipt=receipt, + ), + ) + pd.testing.assert_series_equal(cached.frame.strata, expanded.frame.strata) + + +@pytest.mark.parametrize( + "strata", + [ + None, + pd.Series(["new"], index=pd.Index([6], dtype="int64"), dtype=object), + pd.Series(["old", "new"], index=pd.Index([1, 5], dtype="int64"), dtype=object), + pd.Series(["new"], index=pd.Index([5], dtype="int32"), dtype=object), + pd.Series([pd.NA], index=pd.Index([5], dtype="int64"), dtype=object), + pd.Series([1], index=pd.Index([5], dtype="int64"), dtype="int64"), + ], + ids=[ + "missing", + "unknown-id", + "incumbent-id", + "wrong-id-dtype", + "missing-label", + "wrong-label-dtype", + ], +) +def test_entrant_person_strata_reject_malformed_exact_set(strata: object) -> None: + with pytest.raises(PopulationError, match="strata"): + patch( + _population(), + _entrant_person_expand_node(), + _entrant_person_expand_result(strata), + ) + + +def test_strata_are_rejected_without_entrant_persons() -> None: + result = _lineage_expand_result() + with pytest.raises(PopulationError, match="without entrant persons"): + patch( + _population(), + _lineage_expand_node(), + KernelResult( + expand=result.expand, + weights=result.weights, + strata=pd.Series([], dtype=object), + ), + ) + def test_expand_lineage_rejects_an_unknown_source_id() -> None: with pytest.raises(PopulationError, match="unknown 'person' source ids"): diff --git a/packages/microcosm-graph/tests/test_graph_serialize.py b/packages/microcosm-graph/tests/test_graph_serialize.py index 596bd162e..fe45c0d1a 100644 --- a/packages/microcosm-graph/tests/test_graph_serialize.py +++ b/packages/microcosm-graph/tests/test_graph_serialize.py @@ -147,7 +147,7 @@ def test_generated_parity_graphs_bind_real_kernels_and_direct_bytes( store = ContentStore(tmp_path / name) manifest = run_graph( compile_graph(graph), - sources={"fixture": case}, + sources={"fixture": case / "inputs.csv"}, store=store, kernels=parity_registry(), resume="forbid", diff --git a/tools/graph_acceptance_burndown.py b/tools/graph_acceptance_burndown.py index 923b8ffc4..429b5de6a 100644 --- a/tools/graph_acceptance_burndown.py +++ b/tools/graph_acceptance_burndown.py @@ -12,18 +12,22 @@ marker on a helper would all fool a text search, and the number this prints is the number the charter is scored on. -``--verify`` compares against ``origin/main``, file by file, and exits 1 -if any file's count rose. A file that does not exist on the baseline is -reported as new and constrains nothing; a file that does is a ratchet. It also -refuses a marker that is not ``strict=True`` (a non-strict marker hides an -``xpass``, so a property could go green without anybody noticing), a marker -whose reason names no charter id, and a charter id with no test at all. +``--verify`` compares against the baseline branch by property identity and +exits 1 if any property that was green there is red now, whichever file the +marker sits in and whatever else went green (counts can offset; identities +cannot). A property the charter gained since the baseline may start red. It +also refuses a marker that is not ``strict=True`` (a non-strict marker hides +an ``xpass``, so a property could go green without anybody noticing), a marker +whose reason names no charter id, an id the charter does not list, or an id +other than the one in its own test name, two markers on one property, and a +charter id with no test at all. """ from __future__ import annotations import argparse import ast +import fnmatch import json import re import subprocess @@ -79,9 +83,9 @@ def _literal(node: ast.expr | None) -> object: def markers_in(source: str, file: str = "") -> tuple[Marker, ...]: - """Every ``pytest.mark.xfail`` marker on a test function in ``source``.""" + """Every ``pytest.mark.xfail`` marker on a module-level test function.""" found: list[Marker] = [] - for node in ast.walk(ast.parse(source, filename=file)): + for node in ast.parse(source, filename=file).body: if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): continue for decorator in node.decorator_list: @@ -108,10 +112,371 @@ def markers_in(source: str, file: str = "") -> tuple[Marker, ...]: return tuple(found) +#: The only marks an acceptance test may carry: the strict charter marker, the +#: engine guards, and a plain parametrize. Anything else that pytest would +#: honour — skip, skipif, a non-strict or unnamed xfail, marks on a class or a +#: module, marks smuggled through ``pytest.param`` — could hide a failing +#: known-green property while the ratchet reports it green. +ALLOWED_MARKS = frozenset({"xfail", "requires_uk", "requires_us", "parametrize"}) +#: Runtime calls that suppress a result from inside a test body. +SUPPRESSING_CALLS = frozenset( + { + "pytest.xfail", + "pytest.skip", + "pytest.importorskip", + "unittest.SkipTest", + "unittest.case.SkipTest", + "SkipTest", + "xfail", + "skip", + } +) +PYTEST_SUPPRESSORS = frozenset({"pytest.xfail", "pytest.skip", "pytest.importorskip"}) +UNITTEST_SUPPRESSORS = frozenset({"unittest.SkipTest", "unittest.case.SkipTest"}) +SAFE_PYTEST_CALLS = frozenset({"pytest.approx", "pytest.fail", "pytest.raises"}) +DYNAMIC_NAMESPACE_REFERENCES = frozenset( + { + "globals", + "locals", + "vars", + "exec", + "eval", + "getattr", + "setattr", + "__import__", + "builtins.globals", + "builtins.locals", + "builtins.vars", + "builtins.exec", + "builtins.eval", + "builtins.getattr", + "builtins.setattr", + "builtins.__import__", + "importlib.import_module", + } +) + + +def suppressions_in(source: str, file: str = "") -> tuple[str, ...]: + """Every way ``source`` could suppress a result that the marker scan misses. + + Returns human-readable problems; an empty tuple means the file uses only + the forms the ratchet models: module-level ``test_*`` functions carrying + :data:`ALLOWED_MARKS`, spelled through ``import pytest`` itself. The scan + fails closed: an alias for pytest, a ``from pytest import ...``, a + parametrize over a non-literal, or a ``pytest.param`` carrying marks + anywhere in the module is refused rather than resolved. + """ + problems: list[str] = [] + tree = ast.parse(source, filename=file) + parents = { + child: parent + for parent in ast.walk(tree) + for child in ast.iter_child_nodes(parent) + } + + def mark_name(node: ast.expr) -> str | None: + target = node.func if isinstance(node, ast.Call) else node + name = dotted(target) + parts = name.split(".") + if len(parts) == 3 and parts[:2] == ["pytest", "mark"]: + return parts[-1] + return None + + def rooted_at(node: ast.expr, name: str) -> bool: + while isinstance(node, ast.Attribute | ast.Subscript): + node = node.value + return isinstance(node, ast.Name) and node.id == name + + def inside_direct_mark_decorator(node: ast.expr) -> bool: + child: ast.AST = node + parent = parents.get(child) + while isinstance(parent, ast.Attribute) and parent.value is child: + child = parent + parent = parents.get(child) + if isinstance(parent, ast.Call) and parent.func is child: + child = parent + parent = parents.get(child) + return ( + isinstance(parent, ast.FunctionDef | ast.AsyncFunctionDef) + and child in parent.decorator_list + and mark_name(child) is not None + ) + + def inside_direct_safe_pytest_call(node: ast.expr) -> bool: + child: ast.AST = node + parent = parents.get(child) + while isinstance(parent, ast.Attribute) and parent.value is child: + child = parent + parent = parents.get(child) + return ( + isinstance(parent, ast.Call) + and parent.func is child + and dotted(child) in SAFE_PYTEST_CALLS + ) + + def unsafe_sys_modules_reference(node: ast.expr) -> bool: + """Allow only the suite's literal ``_toy`` module-loader accesses.""" + + if dotted(node) != "sys.modules": + return False + parent = parents.get(node) + if isinstance(parent, ast.Subscript) and parent.value is node: + return not ( + isinstance(parent.slice, ast.Constant) and parent.slice.value == "_toy" + ) + if isinstance(parent, ast.Compare): + operands = (parent.left, *parent.comparators) + return not any( + isinstance(operand, ast.Constant) and operand.value == "_toy" + for operand in operands + ) + return True + + class ModuleBindingScan(ast.NodeVisitor): + """Find bindings executed in the module namespace, not function bodies.""" + + pytestmark = False + pytest_rebound = False + + def _binding(self, name: str | None) -> None: + if name == "pytestmark": + self.pytestmark = True + elif name == "pytest": + self.pytest_rebound = True + + def visit_Name(self, node: ast.Name) -> None: + if isinstance(node.ctx, ast.Store | ast.Del): + self._binding(node.id) + + def visit_Attribute(self, node: ast.Attribute) -> None: + if isinstance(node.ctx, ast.Store | ast.Del): + self._binding(node.attr) + if rooted_at(node.value, "pytest"): + self.pytest_rebound = True + self.generic_visit(node) + + def visit_Subscript(self, node: ast.Subscript) -> None: + if ( + isinstance(node.ctx, ast.Store | ast.Del) + and isinstance(node.slice, ast.Constant) + and isinstance(node.slice.value, str) + ): + self._binding(node.slice.value) + if isinstance(node.ctx, ast.Store | ast.Del) and rooted_at( + node.value, "pytest" + ): + self.pytest_rebound = True + self.generic_visit(node) + + def _visit_definition_expressions( + self, node: ast.FunctionDef | ast.AsyncFunctionDef + ) -> None: + self._binding(node.name) + for decorator in node.decorator_list: + self.visit(decorator) + self.visit(node.args) + if node.returns is not None: + self.visit(node.returns) + for type_parameter in getattr(node, "type_params", ()): + self.visit(type_parameter) + # Inspect deferred bodies conservatively too. A helper invoked at + # module scope can install pytestmark through ``global`` and a test + # body can alias a runtime suppressor. Static control-flow analysis + # cannot prove those bodies harmless, so bindings there fail closed. + for statement in node.body: + self.visit(statement) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._visit_definition_expressions(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._visit_definition_expressions(node) + + def visit_Lambda(self, node: ast.Lambda) -> None: + # Defaults are evaluated in the surrounding scope; the body is not. + self.visit(node.args) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self._binding(node.name) + # A class is independently refused below. Only its expressions that + # execute in the surrounding scope need binding inspection here. + for expression in (*node.decorator_list, *node.bases): + self.visit(expression) + for keyword in node.keywords: + self.visit(keyword.value) + for type_parameter in getattr(node, "type_params", ()): + self.visit(type_parameter) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + self._binding(node.name) + self.generic_visit(node) + + def visit_MatchAs(self, node: ast.MatchAs) -> None: + self._binding(node.name) + self.generic_visit(node) + + def visit_MatchStar(self, node: ast.MatchStar) -> None: + self._binding(node.name) + + def visit_alias(self, node: ast.alias) -> None: + bound = node.asname or node.name.split(".", 1)[0] + if bound == "pytestmark": + self.pytestmark = True + if bound == "pytest" and node.name != "pytest": + self.pytest_rebound = True + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + for alias in node.names: + bound = alias.asname or alias.name + if bound == "pytestmark": + self.pytestmark = True + if bound == "pytest": + self.pytest_rebound = True + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "pytest" and alias.asname not in (None, "pytest"): + problems.append( + f"{file}: pytest is imported under an alias ({alias.asname})" + ) + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith( + "pytest" + ): + problems.append( + f"{file}: 'from {node.module} import ...' hides marker spellings; " + "import pytest itself" + ) + if ( + isinstance(node, ast.ImportFrom) + and (node.module or "").startswith("unittest") + and any(alias.name == "SkipTest" for alias in node.names) + ): + problems.append( + f"{file}: importing unittest SkipTest directly or under an alias " + "is not allowed" + ) + if ( + isinstance(node, ast.Call) + and dotted(node.func).endswith("param") + and any(keyword.arg == "marks" for keyword in node.keywords) + ): + problems.append( + f"{file}: pytest.param(..., marks=...) is not allowed anywhere in " + "an acceptance file" + ) + if isinstance(node, ast.Attribute) and dotted(node) in PYTEST_SUPPRESSORS: + problems.append( + f"{file}: references {dotted(node)}, whose result could be aliased" + ) + if ( + isinstance(node, ast.Attribute) and dotted(node) in UNITTEST_SUPPRESSORS + ) or ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id == "SkipTest" + ): + problems.append( + f"{file}: references {dotted(node)}, which suppresses a test result" + ) + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id == "pytest" + and not inside_direct_mark_decorator(node) + and not inside_direct_safe_pytest_call(node) + ): + problems.append( + f"{file}: references pytest outside a direct pytest.mark decorator; " + "the object could be aliased or mutated" + ) + dynamic_name = dotted(node) + if ( + isinstance(node, ast.Name | ast.Attribute) + and isinstance(node.ctx, ast.Load) + and ( + dynamic_name in DYNAMIC_NAMESPACE_REFERENCES + or unsafe_sys_modules_reference(node) + ) + ): + problems.append( + f"{file}: dynamic module namespace access through " + f"{dynamic_name} is not allowed" + ) + if isinstance(node, ast.ClassDef): + problems.append( + f"{file}: class {node.name} — tests must be module-level functions" + ) + bindings = ModuleBindingScan() + bindings.visit(tree) + if bindings.pytestmark: + problems.append(f"{file}: module-level pytestmark is not allowed") + if bindings.pytest_rebound: + problems.append(f"{file}: module code rebinds pytest, so marks are untrusted") + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + for decorator in node.decorator_list: + name = mark_name(decorator) + if name is None: + # Pytest and unittest both honor arbitrary decorator + # aliases on collected tests. If the spelling is not a + # direct ``pytest.mark.`` expression, the static + # scanner cannot prove that it is non-suppressing, so the + # acceptance ratchet must fail closed. + if node.name.startswith("test_"): + target = ( + decorator.func + if isinstance(decorator, ast.Call) + else decorator + ) + spelling = dotted(target) or type(target).__name__ + problems.append( + f"{file}::{node.name} carries unrecognized decorator " + f"{spelling!r}, which the ratchet cannot prove safe" + ) + continue + if name not in ALLOWED_MARKS: + problems.append( + f"{file}::{node.name} carries mark {name!r}, which is not allowed" + ) + if name == "parametrize" and isinstance(decorator, ast.Call): + cases = decorator.args[1] if len(decorator.args) > 1 else None + if not isinstance(cases, ast.List | ast.Tuple): + problems.append( + f"{file}::{node.name} parametrizes over a non-literal; " + "cases must be written inline" + ) + for argument in ast.walk(decorator): + if ( + isinstance(argument, ast.keyword) + and argument.arg == "marks" + ): + problems.append( + f"{file}::{node.name} smuggles marks through pytest.param" + ) + break + for inner in ast.walk(node): + if ( + inner is not node + and isinstance(inner, ast.FunctionDef | ast.AsyncFunctionDef) + and inner.name.startswith("test_") + ): + problems.append( + f"{file}::{node.name} nests {inner.name}, which pytest would " + "not collect" + ) + if isinstance(node, ast.Call) and dotted(node.func) in SUPPRESSING_CALLS: + problems.append( + f"{file}: runtime {dotted(node.func)}() suppresses a result" + ) + return tuple(problems) + + def tests_in(source: str, file: str = "") -> dict[str, str]: """Charter id to test name, for every ``test__...`` in ``source``.""" named: dict[str, str] = {} - for node in ast.walk(ast.parse(source, filename=file)): + for node in ast.parse(source, filename=file).body: if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): continue match = TEST_ID.match(node.name) @@ -156,6 +521,37 @@ def counts(files: tuple[str, ...]) -> dict[str, tuple[Marker, ...]]: } +def baseline_suite_files(ref: str) -> tuple[str, ...]: + """The acceptance files as of ``ref``, including ones since deleted.""" + # ``git ls-tree`` takes literal paths, not globs: list the directory and + # match the file pattern here. + result = subprocess.run( + [ + "git", + "ls-tree", + "-r", + "--name-only", + ref, + "--", + str(Path(SUITE_GLOB).parent), + ], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + return () + pattern = Path(SUITE_GLOB).name + return tuple( + sorted( + line + for line in result.stdout.splitlines() + if line and fnmatch.fnmatch(Path(line).name, pattern) + ) + ) + + def baseline_source(ref: str, file: str) -> str | None: """``file`` as of ``ref``, or ``None`` when it did not exist there.""" result = subprocess.run( @@ -290,24 +686,97 @@ def verify(ref: str = BASELINE_REF) -> int: for entry in data["properties"]: if entry["state"] == "missing": problems.append(f"charter {entry['id']} has no test in the suite") + for file in sorted(current): + problems.extend(suppressions_in((ROOT / file).read_text(), file)) + + declared = set(charter_ids((ROOT / CHARTER).read_text())) + seen: dict[str, str] = {} + for file in sorted(current): + for marker in current[file]: + if not marker.charter_id: + continue + if marker.charter_id not in declared: + problems.append( + f"{file}::{marker.test} names {marker.charter_id}, which " + f"{CHARTER} does not list" + ) + # The reason is free text; the test name is the binding. A marker + # whose reason names one property while sitting on another's test + # would let a re-red hide behind a known red, so the two must agree + # and each property may carry one marker. + named = TEST_ID.match(marker.test) + if named is None or named.group(1).upper() != marker.charter_id: + problems.append( + f"{file}::{marker.test} claims charter {marker.charter_id} " + "but is not that property's test" + ) + if marker.charter_id in seen: + problems.append( + f"{marker.charter_id} carries two markers: {seen[marker.charter_id]} " + f"and {file}::{marker.test}" + ) + seen.setdefault(marker.charter_id, f"{file}::{marker.test}") if not fetch_baseline(ref): print(f"baseline={ref} unavailable; the ratchet did not run") else: print(f"baseline={ref}") + # The ratchet is on property identities, not counts: a property that + # is green on the baseline and red now is a re-red, whatever else + # went green, whichever file the marker sits in. A property the + # charter gained since the baseline is committed red first (the + # charter's meta-TDD rule), so its marker is not a re-red. + baseline_charter = baseline_source(ref, CHARTER) + known = set(charter_ids(baseline_charter)) if baseline_charter else set() + new_to_charter = declared - known + # A property the baseline charter listed cannot simply vanish: that + # would drop it from scoring (and let a green id be renamed into a + # "new" red one). Retiring a property is a charter change of its own. + for identifier in sorted(known - declared): + problems.append( + f"charter {identifier} was listed on {ref} but is gone from " + f"{CHARTER}; retiring a property needs its own reviewed change" + ) + was_red: set[str] = set() + for file in baseline_suite_files(ref): + source = baseline_source(ref, file) + if source is not None: + was_red |= {m.charter_id for m in markers_in(source, file)} for file in sorted(current): source = baseline_source(ref, file) + ids = {m.charter_id for m in current[file] if m.charter_id} + new_reds = sorted(ids & new_to_charter) + suffix = ( + f" (+{len(new_reds)} new: {', '.join(new_reds)})" if new_reds else "" + ) if source is None: - print(f" [new] {file}: {len(current[file])}") + print(f" [new] {file}: {len(current[file])}{suffix}") continue was = len(markers_in(source, file)) - now = len(current[file]) - print(f" {'rose' if now > was else 'ok':<6} {file}: {was} -> {now}") - if now > was: - problems.append( - f"{file} re-reds {now - was} propert" - f"{'y' if now - was == 1 else 'ies'} ({was} -> {now})" + now = len(current[file]) - len(new_reds) + print( + f" {'rose' if now > was else 'ok':<6} {file}: {was} -> {now}{suffix}" + ) + now_red = {m.charter_id for file in current for m in current[file]} + re_reds = sorted((now_red & declared) - was_red - new_to_charter) + if re_reds: + where = { + identifier: sorted( + file + for file in current + if any(m.charter_id == identifier for m in current[file]) ) + for identifier in re_reds + } + problems.append( + f"re-reds {len(re_reds)} propert" + f"{'y' if len(re_reds) == 1 else 'ies'}: " + + ", ".join( + f"{identifier} (green on {ref}; red in " + f"{', '.join(where[identifier])})" + for identifier in re_reds + ) + ) if problems: print() diff --git a/tools/graph_parity_fixtures.py b/tools/graph_parity_fixtures.py index 00adc9445..8b0c33681 100644 --- a/tools/graph_parity_fixtures.py +++ b/tools/graph_parity_fixtures.py @@ -9,6 +9,7 @@ from collections.abc import Mapping, Sequence from importlib import metadata as importlib_metadata from pathlib import Path +from tempfile import TemporaryDirectory import numpy as np import pandas as pd @@ -28,6 +29,7 @@ from microcosm.frame.kernels import SimulateRulesKernel from microcosm.graph import ( Capabilities, + ContentStore, Determinism, Graph, KernelBase, @@ -42,8 +44,11 @@ SourceRef, StructuralDelta, WeightTransition, + compile_graph, graph_to_json, + run_graph, ) +from microcosm.graph.keys import platform_fingerprint ROOT = Path(__file__).resolve().parents[1] FIXTURES = ( @@ -82,9 +87,7 @@ def run(self, context: KernelContext) -> KernelResult: case = context.params.get("case") if case not in {"fit.qrf", "calibrate", "simulate"}: raise ValueError(f"Unknown parity fixture case {case!r}.") - table = pd.read_csv( - context.sources["fixture"] / "inputs.csv", float_precision="round_trip" - ) + table = pd.read_csv(context.sources["fixture"], float_precision="round_trip") frame = _frame_for_case(str(case), table) return KernelResult(frame=frame, receipt={"case": case, "rows": len(table)}) @@ -437,7 +440,12 @@ def _simulate_case() -> tuple[Graph, pd.DataFrame, pd.DataFrame, object, None]: ) -def _pins(node_id: str, kernel: object, seed: int | None) -> dict[str, object]: +def _pins( + node_id: str, + node_key: str, + kernel: object, + seed: int | None, +) -> dict[str, object]: capabilities = kernel.capabilities # type: ignore[attr-defined] dependencies = { name: importlib_metadata.version(name) @@ -445,10 +453,13 @@ def _pins(node_id: str, kernel: object, seed: int | None) -> dict[str, object]: } return { "node": node_id, + "node_key": node_key, "seed": seed, "kernel": kernel.ref, # type: ignore[attr-defined] "implementation_hash": kernel.implementation_hash(), # type: ignore[attr-defined] "dependencies": dependencies, + "numeric": capabilities.numeric.value, + "platform": platform_fingerprint(), } @@ -465,18 +476,92 @@ def _write_case( (destination / "graph.json").write_text(graph_to_json(graph), encoding="utf-8") inputs.to_csv(destination / "inputs.csv", index=False, lineterminator="\n") direct.to_csv(destination / "direct.csv", index=False, lineterminator="\n") - (destination / "pins.json").write_text( - json.dumps( - _pins(graph.nodes[-1].id, kernel, seed), - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), + node_id = graph.nodes[-1].id + with TemporaryDirectory(prefix="microcosm-parity-") as store_path: + manifest = run_graph( + compile_graph(graph), + sources={"fixture": destination / "inputs.csv"}, + store=ContentStore(Path(store_path)), + kernels=parity_registry(), + resume="forbid", + decisions=(), ) + node_key = manifest.nodes[node_id].key + pins = _pins(node_id, node_key, kernel, seed) + pins["platforms"] = { + platform_fingerprint(): {"node_key": node_key, "direct": "direct.csv"} + } + _write_pins(destination, pins) + + +def _write_pins(destination: Path, pins: dict[str, object]) -> None: + (destination / "pins.json").write_text( + json.dumps(pins, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8", ) +def _platform_slug() -> str: + return platform_fingerprint().replace("/", "-").replace(".", "_") + + +def add_platform_pin(name: str) -> str: + """Record this platform's node key and direct bytes for one existing case. + + A platform-bitwise kernel (amendment 16) is byte-identical only within a + platform, so every platform that CI runs carries its own pin: the node key + the fingerprint produces and the direct call's bytes on that platform. The + authoring platform's top-level pin and ``direct.csv`` are left untouched. + """ + + builders = { + "fit.qrf": _fit_case, + "calibrate": _calibrate_case, + "simulate": _simulate_case, + } + graph, inputs, direct, kernel, seed = builders[name]() + destination = FIXTURES / name + pins = json.loads((destination / "pins.json").read_text(encoding="utf-8")) + if pins["kernel"] != kernel.ref: # type: ignore[attr-defined] + raise SystemExit( + f"{name}: pins are for {pins['kernel']}, builder gives {kernel.ref}" + ) # type: ignore[attr-defined] + stored_inputs = pd.read_csv( + destination / "inputs.csv", float_precision="round_trip" + ) + if not stored_inputs.equals(inputs.reset_index(drop=True)): + raise SystemExit( + f"{name}: inputs.csv no longer matches the builder; regenerate instead" + ) + fingerprint = platform_fingerprint() + if fingerprint == pins["platform"]: + raise SystemExit( + f"{name}: {fingerprint} is the authoring platform; regenerate instead" + ) + with TemporaryDirectory(prefix="microcosm-parity-") as store_path: + manifest = run_graph( + compile_graph(graph), + sources={"fixture": destination / "inputs.csv"}, + store=ContentStore(Path(store_path)), + kernels=parity_registry(), + resume="forbid", + decisions=(), + ) + node_key = manifest.nodes[pins["node"]].key + relative = f"platforms/{_platform_slug()}/direct.csv" + (destination / relative).parent.mkdir(parents=True, exist_ok=True) + direct.to_csv(destination / relative, index=False, lineterminator="\n") + platforms = dict(pins.get("platforms", {})) + platforms.setdefault( + pins["platform"], {"node_key": pins["node_key"], "direct": "direct.csv"} + ) + platforms[fingerprint] = {"node_key": node_key, "direct": relative} + pins["platforms"] = platforms + _write_pins(destination, pins) + return fingerprint + + def generate() -> None: """Regenerate all H1 kernel fixtures deterministically.""" @@ -496,7 +581,18 @@ def generate() -> None: _write_case(name, graph, inputs, direct, kernel, seed) -def main() -> int: +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if args[:1] == ["platform-pin"]: + if len(args) != 2: + raise SystemExit("usage: graph_parity_fixtures.py platform-pin ") + os.environ["POPULACE_FIT_N_JOBS"] = "1" + os.environ["POPULACE_FIT_PREDICT_WORKERS"] = "1" + fingerprint = add_platform_pin(args[1]) + print(f"recorded {args[1]} pin for {fingerprint}") + return 0 + if args: + raise SystemExit("usage: graph_parity_fixtures.py [platform-pin ]") generate() print(f"wrote deterministic H1 fixtures under {FIXTURES.relative_to(ROOT)}") return 0 diff --git a/tools/graph_qrf_platform_probe.py b/tools/graph_qrf_platform_probe.py new file mode 100644 index 000000000..549ef79e1 --- /dev/null +++ b/tools/graph_qrf_platform_probe.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Measure how far ``fit.qrf@1`` draws move between platforms. + +Run once per platform (for example natively on arm64 and under Rosetta from +an x86_64 environment synced to the same lock):: + + uv run python tools/graph_qrf_platform_probe.py + +Each run writes ``/.json`` with every drawn value for 20 +seed-by-regime cases; ``--compare `` then reports the maximum +absolute, relative, and ulp movement and how many cells differ. The +2026-09-03 result is recorded in ``docs/graph-qrf-cross-platform.md`` and is +why the kernel declares ``Numeric.PLATFORM_BITWISE`` (amendment 16). +""" + +import json +import pathlib +import platform +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import fit as fit_qrf + + +def make(regime: str, seed: int, n_donor=600, n_rec=300): + rng = np.random.default_rng(1000 + seed) + x1 = rng.normal(size=n_donor + n_rec) + x2 = rng.integers(0, 5, size=n_donor + n_rec).astype(float) + if regime == "positive": + y = np.exp(0.5 * x1 + 0.1 * x2 + rng.normal(scale=0.3, size=n_donor + n_rec)) + elif regime == "mixed_sign": + y = 3.0 * x1 - 0.7 * x2 + rng.normal(scale=1.0, size=n_donor + n_rec) + elif regime == "near_ties": + y = np.round(x1, 1) + rng.choice([0.0, 1e-9, -1e-9], size=n_donor + n_rec) + elif regime == "zero_inflated": + y = np.where( + rng.random(n_donor + n_rec) < 0.6, + 0.0, + np.abs(rng.normal(size=n_donor + n_rec)) * 1e4, + ) + else: + raise ValueError(regime) + donors = pd.DataFrame({"x1": x1[:n_donor], "x2": x2[:n_donor], "y": y[:n_donor]}) + recipients = pd.DataFrame({"x1": x1[n_donor:], "x2": x2[n_donor:]}) + return donors, recipients + + +def main(out_dir): + results = {} + for regime in ("positive", "mixed_sign", "near_ties", "zero_inflated"): + for seed in range(5): + donors, recipients = make(regime, seed) + model = fit_qrf(donors, ["x1", "x2"], ["y"], weights="none", seed=seed) + drawn = model.predict(recipients) + values = np.asarray(drawn["y"], dtype=np.float64) + results[f"{regime}/{seed}"] = [float(v) for v in values] + arch = platform.machine() + with open(f"{out_dir}/{arch}.json", "w") as f: + json.dump( + {"arch": arch, "python": sys.version.split()[0], "results": results}, f + ) + print("wrote", arch, len(results), "cases") + + +def compare(out_dir: str) -> None: + files = sorted(pathlib.Path(out_dir).glob("*.json")) + files = [f for f in files if f.stem not in {"summary"}] + if len(files) != 2: + raise SystemExit( + f"expected two platform files under {out_dir}, found {len(files)}" + ) + a = json.load(open(files[0]))["results"] + b = json.load(open(files[1]))["results"] + worst = {"abs": 0.0, "rel": 0.0, "ulps": 0, "cells": 0, "differing": 0} + for key in a: + x = np.asarray(a[key]) + y = np.asarray(b[key]) + diff = np.abs(x - y) + rel = diff / np.maximum(np.abs(y), 1e-300) + ulps = [ + abs(int(np.float64(u).view(np.int64)) - int(np.float64(v).view(np.int64))) + for u, v in zip(x, y, strict=True) + ] + moved = int((diff > 0).sum()) + case_abs = float(diff.max()) + case_rel = float(rel[diff > 0].max()) if moved else 0.0 + # The int64-view distance is a ulps count only while both values share + # a sign and a binade; when a quantile draw lands on a different donor + # it is just a large integer, reported for completeness and never + # interpreted as a bound. + case_view_distance = max(ulps) + worst["abs"] = max(worst["abs"], case_abs) + worst["rel"] = max(worst["rel"], case_rel) + worst["ulps"] = max(worst["ulps"], case_view_distance) + worst["cells"] += len(x) + worst["differing"] += moved + if moved: + print( + f"{key}: max_abs={case_abs:.3e} max_rel={case_rel:.3e} " + f"int64_view_distance={case_view_distance} differing={moved}/{len(x)}" + ) + print(f"{files[0].stem} vs {files[1].stem}:", worst) + + +if __name__ == "__main__": + if sys.argv[1] == "--compare": + compare(sys.argv[2]) + else: + main(sys.argv[1])