diff --git a/.changeset/eql-json-tree-ordinal-chain.md b/.changeset/eql-json-tree-ordinal-chain.md new file mode 100644 index 000000000..f52b3ec4e --- /dev/null +++ b/.changeset/eql-json-tree-ordinal-chain.md @@ -0,0 +1,11 @@ +--- +"@cipherstash/eql": minor +--- + +EQL v4 `json_tree` (unreleased): the token chain is derived by ordinal and the walk probes to the first miss. + +A stored entry now carries four fields, `s`, `c`, `p` and `op`. The value selector `vs` lives only in the document's projection `q`, which the engine now reads back as the projection instead of rebuilding it from the entries; the group tags `n` and `g` are gone with it, since nothing read them once `q` was stored. The wrapped token `w` and the ordinal `o` are gone too: a child's token is `H("t" ‖ t(parent) ‖ LE32(o))`, which the server derives as it walks, and the ordinals under a node are dense from zero, so the walk probes them in turn and stops at the first miss. A subtree of `N` nodes costs about `2N` probes (a SHA-256 and a lookup each) and `N` token hashes whatever the document's width, where it cost a probe for every ordinal up to the document's widest node at every node visited. The entry validator accepts the four fields only, so a document written by the earlier build of this unreleased family is refused. + +The client can derive a token only for a chain head and the array positions that directly follow it, since an object member's ordinal is a rank among its siblings' keys. `JsonSubtreeKey::gates()` gives the rest of a path as one gate per step, a selector or `#n` for an array position, and `eql_v4.json_tree_subtree_at(doc, token, gates text[], budget DEFAULT 64)` walks them, replacing the earlier `json_tree_subtree_at(doc, token, selector, budget)`. `json_tree_children` and `json_tree_child_count` take the same `gates`, optional and empty by default. A gated walk returns the node the gates reach with `d = 0` and its descendants beneath it, so a wildcard reaches several nodes and opens as a list; `json_tree_subtree(doc, token, budget)` is now `json_tree_subtree_at` with no gates and still returns the descendants of the token's node from `d = 1`. Past the budget a walk still returns the sealed root alone, with `j = 0` and `d = 0`. `eql_v4_internal.json_tree_unwrap`, `json_tree_xor16` and `json_tree_fanout` are removed; `eql_v4_internal.json_tree_child_token` is added. + +Measured on 10,000 and 100,000 rows of a nested corpus (about 213 entries a document), query-only medians: subtree walks 30 to 55% faster (a 122-entry subtree walked out, 23 ms to 11 ms), `json_tree_children` and `json_tree_child_count` 6 to 9%, a stored nested document 25% smaller (heap 810 MB to 621 MB at 10,000 rows), containment and field queries unchanged within the run's noise. diff --git a/.changeset/eql-json-tree-walk-and-projection.md b/.changeset/eql-json-tree-walk-and-projection.md index 0c7190f91..2357b8115 100644 --- a/.changeset/eql-json-tree-walk-and-projection.md +++ b/.changeset/eql-json-tree-walk-and-projection.md @@ -11,3 +11,5 @@ Also: `eql_v4.json_tree_path_query_first`, the function `->` and the ordering comparisons extract through, is costed at 1000 so a prepared statement's generic plan keeps the functional btree on `eql_v4.ord_term(col -> sel)` instead of filtering a sequential scan (17 ms down to 0.1 ms after the fifth execution). `eql_v4_internal.json_tree_group`, the server-side projection builder, is removed. The `eql-bindings` crate's `JsonTreeDocument` gains the `q` field and `JsonTreeSubtree` becomes a struct with `rows` and an optional `path`; `stack-encrypt`'s `JsonDocument` writes `q` on the wire and ignores it on the way in, `JsonPath::select` evaluates a path over a value, and `JsonSubtree::at`, `path` and `is_sealed_root` carry the fallback. EQL v4 only; the v3 bundle is unchanged. + +**The leaf ordering term `op` is opt-in per path.** A stored leaf carried the order-preserving term on every number and string, about 80 bytes each and the largest field of a nested document, whether or not anything ordered by it. Now the JSON index's options name the paths that order (`{"json": {"ordered": ["$.age", "$.orders[*].total"]}}` on the data plan; `JsonOptions::ordering([...])` in Rust), a wildcard covering every element of an array and a position one element, and no path is ordered by default. An ordering query at a path the index does not order is refused before any key loads rather than silently matching no row. The SQL is unchanged: `eql_v4.ord_term` was already NULL for a leaf without a term. `eql-bindings` gains `v4::JsonTree`, the `json_tree` layout under options, and `stack-encrypt` gains `target::Layout` and `Plan::value().encrypt_as(layout)`, a plan laid out by a value whose queries run under its options. diff --git a/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md b/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md index 007f2faf2..ca84c1cfd 100644 --- a/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md +++ b/docs/plans/2026-10-17-json-index-stack-encrypt-spike.md @@ -6,6 +6,8 @@ **Relates to:** #1231 (containment matches across array elements), #1209 (`Contains`, `ContainedBy`, `AtPath` declared), ADR-0009 (operators select the index), system ADR-0002 (EQL v4 by producer), `packages/eql/docs/plans/stevec-envelope-key-header-rfc.md` (the v3 envelope), `docs/plans/2026-10-04-plan-builder.md` ("The JSON index" under Additions) **Prototypes:** `2026-10-17-json-index-stack-encrypt-spike/` beside this file, three SQL files that run on a stock PostgreSQL 16 with no extension +> **Note (2026-10-22).** The stored form this document designs has changed twice since, and the implementation notes at the end record each change. As of Phase 3b a stored entry carries only `s`, `c`, `p` and `op`. The value selectors live only in the stored projection `q`. The group tags `n` and `g`, the wrapped token `w` and the stored ordinal (`i` here, `o` in the code) are gone. A child's token is derived from its parent's token and its ordinal, not its label, and the walk probes ordinals until the first miss. The sections below are kept as written, with short dated notes where they would otherwise mislead. + ## Summary Stack Encrypt has no JSON index yet. The operators exist (#1209 merged `op::Contains`, `op::ContainedBy` and `op::AtPath` on `next`), the plan-builder document reserves `Json: Index` as "a new core operation", and EQL v4 has a storage-only `public.eql_v4_json` but "no SteVec bindings yet". The v4 JSON wire format is therefore unconstrained by anything shipped. This document proposes what to build, and a spike to prove it. @@ -74,6 +76,8 @@ The flattening is unchanged in what it visits: every node, with arrays indexed u The value entry and its sentinel disappear: `vs` sits on the node's own entry. The entry count halves. `a` disappears: `n` marks an array element, and the chain enumerates an array's elements. +> **2026-10-22:** of the fields in this table a stored entry now keeps `s`, `p`, `c` and `op` (Phase 3b). `vs` moved to the stored projection `q` (Phase 3 stored it, Phase 3b removed it from the entries), and `w`, `n`, `g` and the ordinal are not stored. + What the client gets back for a scalar path (`doc -> 'sel'` where the leaf is a string) is unchanged in shape: one entry, decryptable with the grafted `h`, its own `s` and `c`, and its ordinal `j` (grafted by the extractor, as `h` is today). For a container path the client gets the subtree's entries and rebuilds the value from each entry's decrypted `(label, kind, scalar)` and the depth the walk reports. Whole-document decryption is one AEAD on the root entry when the sealed-root option (next section, on by default) is set, and otherwise the same walk, run by the client over the whole `sv` with the root token it can derive itself. ### 2. The token chain @@ -86,6 +90,8 @@ p(child) = H("p" ‖ iv ‖ t(parent))[..16] stored w(child) = t(child) XOR H("w" ‖ iv ‖ t(parent) ‖ c(child))[..16] stored ``` +> **2026-10-22:** superseded by Phase 3b. A child's token is `H("t" ‖ t(parent) ‖ LE32(o))`, keyed by its ordinal rather than its label, and nothing stores it: the server derives it as it walks, so there is no `w`, no unwrapping and no XOR, and the parent tag is the position-keyed one below. The client can then derive a token only for a chain head and the array positions that directly follow it, and sends gates for the rest of a path. The recursive query, the paragraph on the mask, and "Wildcard paths" below describe the original design. + `H` is SHA-256. Only `t(root)` needs the index key; everything below it is hashing, so the server can do its half with core functions. The tokens are a function of the column and the **positional** path, so the client derives a start token for any path it can name without a wildcard: `$.nested`, `$.a[2].b`. Labels are bound with a kind byte and as the final variable-length field, so `("ab", "c")` and `("a", "bc")` cannot collide. Server side, the subtree under a start token is one recursive query. From `token-chain.sql`, which runs as written on PostgreSQL 16: @@ -142,6 +148,8 @@ The sealed root caps how much a token unlocks. It does nothing about how long: a `p = H("p" ‖ iv ‖ t(parent))` is the same value on every child of a node, so sibling groups are visible to anyone holding the row, with no token. That is the main at-rest residue of the chain. Keying the tag by the child's position removes it: `p_i = H("p" ‖ iv ‖ t(parent) ‖ i)`, with the child's ordinal `i` stored in the clear on the entry. At rest an observer learns only "this is some node's `i`-th child". The walk changes from one hash per walked node to `F` hashes per walked node, where `F` is one more than the largest `i` in the document (public), followed by the same equality join as now; `W × F` hashes for a walk of `W` nodes, which Phase 3 measures. Array children take `i` from their index, which the client needs for reconstruction anyway. Object children take `i` from a per-document shuffled order, derived from the document key, so the ordinal does not rank key names. The storage cost is one small integer per entry. +> **2026-10-22:** the parent tag is still position-keyed, but since Phase 3b the ordinal is not stored and there is no `F`: the ordinals under a node are dense from zero, so the walk probes them in turn and stops at the first miss, about `2N` probes for a subtree of `N` nodes. + ### 3. Exact containment (#1231) **The gap, verified.** `jsonb-semantics.sql` pins PostgreSQL's own answers, and `containment-oracle.sql` runs a plaintext-selector simulation of the v3 flattening (the PRF is injective in practice, so string selectors give the same containment answers as MAC'd ones) beside it: @@ -186,6 +194,8 @@ OR (q.alt IS NOT NULL AND to_query(doc) @> q.alt) -- a second GIN probe, bitmap Then `to_query(a) @> to_query(b)` is exact for arrays of containers with no plpgsql at all: the `sv` arrays check selector presence, and each needle `el` object has to fit inside one stored `el` object, which is the same-element rule. The GIN `jsonb_path_ops` index still serves it, because GIN is lossy for `@>` and the executor rechecks the real operator on the heap tuple; and the projection is computed once per row inside the expression index, so a query pays nothing for it. `alt` stays as it is. Two ways to obtain the grouping server-side, with identical leakage: build the projection from the `g` and `n` tags (a plpgsql `to_query`, immutable, run at index time), or store `sv` itself nested by element, which removes the tags from the wire and makes `->` by selector a `$.**` jsonpath lookup. Phase 2 tries the projection first, measures both, and keeps `groups_ok` only if neither engages the index. Whichever form wins, `n` and `g` are emitted only for elements that are **containers**: a scalar element carries one constraint, so grouping cannot be wrong for it, and the tags would reveal element membership for nothing. +> **2026-10-22:** the projection won, and it is stored rather than built: the client writes it beside `sv` as `q` (Phase 3), and since Phase 3b the entries carry no `n` or `g`. Element grouping is visible at rest through the nesting of `q` instead. The `grp` needle and `groups_ok` above were never built. + **Portability to v3.** The group tags and the recheck do not depend on the chain. If #1231 must also be fixed for rows cipherstash-client writes, the same `g`/`n` fields and the same `groups_ok` can be added to the v3 wire and SQL, with `g` derived from the document key instead of a token. This document does not propose that; it records that the fix is separable. ### 4. Wire format @@ -223,6 +233,8 @@ An `AtPath` needle is `{"sv": [{"vs": "…"}]}`. An `AtPath` term is th With the sealed-root option off, the root entry's `c` is the root's label and kind only, and the top-level entries carry `p` and `w` like every other node. The header records which shape the document has. +> **2026-10-22:** as built, a stored document is `{"v", "i", "h", "sv": [{s, c, p?, op?}], "q"}`, with no `k`; a containment needle is a projection, `{"sv", "el", "alt"?}`, with no `grp`; and a subtree request is a token and one gate per step below it. The size estimates below predate both; the measured sizes are under Phase 3b. + **Size.** Per 16-byte field, hex costs 32 characters plus about six of framing. Measured against the envelope RFC's fixture document (`{"hello":"world-1","nested":{"deep":"constant"},"number":1}`, 5 nodes): | | v3 today | proposed | @@ -262,6 +274,8 @@ On a flat five-node document the proposal is about 8% larger: the two chain fiel | the broadest token a legitimate query can send | none exist | with the sealed root: one top-level field's subtree. Without it: `t($)`, the whole column's shape | | how long a revealed token stays useful | not applicable | for as long as the column lives; epochs (recorded above, not scheduled) would bound it to rows written under the epoch it was issued for | +> **2026-10-22:** parent tags are position-keyed (Phase 1), so siblings do not share a `p` at rest. Since Phase 3b there is no `g`: which leaves sit in the same array element is visible at rest through the nesting of the stored projection `q` instead. + ### 7. Performance - **Encrypt:** per node two PRFs (`s`, `vs`), three hashes (`t`, `p`, mask), one small AEAD. v3 does two MACs and one AEAD over the subtree bytes. Fewer bytes encrypted at every depth above the leaves. @@ -271,10 +285,14 @@ On a flat five-node document the proposal is about 8% larger: the two chain fiel - **Decrypt a document:** with the sealed root, one AEAD on the root entry, as today. Without it, the client walks the chain locally: N hashes plus N small AEADs; the client has the index key, so no extra round trip either way. - **Encrypt, with the sealed root:** one more AEAD, over the whole document, beside the N small ones. +> **2026-10-22:** as built, encrypt is two PRFs per node (`s`, `vs`) and up to three hashes (`t`, `p`, and an object member's rank), with no mask. The walk reads the document once into a map keyed by parent tag (Phase 3) and probes it, about `2N` probes and `3N` hashes for a subtree of `N` nodes (Phase 3b); measurements are under Phase 3 and Phase 3b. + ### 8. Standard PostgreSQL Everything the server does: `sha256(bytea)` (core since PostgreSQL 11), `get_byte` / `set_byte` for a 16-byte XOR in plpgsql, `decode` / `encode`, `WITH RECURSIVE`, `jsonb_array_elements WITH ORDINALITY`, and `jsonb_path_ops` GIN. No pgcrypto, no superuser, no C. The same posture as the v3 surface Supabase runs today. +> **2026-10-22:** with no XOR since Phase 3b, the walk needs no `get_byte` / `set_byte`: it is `sha256`, `decode` / `encode`, a jsonb object lookup and `WITH RECURSIVE`. + ## Further mechanisms (added 2026-10-18) The four above are in the spike: the native-`@>` projection and container-only tags under containment, the sealed root and position-keyed parent tags under the token chain. Four more are recorded here, with what each buys and costs, so that none is rediscovered as free. @@ -309,6 +327,8 @@ The four above are in the spike: the native-`@>` projection and container-only t - **A build change, additive only.** `tasks/build.sh` renders `src/v3/**` as both bundles and `verify_installer_complete.sh` asserts every ordered v3 file is in the v4 installer. The new family needs the renderer to also order and render a v4-only source tree into the v4 bundle, and the completeness check to accept files that exist in one bundle only. Nothing is removed from either bundle, so this cannot block the v4 release: until the build change lands, the spike installs the new family as a separate file over the v4 bundle. - Tests: `mise run test:stack_encrypt_pg` on `next` already installs the v4 bundle and runs a Stack Encrypt PostgreSQL test with fresh encryption under a fake KMS and no credentials. The JSON suite joins it. The SQLx fixture machinery is not needed for this and should not be used: the point of the fake key source is that the whole suite runs on a fork PR. +> **2026-10-22:** as built, the walk functions are `eql_v4.json_tree_subtree(doc, token, budget)`, `json_tree_subtree_at(doc, token, gates, budget)`, `json_tree_children(doc, token, gates)` and `json_tree_child_count(doc, token, gates)`, and there is no `groups_ok`. + **`languages/golang`** - Nothing in the spike. After it, `JSON` moves from "the JSON index is a new operation in the engine" to producible in `encrypt/eql`'s table, and the data plan carries the `json` index with its options (plan-builder Additions, item 7). @@ -339,11 +359,13 @@ The index, the SQL and the live oracle landed in the pull request stacked on thi **Decisions taken by the code.** Of the ten listed below: 1, hex for every sixteen-byte field and base64 for `c` and `h`, pending Phase 3; 2, leaf ordering terms bound to the column only, numbers as the `f64` they round to and strings as text, under the scalar `Ope` index's derivation; 3, wildcard arrays only; 4, group tags on, and only on container array elements; 5, the child count is in every container node's ciphertext, and a truncated subtree is `OpenedType`, not a shorter array; 7, plain SHA-256; 8, the option is the boolean `sealed_root`, on by default, recorded in the header's flags; 9, the projection is built from the tags server-side (`eql_v4.json_tree_query`) and `groups_ok` was not needed; 10, position-keyed parent tags on, the ordinal stored as `o`, object members ranked by a hash under the parent's token. +> **2026-10-22:** 4 and 10 have since moved. The grouping lives only in the stored projection `q`, with no `n` or `g` on an entry (Phase 3 and 3b). Parent tags are still position-keyed and an object member's ordinal is still its rank by a hash under the parent's token, but the ordinal is no longer stored (Phase 3b). + **Deviations.** - *Nonces are random, not derived from the selector and ordinal.* Every entry seals through the engine's one AEAD path (`vitaminc-encrypt`'s AES-256-GCM with its own random 96-bit nonce, carried in `c`), so there is one leaf cipher in the crate and no second AES-GCM call site. Uniqueness under one document key is then probabilistic (`N² / 2^97` at `N` entries) rather than structural. The entry's AAD still binds its ordinal `j`, so a moved or reordered entry fails to open as the design requires; the cost is twelve bytes per entry, which Phase 3 should count. -- *The ordinal is `o`, not `i`*, because `i` is the envelope's identifier, which `->` grafts onto an extracted entry. An extracted entry also carries `j`, its ordinal in the document, which its AAD is bound to, and a walk adds `d`, its depth below the start. -- *The subtree walk returns a node's descendants and not the node itself.* The walk addresses nodes by their parent, and the design's wire has no field that names a node by its own token. The client reads the node's kind from its children's labels; a node with no descendants, an empty container or a scalar, yields no rows, and `->` on its selector returns the node. A self tag per node would close this at sixteen bytes an entry and is left to Phase 4. +- *The ordinal is `o`, not `i`*, because `i` is the envelope's identifier, which `->` grafts onto an extracted entry. An extracted entry also carries `j`, its ordinal in the document, which its AAD is bound to, and a walk adds `d`, its depth below the start. (2026-10-22: since Phase 3b `o` is not stored at all.) +- *The subtree walk returns a node's descendants and not the node itself.* The walk addresses nodes by their parent, and the design's wire has no field that names a node by its own token. The client reads the node's kind from its children's labels; a node with no descendants, an empty container or a scalar, yields no rows, and `->` on its selector returns the node. A self tag per node would close this at sixteen bytes an entry and is left to Phase 4. (2026-10-22: a gated walk, `json_tree_subtree_at`, now returns the node its gates reach at `d = 0`, with no self tag; the ungated `json_tree_subtree` still returns descendants only.) - *Containment is not `STRICT` in SQL.* PostgreSQL inlines a `STRICT` SQL function only when it can prove the body is NULL for a NULL argument, and the `OR` that adds the needle's `alt` probe is not provably NULL; un-inlined, the call was an opaque filter and the GIN index never engaged. `eql_v4.json_tree_contains` and the `@>` / `<@` wrappers over it are declared without `STRICT` and return NULL for a NULL side on their own. - *`<@` cannot match a stored root scalar inside a needle root array*, since a stored document carries no `alt` pair. The oracle records the case; `@>` is exact. - *Value selectors compare strings byte for byte*, with no Unicode normalization, because `jsonb @>` does, and the oracle is agreement with `jsonb @>`. The scalar equality index normalizes to NFC; a JSON value does not. Numbers canonicalize by value, so `2` and `2.0` are one value, as `jsonb` compares them. @@ -362,8 +384,82 @@ The family was measured at 10,000 rows in the `cipherstash/benches` repository ( Still open from the measurements, filed as a follow-up: `->` scans `sv` on every extraction (0.8 ms on a nested document; an index of `sv` by `s` would make it a lookup); `json_tree_contains` plans a second GIN probe for `alt` even when the needle carries none; `ORDER BY ord_term(col -> sel)` measured 0.92 ms against v3's 0.25 ms on an index scan both ways, unexplained; and the nested document is 24 times its plaintext (hex tags, a random nonce per entry, the sealed-root copy), which decisions 1 and 6 above and the opaque-subtree policy are the levers on. +## Implementation notes (2026-10-22, Phase 3b: the ordinal-chained walk) + +A review of the stored form after the Phase 3 measurements found three fields nothing read and two the walk did not need, and replaced the chain step. The wire is now `{"v", "i", "h", "sv": [{s, c, p?, op?}], "q"}`. + +- **`vs`, `n` and `g` are gone from the entries.** Once `q` was stored (Phase 3), no SQL function read an entry's value selector or group tags: containment reads `q` alone, and the engine read them only to rebuild `q` on deserialize. The engine now takes the stored `q` as the projection. That is no change in trust: `q` and the entry fields were equally unauthenticated (only `c` is, through its AAD), and a tampered `q` changes which rows a containment query returns and nothing else. On the nested benchmark document (213 entries on average) the values of those three fields were 15% of `sv`. +- **Tokens chain by ordinal.** `t(child) = H("t" ‖ t(parent) ‖ LE32(o))` replaces the label-derived step. The server derives every hit's token from its parent's as it walks, so `w` (the wrapped token) is not stored. The cost is that the client cannot derive an object member's token, because an ordinal is a rank among sibling keys it does not know. It derives a chain head's token and those of the array positions that directly follow the head, and sends one gate per further step (`JsonSubtreeKey::gates()`: the step's selector, or `#n` for a position); `json_tree_subtree_at(doc, token, gates, budget)` descends through them, and `json_tree_children` and `json_tree_child_count` take the same gates. A wildcard step's selector is every element's, which is the old `subtree_at` gate generalised to every depth. +- **The walk probes to the first miss.** The ordinals under a node are dense from zero (positions for arrays, a permutation for objects), so the walk tries `0, 1, 2, …` and stops at the first miss. `o` is not stored and `json_tree_fanout` is gone. A subtree of `N` nodes costs about `2N` probes (`N` hits, and a miss for every node visited), each a SHA-256 and a map lookup, plus a SHA-256 per hit for its token, where it cost `F` probes per visited node, `F` being the widest node in the whole document; on the nested corpus a wide `items` array taxed every object walk in the same document. +- **Depth is reported from the reached node.** A gated walk returns the node its gates reach with `d = 0` and its descendants beneath it, so the client can tell "the value at the path" (one reached node) from "the children of a node" (no `d = 0` row) without the path. A wildcard reaches several nodes and opens as a list. The ungated `json_tree_subtree(doc, token)` is now `json_tree_subtree_at` with no gates, and returns the token's node's descendants from `d = 1` as before. +- **Leakage.** At rest, the same as Phase 3 less what the stored ordinals told: their multiset gave, for every `k`, how many nodes in the chain have more than `k` children. `p` is still salted by the document `iv`. Under a token, the same: a token still unlocks the shape of that subtree in every row. The `w` mask's binding to `c` (a swapped ciphertext broke the walk on the server) is gone; the swap still fails when the client opens the entry, because `c` is bound to `s` and `j` in its AAD. + +### Measured (cipherstash/benches 2d9d5f2) + +Measured in `cipherstash/benches` (branch `feat/json-tree-benches`, commit 2d9d5f2, `report/JSON_TREE_COMPARISON.md`, "Third measurement") on the machine of the second Phase 3 measurement (Apple M1 Max, PostgreSQL 17.10), against the same documents: the ingest resealed the plaintext each row already held, so every scenario returned the same rows before and after. "Before" is the Phase 3 branch with its fixes, "after" this change. Query-only criterion medians over 10 samples, no decrypt, in milliseconds, on the nested corpus. + +| scenario | 10,000 rows | 100,000 rows | +|---|--:|--:| +| `subtree/orders`, default budget (falls back to the sealed root) | 9.48 → 5.71 (−40%) | 8.77 → 5.46 (−38%) | +| `subtree_at/items`, `$.orders[*].items`, default budget | 12.95 → 8.21 (−37%) | 10.92 → 7.18 (−34%) | +| `subtree/orders`, walked out (budget 0; 122 / 130 entries) | 22.97 → 10.92 (−52%) | 24.57 → 11.35 (−54%) | +| `subtree_at/items`, walked out (budget 0; 87 / 105 entries) | 23.12 → 10.93 (−53%) | 24.83 → 11.27 (−55%) | +| `subtree/company` | 2.84 → 1.99 (−30%) | 2.94 → 1.96 (−33%) | +| `subtree/user` | 3.90 → 2.38 (−39%) | 3.99 → 2.45 (−39%) | +| `children/orders` | 1.63 → 1.54 (−6%) | 1.64 → 1.49 (−9%) | +| `child_count/orders` | 1.62 → 1.53 (−6%) | 1.66 → 1.52 (−9%) | +| `path_query/wildcard` | 2.24 → 1.64 (−27%) | 1.80 → 1.41 (−22%) | +| `field_order/nested` | 4.81 → 3.81 (−21%) | 5.23 → 4.12 (−21%) | +| `extract/nested` | 0.59 → 0.47 (−19%) | 0.63 → 0.54 (−15%) | + +The walks the change was aimed at halved when walked out. Scenarios the walk does not touch improved too (`path_query`, `field_order`, `extract`): they read entries through `->` and the path functions, which scan an `sv` 30% shorter in a datum 24% smaller to decompress. `children` and `child_count` moved least; the report's explanation, not yet tested, is that both are dominated by the per-call fixed cost of detoasting the document and building the parent map over all its entries, which this change does not touch, and probing five to seven children instead of the document's widest fan-out saves only a few dozen hashes on top of it. That fixed cost is the next lever for every walk. + +Containment and field queries did not change beyond noise. The nested corpus's containment scenarios moved −12% to +7%. The flat corpus read +7% to +13% on the GIN-served containment and equality scenarios, but its operands, SQL, plans, projection and index are byte-identical before and after, and the native `jsonb` containment controls over unchanged plaintext, with freshly built indexes, drifted +5% to +19% in the same run; that points at the machine, not the format. The 100,000-row nested GIN build went from 58 s to 65 s on one sample, with the same projection and a smaller heap, which reads the same way. + +| size | before | after | +|---|--:|--:| +| nested document, characters (213 entries) | 113,458 | 85,157 (−25%) | +| nested `sv`, characters | 94,324 | 66,023 (−30%) | +| nested `q`, characters | 18,961 | 18,961 | +| nested `pg_column_size`, 10,000 rows | 81,264 B | 62,143 B (−24%) | +| nested heap with TOAST, 10,000 / 100,000 rows | 810 MB / 8,093 MB | 621 MB / 6,201 MB (−23%) | +| nested containment GIN, 10,000 / 100,000 rows | 40 MB / 358 MB | 40 MB / 358 MB | +| nested GIN build, 10,000 / 100,000 rows | 5 s / 58 s | 5 s / 65 s | +| flat document, characters (5 entries) | 2,346 | 2,136 (−9%) | +| flat `pg_column_size` | 2,452 B | 2,250 B (−8%) | + +`sv` shrank by 30% where 15% was expected from the three unread fields alone: the five removed fields carried 20,827 characters of values per nested document, and their keys and quotes about 7,500 more. The nested document is now 21 times its plaintext in characters (was 28) and 26 times in heap (was 34). The largest field is now `op`, the CLLW OPE term on the 156 number and string leaves: 24,832 characters a document. + +**Correctness.** A new benches task, `mise run check:v4:json_tree `, ran after ingest at both tiers. For 25 rows in a fixed pseudo-random order it decrypted every walk form the SQL returns and compared it with the row's plaintext: `json_tree_subtree` at `$.company`, `$.user` and `$.orders`, walked out and under the default budget (opening the sealed root at the path when the walk fell back); `json_tree_subtree_at` for `$.user.contact` (one gate) and `$.orders[*].items` (two gates); `json_tree_children` and `json_tree_child_count` at `$.orders`; every order by its position-derived token; `json_tree_path_query`; and `->`. For 10 sampled needles per predicate it compared the full id set of the encrypted containment, equality, range and ordering queries with native `jsonb` over the plaintext. 462 checks passed and none failed at each of 10,000 and 100,000 rows (350 walk checks, 112 predicate checks). + +## Implementation notes (2026-10-22, Phase 3c: the ordering term is opt-in per path) + +After the Phase 3b measurement the largest field in a stored document was `op`, the CLLW OPE term on every number and string leaf: 24,832 of the nested document's 85,157 characters, about 80 bytes of hex a leaf, on 156 leaves of which the benchmark's queries order two. An ordering term is also the one field that reveals something across rows at rest (the order of the leaf's value among every row that holds one), on fields nobody asked to order. "Further mechanisms" above proposed a per-path index policy with "everything indexed" as the default; the `op` half of it is now built, and the default is reversed: **no path is ordered unless the index says so.** + +- **`JsonOptions` gains `ordered`**, a set of `JsonPath`s, beside `sealed_root`. The walk derives a leaf's `op` only when some ordered path covers the node's positional path (`JsonPath::covers`): a key step matches the same key, a wildcard step covers any array position and the wildcard, a position covers that position alone. So `$.orders[*].total` orders the total of every order and `$.scores[0]` the first score only; literal positions and wildcards are both allowed. The set is part of the index's identity like `sealed_root`, and crosses the data-plan wire as `{"json": {"sealed_root": true, "ordered": ["$.age", "$.orders[*].total"]}}`, paths as text, written in canonical order. It is not recorded in the header: a stored leaf shows whether it carries a term. +- **An ordering query at a path the index does not order is refused** (`TermError::JsonPathNotOrdered`) when the query is awaited, before any keyset loads. Without the refusal, `ord_term` returns NULL for every stored leaf at the path and the comparison silently matches no row. A wildcard query path is covered by a wildcard only, since a position orders elements the wildcard would compare against. +- **A plan can be laid out by a value.** The typed plan had no place for options: `encrypt_into::()` reads the type's own `EncryptFrom`, which is static, so every typed plan sealed under the defaults and a query under other options was refused as `PlanError::IndexOptions`. The new `target::Layout` trait is a target's layout chosen by value, with the index it answers queries through; `Json::new(options)` implements it for `JsonDocument`, and `Plan::value().encrypt_as(Json::new(options))` builds a `ValuePlan<_, Laid>` that seals under the options and resolves every query through the plan's own index, so a query does not name the options again. The EQL type has the same by value: `eql_bindings::v4::JsonTree::new(options)` lays a value out as a `JsonTreeDocument` under an identifier. +- **Nothing changes in SQL.** `ord_term` was already NULL for an entry without `op`, and the entry validator already admitted its absence; the functional btree on `eql_v4.ord_term(col -> sel)` indexes the leaves at one selector, which under the opt-in are the leaves of one ordered path. + +What remains of the per-path policy is the `vs` half (equality off for a low-cardinality field) and opaque subtrees; neither is scheduled. + +### Measured (cipherstash/benches cd9643d) + +The benches re-sealed both corpora with one ordered path each, the one their range and ordering scenarios use (`$.age` on the flat corpus, `$.company.salary` on the nested), at 10,000 and 100,000 rows, and the correctness check passed at both tiers (462 of 462). "Before" is the Phase 3b form with `op` on every number and string leaf. + +| | flat, 10,000 rows | nested, 10,000 rows | nested, 100,000 rows | +|---|--:|--:|--:| +| `pg_column_size` per row | 2,250 → 1,596 B (−29%) | 62,143 → 40,578 B (−35%) | 62,092 → 40,549 B | +| heap with TOAST | 27.1 → 17.0 MiB | 620.9 → 407.5 MiB (−34%) | 6,201 → 4,069 MiB | +| containment GIN | 0.98 → 1.03 MiB | 39.9 MiB, unchanged | 358 MiB, unchanged | +| total with indexes | 28.4 → 18.4 MiB | 662.0 → 448.6 MiB | 6,569 → 4,437 MiB | + +The nested document is now 18 times its plaintext on disk (was 26 at Phase 3b, 40 at Phase 3). Two side effects: a flat document at 1,596 bytes sits under PostgreSQL's TOAST threshold, so the flat heap moves from TOAST into the main relation (0.5 to 17 MiB at 10,000 rows) and the earlier note about a TOASTed column making a sequential scan look cheap no longer applies to that corpus; and the flat GIN grew 12% at 100,000 rows with an identical projection, because the rows now spread over many more heap pages and the index's row-address deltas are wider. Query timings were not re-run for this measurement. + ## Decisions to take before Phase 1 +> **2026-10-22:** these are the open questions as they stood before Phase 1; the implementation notes above record how each was taken. Since then, decision 4's group tags exist only as the nesting of the stored projection `q`, decision 9 was settled by storing the projection (Phase 3), and decision 10's ordinal is not stored (Phase 3b). + 1. **Token encoding on the wire:** hex, matching v3's `s`, or base64url, saving about 40 characters per entry. Proposal: measure in Phase 3, default to base64url. 2. **Leaf ordering terms bound to the column only, or to the column and the selector.** Column-only reuses the v4 scalar ordering query domains unchanged and matches the ADR-0009 appendix; selector-bound stops cross-path order comparison at the cost of a JSON-specific query domain. Proposal: column-only for the spike, revisit in the ADR. 3. **Array index modes.** Wildcard only, with the chain serving positional access, retiring `[@]` and `[n]`. Proposal: yes. diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index 0262b5f51..a92898dd0 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -23,9 +23,9 @@ use schemars::{schema_for, JsonSchema, Schema}; use serde::{Deserialize, Serialize}; use serde_json::Value; use stack_encrypt::json::{ - JsonDocument, JsonLeafQuery, JsonNode, JsonPath, JsonQuery, JsonSubtree, + JsonDocument, JsonLeafQuery, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, }; -use stack_encrypt::target::{AeadContext, ExpectedContext, IndexSpec}; +use stack_encrypt::target::{AeadContext, ExpectedContext, IndexSpec, Json, Layout}; use stack_encrypt::{ DecryptField, DecryptInto, Decryptable, Decryption, EncryptFrom, Encryption, Error, NonEmpty, }; @@ -35,9 +35,8 @@ use crate::v3::terms::{EntryCiphertext, KeyHeader, OpeCllw, Selector}; use crate::v4::{DomainType, SchemaVersion}; use crate::Identifier; -/// A sixteen-byte chain tag, hex: a parent tag `p`, a wrapped token `w`, -/// a group tag `n` or `g`. Per document, salted by the document's `iv`, so -/// unlinkable across rows at rest. +/// A sixteen-byte chain tag, hex: a parent tag `p`. Per document, salted +/// by the document's `iv`, so unlinkable across rows at rest. #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)] pub struct JsonTreeTag(pub String); @@ -45,9 +44,11 @@ pub struct JsonTreeTag(pub String); /// [entry], q: group}`. Strict. `h` is the document's key header, stored /// once (every entry is sealed under the document's one data key); `sv` /// holds one entry per node in depth-first order, the root first; `q` is -/// the document's containment projection, its `{s, vs}` pairs nested by -/// array element, which the client writes beside the entries so the -/// server's `eql_v4.json_tree_query` reads it rather than rebuilding it. +/// the document's containment projection, one `{s, vs}` pair per node +/// nested by array element, which the client writes beside the entries +/// and which is the only place the value selectors live: the server's +/// `eql_v4.json_tree_query` reads it, and the engine takes it as the +/// projection when it reads a document back. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct JsonTreeDocument { @@ -58,35 +59,21 @@ pub struct JsonTreeDocument { pub q: JsonTreeQueryGroup, } -/// `public.eql_v4_json_tree_entry`: one node. `s` and `vs` are the path -/// and value selectors, `c` the node's own ciphertext; `o`, `p` and `w` -/// the chain fields below the heads; `n` and `g` the group tags of -/// container array elements and what sits inside them; `op` the leaf -/// ordering term. `h`, `j`, `d`, `v` and `i` are what the extractors graft -/// onto an entry they return, so it opens on its own. +/// `public.eql_v4_json_tree_entry`: one node. `s` is the path selector, +/// `c` the node's own ciphertext; `p` the parent tag below the chain's +/// heads, which the walk probes by ordinal; `op` the leaf ordering term. +/// `h`, `j`, `d`, `v` and `i` are what the extractors graft onto an entry +/// they return, so it opens on its own. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct JsonTreeEntry { pub s: Selector, - pub vs: Selector, pub c: EntryCiphertext, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] - pub o: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] pub p: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] - pub w: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] - pub n: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] - pub g: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] pub op: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] @@ -237,9 +224,11 @@ impl JsonTreeDocument { serde_json::from_value(value).map_err(codec) } - /// The engine's document: the header and entries without the envelope. + /// The engine's document: the header, entries and projection without + /// the envelope. fn document(&self) -> Result { - serde_json::from_value(serde_json::json!({ "h": self.h, "sv": self.sv })).map_err(codec) + serde_json::from_value(serde_json::json!({ "h": self.h, "sv": self.sv, "q": self.q })) + .map_err(codec) } } @@ -281,19 +270,62 @@ impl JsonTreeEntry { // Stack Encrypt targets // ============================================================================= -/// A document seals as the JSON index lays it out, under the column it is -/// stored in: the [`Identifier`] is the context the data key is minted -/// under and every selector derives under, and is written to `i`. +/// The `json_tree` layout under the JSON index's options: what seals a +/// value into a [`JsonTreeDocument`] under options other than the +/// defaults, such as the paths that carry an ordering term +/// ([`JsonOptions::ordered`]). A [`Layout`], so a Rust program runs it as +/// `keyset.run(JsonTree::new(options).encryption(), &value, column)`, and +/// its index is the engine's [`Json`] under the same options, which a +/// query derives through. [`JsonTreeDocument`]'s own [`EncryptFrom`] is +/// this layout under the defaults. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct JsonTree { + index: Json, +} + +impl JsonTree { + /// The layout under `options`. + pub fn new(options: JsonOptions) -> Self { + Self { + index: Json::new(options), + } + } + + /// The index the layout seals under, with its options. + pub fn index(&self) -> &Json { + &self.index + } +} + +/// A document seals as the JSON index lays it out under the layout's +/// options, under the column it is stored in: the [`Identifier`] is the +/// context the data key is minted under and every selector derives +/// under, and is written to `i`. +impl Layout for JsonTree { + type Target = JsonTreeDocument; + type Index = Json; + fn encryption<'s, K: 'static>( + &self, + ) -> Encryption<'s, Value, JsonTreeDocument, K, NonEmpty> { + Layout::::encryption::(&self.index) + .accepting::>() + .map_with_context(|document, i: NonEmpty| (i, document)) + .try_map(|(i, document)| JsonTreeDocument::envelope(i.into_inner(), &document)) + } + fn index(&self) -> &Json { + &self.index + } +} + +/// A document seals as the JSON index lays it out under the default +/// options ([`JsonTree::default`]): no path ordered, the root sealed. impl EncryptFrom for JsonTreeDocument { type Context = NonEmpty; fn encryption<'s, K: 'static>() -> Encryption<'s, Value, Self, K, Self::Context> { - >::encryption() - .accepting::>() - .map_with_context(|document, i: NonEmpty| (i, document)) - .try_map(|(i, document)| Self::envelope(i.into_inner(), &document)) + JsonTree::default().encryption() } fn indexes() -> Vec { - >::indexes() + JsonTree::default().specs() } } @@ -353,7 +385,7 @@ impl DecryptInto for JsonTreeSubtree { } /// The rows of a walk open to the complete subtrees they hold, in order: -/// one per node whose selector `json_tree_subtree_at` matched. +/// one per node the gates of `json_tree_subtree_at` reached. impl DecryptInto> for JsonTreeSubtree { type Context = NonEmpty; fn decryption(self, context: Self::Context) -> Decryption, K> { @@ -421,7 +453,7 @@ mod tests { fn a_document_carries_its_projection() { let document = serde_json::json!({ "v": 4, "i": {"t": "users", "c": "profile"}, "h": "AA==", - "sv": [{"s": "aa", "vs": "bb", "c": "AA=="}], + "sv": [{"s": "aa", "c": "AA=="}], "q": {"sv": [{"s": "aa", "vs": "bb"}], "el": []}, }); let stored: JsonTreeDocument = serde_json::from_value(document.clone()).unwrap(); @@ -451,7 +483,7 @@ mod tests { #[test] fn an_entry_without_the_grafted_fields_does_not_open() { let entry: JsonTreeEntry = serde_json::from_value(serde_json::json!({ - "s": "00", "vs": "00", "c": "AA==" + "s": "00", "c": "AA==" })) .unwrap(); assert!(entry.node().is_err()); diff --git a/packages/eql/src/v3/json_tree/functions.sql b/packages/eql/src/v3/json_tree/functions.sql index c23ea403e..e2e5d1fdf 100644 --- a/packages/eql/src/v3/json_tree/functions.sql +++ b/packages/eql/src/v3/json_tree/functions.sql @@ -33,26 +33,6 @@ AS $$ SELECT substr(sha256(x), 1, 16) $$; ---! @brief XOR of two sixteen-byte strings. ---! @internal ---! @param a bytea Sixteen bytes. ---! @param b bytea Sixteen bytes. ---! @return bytea `a XOR b`, byte by byte. -CREATE FUNCTION {{prefix}}_internal.json_tree_xor16(a bytea, b bytea) - RETURNS bytea - LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE -AS $$ -DECLARE - r bytea := a; - i int; -BEGIN - FOR i IN 0..15 LOOP - r := set_byte(r, i, get_byte(a, i) # get_byte(b, i)); - END LOOP; - RETURN r; -END; -$$; - --! @brief A 32-bit integer as four little-endian bytes. --! @internal --! @param n integer The integer, non-negative. @@ -92,35 +72,20 @@ AS $$ SELECT encode({{prefix}}_internal.json_tree_h16('p'::bytea || iv || token || {{prefix}}_internal.json_tree_le32(o)), 'hex') $$; ---! @brief Unwrap a child's token from its entry, given its parent's token: ---! `w XOR H("w" ‖ iv ‖ token ‖ c)`. +--! @brief The token of the child at ordinal `o` under the node with token +--! `token`: `H("t" ‖ token ‖ LE32(o))`. Column-wide like every +--! token, with no document salt, so a client derives an array +--! element's token from the path alone and the walk derives a hit's +--! token from its parent's on the spot. Nothing stores a token. --! @internal ---! @param iv bytea The document's iv. ---! @param token bytea The parent's token. ---! @param entry jsonb The child's entry, carrying `w` and `c`. +--! @param token bytea The parent's token, sixteen bytes. +--! @param o integer The child's ordinal. --! @return bytea The child's token, sixteen bytes. -CREATE FUNCTION {{prefix}}_internal.json_tree_unwrap(iv bytea, token bytea, entry jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_child_token(token bytea, o integer) RETURNS bytea LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT {{prefix}}_internal.json_tree_xor16( - decode(entry ->> 'w', 'hex'), - {{prefix}}_internal.json_tree_h16('w'::bytea || iv || token || decode(entry ->> 'c', 'base64')) - ) -$$; - ---! @brief How many ordinals a walk tries under each node: one more than the ---! largest ordinal any entry of the document carries. ---! @internal ---! @param val jsonb The document. ---! @return integer The fan-out bound; `0` for a document with no chained ---! entry. -CREATE FUNCTION {{prefix}}_internal.json_tree_fanout(val jsonb) - RETURNS integer - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -AS $$ - SELECT COALESCE(MAX((elem ->> 'o')::integer), -1) + 1 - FROM jsonb_array_elements(val -> 'sv') AS elem + SELECT {{prefix}}_internal.json_tree_h16('t'::bytea || token || {{prefix}}_internal.json_tree_le32(o)) $$; --! @brief Whether the document's root is sealed: its header's sealed-root @@ -141,8 +106,8 @@ $$; --! whose keys are each entry's `p` and whose values are the entry --! with its ordinal `j` in `sv` added. A walk looks a child up by --! the tag it computes, `map -> tag`, instead of scanning `sv` for ---! it, so the document is read once per walk and a level costs its ---! fan-out in hashes, not its fan-out times the document's size. +--! it, so the document is read once per walk and a probe costs one +--! hash and one lookup, not a pass over the document. --! @internal --! @param val jsonb The document. --! @return jsonb The map; `{}` for a document with no chained entry. @@ -364,122 +329,201 @@ $$; -- The token chain: children and subtrees by token ------------------------------------------------------------------------------ ---! The walk, in every function below: for the node in hand, compute the ---! parent tag a child at each ordinal under the document's fan-out would ---! carry, and look each tag up in the document's parent map ---! ({{prefix}}_internal.json_tree_by_parent). A hit is a child; its token ---! unwraps under the parent's and the walk recurses. The map and the ---! document's header fields are computed once per call, so a level costs ---! its fan-out in SHA-256 evaluations and one lookup each, and the ---! document is never scanned per node. - ---! @brief The children of the node whose token is `token`, as rows. +--! The walk, in every function below. A node's children sit at the dense +--! ordinals `0, 1, 2, …` under it, and a child's parent tag is +--! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`. So from a token the walk probes +--! ordinal 0, then 1, and so on, looking each tag up in the document's +--! parent map ({{prefix}}_internal.json_tree_by_parent), and stops at the +--! first miss. A hit's own token is `H("t" ‖ t(parent) ‖ LE32(o))`, +--! derived on the spot ({{prefix}}_internal.json_tree_child_token), and the +--! walk recurses under it. A subtree of `N` nodes therefore costs about +--! `2N` probes, `N` hits and a miss for every node visited (the end of a +--! parent's children, or a leaf's lack of any), each probe one SHA-256 and +--! one map lookup, plus one SHA-256 per hit for its token: about `3N` +--! hashes whatever the document's width. The map and the header fields +--! are computed once per call, so the document is read once. +--! +--! **Gates.** An object member's ordinal is a function of its siblings' +--! keys, which the client does not know, so the client derives a token +--! only for a chain head and the array positions that directly follow it, +--! and walks the rest through one gate per further step: either the step's selector, +--! which the entry at that depth must carry in `s`, or `#n`, an array +--! position, which is the one ordinal probed at that depth. A wildcard +--! step's selector is every element's, so it admits them all. Entries are +--! returned from the depth of the last gate down: the node each gate chain +--! reaches, and everything beneath it. + +--! @brief The children of the node the gates reach from the node whose +--! token is `token`, as rows. --! --! What `jsonb_array_elements` is for an encrypted array: one row per ---! element, in ordinal order, each opening on its own. Each child's token ---! is unwrapped on the way, though not returned. +--! element, in the document's order, each opening on its own. With no +--! gates, the children of the token's own node; with gates, the walk +--! descends through them as `json_tree_subtree_at` does and returns one +--! level below the node they reach. --! --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The parent's token, hex: derived by the client from the ---! path alone, and never stored. +--! @param token text The start node's token, hex: derived by the client +--! from the path alone, and never stored. +--! @param gates text[] One gate per step below the token, as +--! `json_tree_subtree_at` takes them. Default none. --! @return SETOF public.{{prefix}}_json_tree_entry The children, each with `d = 1`. -CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_tree, token text) +CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_tree, token text, gates text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF public.{{prefix}}_json_tree_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - WITH + WITH RECURSIVE doc AS MATERIALIZED ( SELECT {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, - {{prefix}}_internal.json_tree_fanout(e::jsonb) AS fanout, {{prefix}}_internal.json_tree_by_parent(e::jsonb) AS map, - {{prefix}}.json_tree_meta(e::jsonb) AS meta + {{prefix}}.json_tree_meta(e::jsonb) AS meta, + COALESCE(array_length(gates, 1), 0) AS gated + ), + -- The walk of `json_tree_subtree_at`, cut one level below the gates. + walk(depth, pt, o, entry, t, ppass, pass) AS ( + SELECT 0, NULL::bytea, NULL::integer, NULL::jsonb, decode(token, 'hex'), true, true + UNION ALL + SELECT probe.depth, probe.pt, probe.o, hit.entry, + {{prefix}}_internal.json_tree_child_token(probe.pt, probe.o), + probe.ppass, + probe.ppass AND ( + probe.depth > doc.gated + OR COALESCE(gates[probe.depth], '') LIKE '#%' + OR hit.entry ->> 's' = gates[probe.depth]) + FROM walk AS w + CROSS JOIN doc + CROSS JOIN LATERAL ( + SELECT w.depth + 1 AS depth, + w.t AS pt, + CASE WHEN COALESCE(gates[w.depth + 1], '') LIKE '#%' + THEN substr(gates[w.depth + 1], 2)::integer + ELSE 0 END AS o, + w.pass AS ppass + WHERE w.pass AND w.depth <= doc.gated + UNION ALL + SELECT w.depth, w.pt, w.o + 1, w.ppass + WHERE w.entry IS NOT NULL AND COALESCE(gates[w.depth], '') NOT LIKE '#%' + ) AS probe + CROSS JOIN LATERAL ( + SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, probe.pt, probe.o) AS entry + ) AS hit + WHERE hit.entry IS NOT NULL ) - SELECT (doc.meta || c.entry || '{"d": 1}'::jsonb)::public.{{prefix}}_json_tree_entry - FROM doc - CROSS JOIN generate_series(0, doc.fanout - 1) AS o - CROSS JOIN LATERAL ( - SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, decode(token, 'hex'), o) AS entry - ) AS c - WHERE c.entry IS NOT NULL - ORDER BY o + SELECT (doc.meta || w.entry || '{"d": 1}'::jsonb)::public.{{prefix}}_json_tree_entry + FROM walk AS w, doc + WHERE w.entry IS NOT NULL AND w.pass AND w.depth = doc.gated + 1 + ORDER BY (w.entry ->> 'j')::integer $$; ---! @brief How many children the node whose token is `token` has: the length ---! of an encrypted array, or the member count of an object. +--! @brief How many children the node the gates reach from `token` has: +--! the length of an encrypted array, or the member count of an +--! object. --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The node's token, hex. +--! @param token text The start node's token, hex. +--! @param gates text[] One gate per step below the token. Default none. --! @return integer The count; `0` for a node with no children, a token that --! names no node included. -CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_tree, token text) +CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_tree, token text, gates text[] DEFAULT ARRAY[]::text[]) RETURNS integer LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT count(*)::integer FROM {{prefix}}.json_tree_children(e, token) + SELECT count(*)::integer FROM {{prefix}}.json_tree_children(e, token, gates) $$; ---! @brief Every entry under the node whose token is `token`, as rows: the ---! recursive walk of the chain, in core SQL. +--! @brief The entries under the node the gates reach from the node whose +--! token is `token`, as rows: the recursive walk of the chain, in +--! core SQL. --! ---! Starting from the token, each level finds its children by parent tag, ---! unwraps their tokens, and recurses. A token that names no node yields no ---! row, and nothing in a row is linkable to anything without the token. The ---! node itself is not returned: the walk addresses nodes by their parent, ---! and the client reads the node's kind from its children's labels, or ---! fetches the node by its selector. +--! Each level probes the ordinals under the node in hand until the first +--! miss, derives each hit's token, and recurses. A token that names no node +--! yields no row, and nothing in a row is linkable to anything without the +--! token. At depth `d` (1 below the token) the gate `gates[d]`, if there +--! is one, decides which hits the walk descends through: an entry whose +--! `s` is the gate's selector, or, for a gate `#n`, the one hit at +--! ordinal `n`. Rows are returned from the depth of the last gate down: the +--! node a gate chain reaches, with `d = 0`, and everything beneath it, with +--! `d` its depth below that node. The start node is never returned. With +--! no gates, every descendant of the start node, with `d` its depth below +--! it, from 1. --! ---! **The budget.** A walk costs the fan-out in hashes per node it visits, ---! and returns about four hundred characters per entry, where a sealed root ---! holds the whole document in one ciphertext of about the plaintext's ---! size. So when the root is sealed and the subtree has more than `budget` ---! entries, the walk stops and returns the root entry alone, with `d = 0`: ---! the client opens it to the whole document and takes the path itself, ---! which is cheaper on both sides than the walk past that point. The ---! default of 64 is past the crossover by bytes (about a dozen entries) by ---! a margin that keeps a small subtree arriving as the entries it is. A ---! `budget` of `0` never falls back. A token that names no node yields no ---! row under any budget, and a document whose root is not sealed is always ---! walked in full, since its root holds no document to return. +--! **The budget.** A walk returns about two hundred characters per entry, +--! where a sealed root holds the whole document in one ciphertext of about +--! the plaintext's size. So when the root is sealed and the walk would +--! return more than `budget` entries, it stops and returns the root entry +--! alone, with `d = 0`: the client opens it to the whole document and takes +--! the path itself, which is cheaper on both sides than the walk past that +--! point. The default of 64 is past the crossover by bytes (about a dozen +--! entries) by a margin that keeps a small subtree arriving as the entries +--! it is. A `budget` of `0` never falls back. A token that names no node +--! yields no row under any budget, and a document whose root is not sealed +--! is always walked in full, since its root holds no document to return. --! --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The subtree's token, hex. +--! @param token text The start node's token, hex. +--! @param gates text[] One gate per step below the token: a selector, hex, +--! or `#n` for the array position `n`. Empty for the whole subtree. --! @param budget integer The most entries to return before falling back to ---! the sealed root; `0` to always walk the subtree out. Default 64. ---! @return SETOF public.{{prefix}}_json_tree_entry The descendants, each with its ---! depth `d` below the node, in depth-first order; or the root ---! entry alone, with `d = 0`, past the budget. -CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_tree, token text, budget integer DEFAULT 64) +--! the sealed root; `0` to always walk out. Default 64. +--! @return SETOF public.{{prefix}}_json_tree_entry The entries, each with its +--! depth `d` below the node the gates reach, in document order; or +--! the root entry alone, with `j = 0` and `d = 0`, past the budget. +CREATE FUNCTION {{prefix}}.json_tree_subtree_at(e public.{{prefix}}_json_tree, token text, gates text[], budget integer DEFAULT 64) RETURNS SETOF public.{{prefix}}_json_tree_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ WITH RECURSIVE doc AS MATERIALIZED ( SELECT {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, - {{prefix}}_internal.json_tree_fanout(e::jsonb) AS fanout, {{prefix}}_internal.json_tree_by_parent(e::jsonb) AS map, {{prefix}}.json_tree_meta(e::jsonb) AS meta, e::jsonb -> 'sv' -> 0 AS root, - budget > 0 AND {{prefix}}_internal.json_tree_sealed_root(e::jsonb ->> 'h') AS capped + budget > 0 AND {{prefix}}_internal.json_tree_sealed_root(e::jsonb ->> 'h') AS capped, + COALESCE(array_length(gates, 1), 0) AS gated ), - walk(depth, t, entry) AS ( - SELECT 0, decode(token, 'hex'), NULL::jsonb + -- One row per hit, after the seed row for the start node. `pt` and `o` + -- are where the hit sits, its parent's token and its ordinal; `t` is its + -- own token; `ppass` whether every gate down to its parent admitted the + -- parent, `pass` whether this hit's gate admits it too. + walk(depth, pt, o, entry, t, ppass, pass) AS ( + SELECT 0, NULL::bytea, NULL::integer, NULL::jsonb, decode(token, 'hex'), true, true UNION ALL - SELECT w.depth + 1, - {{prefix}}_internal.json_tree_unwrap(doc.iv, w.t, c.entry), - c.entry + SELECT probe.depth, probe.pt, probe.o, hit.entry, + {{prefix}}_internal.json_tree_child_token(probe.pt, probe.o), + probe.ppass, + probe.ppass AND ( + probe.depth > doc.gated + OR COALESCE(gates[probe.depth], '') LIKE '#%' + OR hit.entry ->> 's' = gates[probe.depth]) FROM walk AS w CROSS JOIN doc - CROSS JOIN generate_series(0, doc.fanout - 1) AS o CROSS JOIN LATERAL ( - SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, w.t, o) AS entry - ) AS c - WHERE c.entry IS NOT NULL + -- Down: the first ordinal under this node, or the one position an + -- array gate names, from a node its gates admitted. + SELECT w.depth + 1 AS depth, + w.t AS pt, + CASE WHEN COALESCE(gates[w.depth + 1], '') LIKE '#%' + THEN substr(gates[w.depth + 1], 2)::integer + ELSE 0 END AS o, + w.pass AS ppass + WHERE w.pass + UNION ALL + -- Across: the next ordinal under the same parent, from any hit, + -- since a hit its gate refused still has siblings to find; not + -- when an array gate pins this depth to one position. + SELECT w.depth, w.pt, w.o + 1, w.ppass + WHERE w.entry IS NOT NULL AND COALESCE(gates[w.depth], '') NOT LIKE '#%' + ) AS probe + CROSS JOIN LATERAL ( + SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, probe.pt, probe.o) AS entry + ) AS hit + WHERE hit.entry IS NOT NULL ), -- One row past the budget is enough to know the walk went over; a walk -- under it is the subtree complete. Uncapped, the limit is NULL: all. found AS MATERIALIZED ( SELECT w.depth, w.entry - FROM walk AS w - WHERE w.entry IS NOT NULL + FROM walk AS w, doc + WHERE w.entry IS NOT NULL AND w.pass AND w.depth >= doc.gated LIMIT (SELECT CASE WHEN doc.capped THEN budget + 1 END FROM doc) ), over AS MATERIALIZED ( @@ -493,82 +537,27 @@ AS $$ WHERE over.fallback UNION ALL SELECT (f.entry ->> 'j')::integer, - (doc.meta || f.entry || jsonb_build_object('d', f.depth))::public.{{prefix}}_json_tree_entry + (doc.meta || f.entry || jsonb_build_object('d', f.depth - doc.gated))::public.{{prefix}}_json_tree_entry FROM found AS f, doc, over WHERE NOT over.fallback ) AS rows ORDER BY rows.j $$; ---! @brief The subtrees under the node whose token is `token` whose path ---! continues through a node with selector `selector`: how a path ---! with a wildcard below its last positional step is walked. ---! ---! The walk starts at the token and emits an entry once it has passed ---! through, or reached, an entry carrying the selector: for ---! `$.orders[*].items` it starts at `$.orders` and returns every element's ---! `items` subtree, each complete. The budget is `json_tree_subtree`'s, ---! counted over the entries emitted: past it, the sealed root alone, with ---! `d = 0`, and the client takes the wildcard path over the whole document. ---! +--! @brief Every entry under the node whose token is `token`: +--! `json_tree_subtree_at` with no gates. --! @param e public.{{prefix}}_json_tree The stored document. ---! @param token text The anchor's token, hex. ---! @param selector text The whole path's selector, hex. +--! @param token text The subtree's token, hex. --! @param budget integer The most entries to return before falling back to ---! the sealed root; `0` to always walk out. Default 64. ---! @return SETOF public.{{prefix}}_json_tree_entry The matching subtrees' entries, ---! each with its depth `d` below the anchor, in depth-first order; ---! or the root entry alone, with `d = 0`, past the budget. -CREATE FUNCTION {{prefix}}.json_tree_subtree_at(e public.{{prefix}}_json_tree, token text, selector text, budget integer DEFAULT 64) +--! the sealed root; `0` to always walk the subtree out. Default 64. +--! @return SETOF public.{{prefix}}_json_tree_entry The descendants, each with its +--! depth `d` below the node, in document order; or the root entry +--! alone, with `d = 0`, past the budget. +CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_tree, token text, budget integer DEFAULT 64) RETURNS SETOF public.{{prefix}}_json_tree_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - WITH RECURSIVE - doc AS MATERIALIZED ( - SELECT {{prefix}}_internal.json_tree_iv(e::jsonb ->> 'h') AS iv, - {{prefix}}_internal.json_tree_fanout(e::jsonb) AS fanout, - {{prefix}}_internal.json_tree_by_parent(e::jsonb) AS map, - {{prefix}}.json_tree_meta(e::jsonb) AS meta, - e::jsonb -> 'sv' -> 0 AS root, - budget > 0 AND {{prefix}}_internal.json_tree_sealed_root(e::jsonb ->> 'h') AS capped - ), - walk(depth, t, entry, hit) AS ( - SELECT 0, decode(token, 'hex'), NULL::jsonb, false - UNION ALL - SELECT w.depth + 1, - {{prefix}}_internal.json_tree_unwrap(doc.iv, w.t, c.entry), - c.entry, - w.hit OR c.entry ->> 's' = selector - FROM walk AS w - CROSS JOIN doc - CROSS JOIN generate_series(0, doc.fanout - 1) AS o - CROSS JOIN LATERAL ( - SELECT doc.map -> {{prefix}}_internal.json_tree_parent_tag(doc.iv, w.t, o) AS entry - ) AS c - WHERE c.entry IS NOT NULL - ), - found AS MATERIALIZED ( - SELECT w.depth, w.entry - FROM walk AS w - WHERE w.entry IS NOT NULL AND w.hit - LIMIT (SELECT CASE WHEN doc.capped THEN budget + 1 END FROM doc) - ), - over AS MATERIALIZED ( - SELECT doc.capped AND (SELECT count(*) FROM found) > budget AS fallback FROM doc - ) - SELECT rows.entry - FROM ( - SELECT 0 AS j, - (doc.meta || doc.root || '{"j": 0, "d": 0}'::jsonb)::public.{{prefix}}_json_tree_entry AS entry - FROM doc, over - WHERE over.fallback - UNION ALL - SELECT (f.entry ->> 'j')::integer, - (doc.meta || f.entry || jsonb_build_object('d', f.depth))::public.{{prefix}}_json_tree_entry - FROM found AS f, doc, over - WHERE NOT over.fallback - ) AS rows - ORDER BY rows.j + SELECT * FROM {{prefix}}.json_tree_subtree_at(e, token, ARRAY[]::text[], budget) $$; ------------------------------------------------------------------------------ diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index d6647551b..060ef9b17 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -22,34 +22,34 @@ --! at the repository root. The document's per-entry fields: --! `s` path selector (16 bytes, hex): every element of an array shares its --! parent's, so a path reads `$.a[*].x`. ---! `vs` value selector (hex): the node's kind and, for a scalar, its value, ---! bound to `s`. Containment is a set comparison over `{s, vs}` pairs. --! `c` the node's own ciphertext (base64): its label, its kind and, for a --! scalar, its value. The root's is the whole document when the root is --! sealed (the default). ---! `o` the node's ordinal under its parent, `p` the parent tag ---! `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))` and `w` the node's own token ---! wrapped under its parent's: present below the chain's heads. ---! `n` the group tag of an array element that is a container, ---! `g` the group tag of the nearest enclosing such element. +--! `p` the parent tag `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`, where `o` is +--! the node's ordinal under its parent: present below the chain's +--! heads. The ordinals under a node are dense from 0, so a walk +--! holding `t(parent)` probes them in turn until the first miss, and +--! derives each hit's own token as `H("t" ‖ t(parent) ‖ LE32(o))`. +--! Nothing stores a token or an ordinal. --! `op` the leaf ordering term (CLLW OPE, hex) on number and string leaves. --! The header `h` (base64) carries the document's data key material once; the --! SQL reads only the sealed-root flag at byte offset 1 and the ZeroKMS `iv` ---! at byte offset 22 of it, the per-document salt of the chain tags, and +--! at byte offset 22 of it, the per-document salt of the parent tags, and --! otherwise carries it opaquely. `q` is the document's containment ---! projection, `{"sv": [{s, vs}, …], "el": [group, …]}`: its `{s, vs}` ---! pairs nested by array element, written by the client beside `sv` so the ---! server compares it with a needle by native `jsonb @>` and never ---! rebuilds it from the entries. +--! projection, `{"sv": [{s, vs}, …], "el": [group, …]}`: one `{s, vs}` pair +--! per node, `vs` being the value selector of the node's kind and, for a +--! scalar, its value, bound to `s`, nested by array element. The client +--! writes it beside `sv` and it is the only place the value selectors +--! live: the server compares it with a needle by native `jsonb @>`. --! @brief Validate one JSON tree entry. --! @internal --! @param val jsonb Candidate entry. ---! @return boolean True when `val` is an object with string `s`, `vs` and `c`, ---! an optional numeric `o`, optional string `p`, `w`, `n`, `g` and ---! `op`, and the metadata the extractors graft: string `h`, numeric ---! `j` (the entry's ordinal in the document) and `d` (its depth below ---! a walk's start), and the envelope's `v` and `i`. No other key. +--! @return boolean True when `val` is an object with string `s` and `c`, +--! optional string `p` and `op`, and the metadata the extractors +--! graft: string `h`, numeric `j` (the entry's ordinal in the +--! document) and `d` (its depth below a walk's start), and the +--! envelope's `v` and `i`. No other key. CREATE OR REPLACE FUNCTION public.{{prefix}}_is_valid_json_tree_entry_payload(val jsonb) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE @@ -57,18 +57,13 @@ AS $$ SELECT COALESCE( jsonb_typeof(val) = 'object' AND jsonb_typeof(val -> 's') = 'string' - AND jsonb_typeof(val -> 'vs') = 'string' AND jsonb_typeof(val -> 'c') = 'string' - AND (NOT (val ? 'o') OR jsonb_typeof(val -> 'o') = 'number') AND (NOT (val ? 'p') OR jsonb_typeof(val -> 'p') = 'string') - AND (NOT (val ? 'w') OR jsonb_typeof(val -> 'w') = 'string') - AND (NOT (val ? 'n') OR jsonb_typeof(val -> 'n') = 'string') - AND (NOT (val ? 'g') OR jsonb_typeof(val -> 'g') = 'string') AND (NOT (val ? 'op') OR jsonb_typeof(val -> 'op') = 'string') AND (NOT (val ? 'h') OR jsonb_typeof(val -> 'h') = 'string') AND (NOT (val ? 'j') OR jsonb_typeof(val -> 'j') = 'number') AND (NOT (val ? 'd') OR jsonb_typeof(val -> 'd') = 'number') - AND val - ARRAY['s', 'vs', 'c', 'o', 'p', 'w', 'n', 'g', 'op', 'h', 'j', 'd', 'v', 'i']::text[] = '{}'::jsonb, + AND val - ARRAY['s', 'c', 'p', 'op', 'h', 'j', 'd', 'v', 'i']::text[] = '{}'::jsonb, false ) $$; diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 0370c8139..f89e1098d 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -11,12 +11,13 @@ mod common; use eql_bindings::v4::json_tree::{ - JsonTreeDocument, JsonTreeEntry, JsonTreeLeafQuery, JsonTreeQuery, JsonTreeSubtree, + JsonTree, JsonTreeDocument, JsonTreeEntry, JsonTreeLeafQuery, JsonTreeQuery, JsonTreeSubtree, }; use eql_bindings::Identifier; use serde_json::{json, Map, Value}; use stack_encrypt::json::{JsonDocument, JsonOptions, JsonPath}; -use stack_encrypt::plan::{Typed, ValuePlan}; +use stack_encrypt::plan::{Laid, ValuePlan}; +use stack_encrypt::target::{Json, Layout}; use stack_encrypt::{op, NonEmpty, Plan}; fn column() -> NonEmpty { @@ -29,10 +30,12 @@ fn fixture() -> Value { type Cipher = stack_encrypt::StackCipher; +/// Seal `value` as the EQL type under the identifier, laid out by the +/// `json_tree` layout with the same ordered paths as [`plan`]. async fn seal(cipher: &Cipher, value: &Value) -> JsonTreeDocument { cipher .default_keyset() - .encrypt_as::(value, column()) + .run(JsonTree::new(options()).encryption(), value, column()) .await .unwrap() } @@ -40,17 +43,26 @@ async fn seal(cipher: &Cipher, value: &Value) -> JsonTreeDocument { /// The plan a query derives through: the engine's own document target /// under the label `users/profile`, which is the same context as the /// identifier the EQL target stores (the same two parts, rendered -/// `users/profile`), so its selectors are the stored document's. An EQL -/// type takes its context as an identifier and runs through `encrypt_as`, -/// not through the chain. -fn plan() -> ValuePlan> { +/// `users/profile`), so its selectors are the stored document's, laid +/// out by the JSON index with `$.hello` and `$.number` ordered, the paths +/// the ordering queries below ask at. An EQL type takes its context as +/// an identifier and runs through `encrypt_as`, not through the chain. +fn plan() -> ValuePlan> { Plan::value::() .context("users/profile") - .encrypt_into::() + .encrypt_as(Json::new(options())) .build() .unwrap() } +/// The index options the corpus is sealed under: `$.hello` and +/// `$.number` carry ordering terms. +fn options() -> JsonOptions { + JsonOptions::default() + .ordering(["$.hello", "$.number"]) + .unwrap() +} + fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } @@ -151,8 +163,12 @@ async fn the_query_operands_are_the_engines_terms_in_the_sql_contract() { assert_eq!(engine.entries().len(), typed.sv.len()); for (a, b) in engine.entries().iter().zip(&typed.sv) { assert_eq!(hex(a.selector()), b.s.0); - assert_eq!(hex(a.value_selector()), b.vs.0); } + assert_eq!( + serde_json::to_value(engine.projection()).unwrap(), + serde_json::to_value(&typed.q).unwrap(), + "and the same projection" + ); let opened: Value = cipher.open(engine).using(&plan).await.unwrap(); assert_eq!(opened, fixture()); } @@ -466,7 +482,7 @@ fn postgres_extracts_walks_and_orders_by_path() { &runtime, keyset.json_subtree_key( &JsonPath::parse(path).unwrap(), - JsonOptions::default(), + &JsonOptions::default(), column(), ), ) @@ -522,6 +538,7 @@ fn postgres_extracts_walks_and_orders_by_path() { // the path alone: six entries in row 1, reassembling the sub-document; // two in row 2. let nested = subtree_key("$.nested"); + assert!(nested.gates().is_empty(), "a chain head's token"); for (id, expected) in [ (1, fixture()["nested"].clone()), (2, rows[1].1["nested"].clone()), @@ -597,13 +614,17 @@ fn postgres_extracts_walks_and_orders_by_path() { .get(0); assert_eq!(garbage, 0); - // A wildcard below the anchor: every element's `x`, each a complete - // subtree, from the list's token gated on the whole path's selector. + // A path below the head walks from the head's token through one gate + // per step: `$.nested.list[*].x` is the list's selector, then every + // element's (the wildcard's), then x's; each `x` comes back as a + // complete subtree. let xs = subtree_key("$.nested.list[*].x"); + assert_eq!(xs.token(), nested.token(), "anchored at the head"); + assert_eq!(xs.gates().len(), 3); let walked: Vec = db .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text) e WHERE id = 1", - &[&xs.token(), &xs.selector().unwrap().to_string()], + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[]) e WHERE id = 1", + &[&xs.token(), &xs.gates()], ) .unwrap() .iter() @@ -619,8 +640,8 @@ fn postgres_extracts_walks_and_orders_by_path() { // wildcard path selects every subtree over the whole document. let root: Vec = db .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text, 1) e WHERE id = 1", - &[&xs.token(), &xs.selector().unwrap().to_string()], + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[], 1) e WHERE id = 1", + &[&xs.token(), &xs.gates()], ) .unwrap() .iter() @@ -638,12 +659,33 @@ fn postgres_extracts_walks_and_orders_by_path() { .unwrap(); assert_eq!(values, vec![json!(1), json!(2)]); - // The array functions, by the list's token. + // A positional path below a key step: the second element's subtree, + // through the list's selector and the position `#1`. + let second = subtree_key("$.nested.list[1]"); + assert_eq!(second.gates()[1], "#1"); + let walked: Vec = db + .query( + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_subtree_at(doc, $1::text, $2::text[]) e WHERE id = 1", + &[&second.token(), &second.gates()], + ) + .unwrap() + .iter() + .map(entry) + .collect(); + let value: Value = wait( + &runtime, + cipher.decrypt_as(JsonTreeSubtree::new(walked), column()), + ) + .unwrap(); + assert_eq!(value, json!({"x": 2})); + + // The array functions, by the head's token and the list's gate. let list = subtree_key("$.nested.list"); + assert_eq!(list.gates().len(), 1); let counts: Vec<(i32, i32)> = db .query( - "SELECT id, eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_rows ORDER BY id", - &[&list.token()], + "SELECT id, eql_v4.json_tree_child_count(doc, $1::text, $2::text[]) FROM json_tree_rows ORDER BY id", + &[&list.token(), &list.gates()], ) .unwrap() .iter() @@ -652,8 +694,8 @@ fn postgres_extracts_walks_and_orders_by_path() { assert_eq!(counts, [(1, 2), (2, 0), (3, 0)]); let elements: Vec = db .query( - "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_children(doc, $1::text) e WHERE id = 1", - &[&list.token()], + "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_children(doc, $1::text, $2::text[]) e WHERE id = 1", + &[&list.token(), &list.gates()], ) .unwrap() .iter() @@ -661,9 +703,18 @@ fn postgres_extracts_walks_and_orders_by_path() { .collect(); assert_eq!(elements.len(), 2); assert_eq!( - elements.iter().map(|e| e.o).collect::>(), - [Some(0), Some(1)] + elements.iter().map(|e| (e.j, e.d)).collect::>(), + [(Some(5), Some(1)), (Some(7), Some(1))] ); + // The head's own children, with no gate: deep and list. + let members: i32 = db + .query_one( + "SELECT eql_v4.json_tree_child_count(doc, $1::text) FROM json_tree_rows WHERE id = 1", + &[&nested.token()], + ) + .unwrap() + .get(0); + assert_eq!(members, 2); // Orderings at a path: the rows whose `$.number` is in range, and // sorted by the leaf's term. diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 092dcf737..62c79fa50 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -390,6 +390,15 @@ impl From for Error { } } +/// A text that is not a JSON path is a term error, as it is when an +/// accessor's path is parsed: a caller building a JSON index's options +/// (`JsonOptions::ordering`) can `?` it. +impl From for Error { + fn from(error: crate::json::JsonPathError) -> Self { + Error::Term(error.into()) + } +} + /// The CipherStash cipher, scoped to one client: a ZeroKMS client (a /// [`DataKeySource`] — production: [`kms::StackKms`](crate::kms::StackKms); tests: /// `stack_encrypt::kms::FakeDataKeySource`) and the keysets that client uses. diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index 9c396591e..880e75970 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -97,6 +97,7 @@ mod tests { TermError::JsonTooWide => TermError::JsonTooWide, TermError::JsonSealedRoot => TermError::JsonSealedRoot, TermError::JsonLeafNotOrdered => TermError::JsonLeafNotOrdered, + TermError::JsonPathNotOrdered => TermError::JsonPathNotOrdered, ])); errors.extend(boxed(variants![ TermBytesError::WrongEqualityTermLength(_) => { diff --git a/packages/stack-encrypt/src/dynamic/term.rs b/packages/stack-encrypt/src/dynamic/term.rs index 7df0ed7e9..13bec13e3 100644 --- a/packages/stack-encrypt/src/dynamic/term.rs +++ b/packages/stack-encrypt/src/dynamic/term.rs @@ -77,10 +77,14 @@ use crate::{Encrypt, IntoPrfContext, KeysetCipher, NonEmpty}; /// A JSON index is `"json"`, or an object mapping `"json"` to its options: /// /// ```text -/// { "json": { "sealed_root": } } +/// { "json": { "sealed_root": , "ordered": [, …] } } /// ``` /// -/// `sealed_root` defaults to `true` ([`JsonOptions::default`]). A data plan +/// `sealed_root` defaults to `true` and `ordered` to none +/// ([`JsonOptions::default`]); each path in `ordered` is text +/// [`JsonPath::parse`](crate::json::JsonPath::parse) takes, such as +/// `"$.age"` or `"$.orders[*].total"`, and one it refuses refuses the +/// index. A data plan /// reads and writes the JSON index and refuses to run it /// ([`Reason::JsonIndexPending`]): the engine's typed path produces a JSON /// document, and the binding's half is not written yet. The other four @@ -210,6 +214,7 @@ impl Serialize for IndexSpec { #[derive(Serialize)] struct JsonOut { sealed_root: bool, + ordered: Vec, } #[derive(Serialize)] struct MatchOut { @@ -242,6 +247,7 @@ impl Serialize for IndexSpec { IndexSpec::Json(options) if *options == JsonOptions::default() => Out::DefaultJson, IndexSpec::Json(options) => Out::Json(JsonOut { sealed_root: options.sealed_root, + ordered: options.ordered.iter().map(ToString::to_string).collect(), }), } .serialize(s) @@ -279,29 +285,36 @@ impl From for IndexSpec { } /// A JSON index's options object: each option optional, defaulting to -/// [`JsonOptions::default`]'s. +/// [`JsonOptions::default`]'s, and each ordered path parsed. #[derive(Deserialize)] -#[serde(from = "JsonOptionsWire")] +#[serde(try_from = "JsonOptionsWire")] pub(super) struct JsonWire(pub(super) JsonOptions); #[derive(Deserialize)] #[serde(deny_unknown_fields, default)] struct JsonOptionsWire { sealed_root: bool, + ordered: Vec, } impl Default for JsonOptionsWire { fn default() -> Self { - let JsonOptions { sealed_root } = JsonOptions::default(); - Self { sealed_root } + Self { + sealed_root: JsonOptions::default().sealed_root, + ordered: Vec::new(), + } } } -impl From for JsonWire { - fn from(wire: JsonOptionsWire) -> Self { - JsonWire(JsonOptions { +impl TryFrom for JsonWire { + type Error = crate::json::JsonPathError; + fn try_from(wire: JsonOptionsWire) -> Result { + let options = JsonOptions { sealed_root: wire.sealed_root, - }) + ..JsonOptions::new() + } + .ordering(wire.ordered)?; + Ok(JsonWire(options)) } } diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 5f928ff0b..0c2e66fc3 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -19,7 +19,7 @@ use vitaminc_prf::{BlockVisitor, IntoPrfContext, PrfValue}; use vitaminc_protected::{Controlled, Protected}; use super::path::{JsonPath, Segment}; -use super::query::{JsonSelector, JsonSubtreeKey}; +use super::query::{Gate, JsonSelector, JsonSubtreeKey}; use super::{JsonDocument, JsonEntry, JsonHeader, JsonNode, JsonOptions, JsonSubtree, Tag}; use crate::sem::TermError; use crate::target::{AeadContext, CallerContext, Decryption, Pending, Request}; @@ -235,31 +235,22 @@ fn h16(parts: &[&[u8]]) -> Tag { truncate(&hasher.finalize()) } -/// `H("t" ‖ t(parent) ‖ 0x01 ‖ key)` or `H("t" ‖ t(parent) ‖ 0x02 ‖ LE64(index))`. -fn child_token(parent: &Tag, label: &Label) -> Tag { - match label { - Label::Key(key) => h16(&[b"t", parent, &[0x01], key.as_bytes()]), - Label::Index(index) => h16(&[b"t", parent, &[0x02], &index.to_le_bytes()]), - Label::Root => *parent, - } +/// `H("t" ‖ t(parent) ‖ LE32(ordinal))`: the token of the child at +/// `ordinal` under a node. Column-wide like every token (no document salt), +/// so the client derives an array element's token from the path alone, and +/// the server derives every hit's from its parent's as it walks; no token +/// is ever stored. +pub(super) fn child_token(parent: &Tag, ordinal: u32) -> Tag { + h16(&[b"t", parent, &ordinal.to_le_bytes()]) } /// `H("p" ‖ iv ‖ t(parent) ‖ LE32(ordinal))`: position-keyed, so siblings -/// are unlinkable at rest. +/// are unlinkable at rest, and salted by the document, so the same position +/// in two rows is too. pub(super) fn parent_tag(iv: &[u8], parent: &Tag, ordinal: u32) -> Tag { h16(&[b"p", iv, parent, &ordinal.to_le_bytes()]) } -/// `H("w" ‖ iv ‖ t(parent) ‖ c)`: the mask a child's token is wrapped under. -pub(super) fn mask(iv: &[u8], parent: &Tag, c: &[u8]) -> Tag { - h16(&[b"w", iv, parent, c]) -} - -/// `H("g" ‖ iv ‖ t(element))`: an array element's group tag. -fn group_tag(iv: &[u8], token: &Tag) -> Tag { - h16(&[b"g", iv, token]) -} - /// `H("o" ‖ t(parent) ‖ key)`: what an object's members are ranked by for /// their ordinals, so an ordinal does not rank key names. fn rank(parent: &Tag, key: &str) -> [u8; 32] { @@ -270,14 +261,6 @@ fn rank(parent: &Tag, key: &str) -> [u8; 32] { hasher.finalize().into() } -pub(super) fn xor(a: &Tag, b: &Tag) -> Tag { - let mut out = [0; 16]; - for (i, byte) in out.iter_mut().enumerate() { - *byte = a[i] ^ b[i]; - } - out -} - /// A number's canonical bytes for its value selector: an integral value, /// whichever JSON spelling it had, is its `i128`; anything else is its /// `f64`. So `2` and `2.0` are one value, as `jsonb` compares them, and @@ -331,7 +314,8 @@ fn value_selector( /// The leaf ordering term: the OPE term of a number as an `f64` or of a /// string, under the column's context, as the scalar `Ope` index derives /// it. Nothing for any other kind. An integer wider than 53 bits orders by -/// the `f64` it rounds to. +/// the `f64` it rounds to. The walk asks for it only at a path the +/// options order. fn leaf_op( prf: &HmacSha256Prf, context: &Context<'_>, @@ -366,7 +350,7 @@ struct Walked { struct Walker<'p> { prf: &'p HmacSha256Prf, context: &'p Context<'p>, - options: JsonOptions, + options: &'p JsonOptions, nodes: Vec, plain: Vec, } @@ -374,7 +358,7 @@ struct Walker<'p> { /// Flatten `value` into its terms and its nodes' plaintexts. fn walk( value: &Value, - options: JsonOptions, + options: &JsonOptions, prf: &HmacSha256Prf, context: &Context<'_>, ) -> Result { @@ -446,9 +430,13 @@ impl Walker<'_> { self.context, &positional.token_bytes().unwrap_or_default(), )?), - _ => parent_token.map(|p| child_token(&p, &label)), + _ => parent_token.zip(ordinal).map(|(p, o)| child_token(&p, o)), + }; + let op = if self.options.orders(positional) { + leaf_op(self.prf, self.context, value)? + } else { + None }; - let op = leaf_op(self.prf, self.context, value)?; let container_element = matches!(label, Label::Index(_)) && kind.is_container(); let index = self.nodes.len(); self.nodes.push(TermNode { @@ -543,7 +531,7 @@ fn shuffled_ordinals(parent: &Tag, map: &Map) -> BTreeMap( value: &Value, - options: JsonOptions, + options: &JsonOptions, cipher: &'a KeysetCipher<'_, K>, context: NonEmpty, ) -> Pending<'a, JsonTerms, K> { @@ -560,7 +548,7 @@ pub(crate) fn terms_pending<'a, K>( /// column and its ordinal. pub(crate) fn seal_pending<'a, K: 'static>( value: Value, - options: JsonOptions, + options: &JsonOptions, cipher: &'a KeysetCipher<'_, K>, context: NonEmpty, ) -> Pending<'a, JsonDocument, K> { @@ -584,12 +572,13 @@ pub(crate) fn seal_pending<'a, K: 'static>( None }; let keyset_id = cipher.keyset_id(); + let sealed_root = options.sealed_root; Pending::request( cipher, vec![Request::generate_under(descriptor)], move |responses| { let key = responses.next_generated_key()?; - seal(walked, whole, keyset_id, key, &aad, options) + seal(walked, whole, keyset_id, key, &aad, sealed_root) }, ) } @@ -643,7 +632,7 @@ fn seal( keyset_id: Uuid, key: stack_kms::DataKeyWithTag, aad: &Context<'_>, - options: JsonOptions, + sealed_root: bool, ) -> Result { if key.tag.len() > usize::from(u16::MAX) { return Err(Error::Other( @@ -673,38 +662,21 @@ fn seal( .map_err(|_| Error::Aead)?; let c = c.as_ref().to_vec(); let parent_token = node.parent.and_then(|p| nodes[p].token); - let (o, p, w) = match (parent_token, node.ordinal, node.token) { - (Some(parent), Some(o), Some(token)) => ( - Some(o), - Some(parent_tag(&iv, &parent, o)), - Some(xor(&token, &mask(&iv, &parent, &c))), - ), - _ => (None, None, None), - }; - let n = if node.container_element { - node.token.map(|t| group_tag(&iv, &t)) - } else { - None - }; - let g = node - .element - .and_then(|e| nodes[e].token) - .map(|t| group_tag(&iv, &t)); + let p = parent_token + .zip(node.ordinal) + .map(|(parent, o)| parent_tag(&iv, &parent, o)); entries.push(JsonEntry { s: node.s, - vs: node.vs, c, - o, p, - w, - n, - g, op: node.op.clone(), }); } + let projection = super::query::group_of(nodes, None); Ok(JsonDocument::new( - JsonHeader::new(keyset_id, iv, tag, options.sealed_root), + JsonHeader::new(keyset_id, iv, tag, sealed_root), entries, + projection, )) } @@ -941,9 +913,9 @@ impl DecryptInto for JsonNode { /// Open the rows of a walk: each complete subtree the rows hold, as a /// value, in order. What `json_tree_subtree_at` returns is one subtree per -/// node whose selector matched; what `json_tree_subtree` returns is the -/// children of one node, which [`DecryptInto`] assembles into that -/// node's value. The sealed root a walk fell back to opens whole, and the +/// node its gates reached, each rooted at depth `0`; what +/// `json_tree_subtree` returns is the children of one node, which +/// [`DecryptInto`] assembles into that node's value. The sealed root a walk fell back to opens whole, and the /// path attached with [`JsonSubtree::at`] selects over it: one value per /// node the path reaches. impl DecryptInto> for JsonSubtree { @@ -960,14 +932,19 @@ impl DecryptInto> for JsonSubtree { } } -/// Open the rows of `json_tree_subtree(doc, token)` into the value under -/// that token: an array when the rows are elements, an object when they are -/// members. No rows is no value ([`Error::OpenedType`]): an empty container -/// and a scalar both have no descendants, and `->` on the node's selector -/// returns the node itself. The sealed root a walk fell back to opens -/// whole, and the path attached with [`JsonSubtree::at`] selects the one -/// value under it; a path that reaches no node, or more than one, is -/// [`Error::OpenedType`]. +/// Open the rows of a walk into the one value they hold. For +/// `json_tree_subtree(doc, token)`, the rows are the descendants of the +/// token's node and the value is that node's: an array when the rows are +/// elements, an object when they are members. For a gated +/// `json_tree_subtree_at`, the rows are the node the gates reached, at +/// depth `0`, and its descendants, and the value is that node's; more than +/// one reached node (a wildcard) is [`Error::OpenedType`], and opens +/// through [`DecryptInto>`] instead. No rows is no value +/// ([`Error::OpenedType`]): an empty container and a scalar both have no +/// descendants, and `->` on the node's selector returns the node itself. +/// The sealed root a walk fell back to opens whole, and the path attached +/// with [`JsonSubtree::at`] selects the one value under it; a path that +/// reaches no node, or more than one, is [`Error::OpenedType`]. impl DecryptInto for JsonSubtree { type Context = AeadContext; fn decryption(self, context: Self::Context) -> Decryption { @@ -982,7 +959,15 @@ impl DecryptInto for JsonSubtree { } }); } - open_subtree(self, cipher, context).try_map(|forest| { + let reached = self.nodes.iter().any(|n| n.depth == Some(0)); + open_subtree(self, cipher, context).try_map(move |forest| { + if reached { + let mut values: Vec = forest.into_iter().map(|(_, v)| v).collect(); + return match values.len() { + 1 => Ok(values.remove(0)), + _ => Err(shape("one node at the walked path")), + }; + } let mut members = Map::new(); let mut elements: Vec<(u64, Value)> = Vec::new(); for (label, value) in forest { @@ -1078,14 +1063,23 @@ fn open_fallback<'a, K>( // Extraction keys, on the keyset cipher // ============================================================================= -/// The token of the node at `path`, which must be positional: a chain head -/// by PRF, then one hash per step below it. -fn token_for( +/// The key that walks `path`: the token of the deepest node on it the +/// client can derive, and one gate per step below that. +/// +/// A chain head's token is a PRF of its path. An array element's is one +/// hash under its parent's, by its position, so the client follows array +/// steps as far as they run from the head. An object member's ordinal is a +/// rank among its siblings' keys, which the client does not know, so from +/// the first key step on the client cannot derive tokens and the server +/// walks instead, through gates: the step's selector for a key or a +/// wildcard (every element's selector is the wildcard's, so a wildcard +/// admits them all), the position for an array step. +fn subtree_key_for( prf: &HmacSha256Prf, context: &Context<'_>, - options: JsonOptions, + options: &JsonOptions, path: &JsonPath, -) -> Result { +) -> Result { let segments = path.segments(); let (head, rest) = if options.sealed_root { let Some((first, rest)) = segments.split_first() else { @@ -1096,22 +1090,45 @@ fn token_for( } else { (JsonPath::root(), segments) }; - let mut token = prf16( - prf, - TOKEN_DOMAIN, - context, - &head.token_bytes().ok_or(TermError::JsonSealedRoot)?, - )?; + // A wildcard head names no one node. + let head_bytes = head.token_bytes().ok_or(TermError::JsonSealedRoot)?; + let mut token = prf16(prf, TOKEN_DOMAIN, context, &head_bytes)?; + let mut prefix = head; + let mut rest = rest.iter(); + let mut gates = Vec::new(); + for segment in rest.by_ref() { + prefix = prefix.then(segment.clone()); + match segment { + Segment::Index(index) => { + let o = u32::try_from(*index).map_err(|_| TermError::JsonTooWide)?; + token = child_token(&token, o); + } + Segment::Key(_) | Segment::Wildcard => { + gates.push(Gate::Selector(prf16( + prf, + SELECTOR_DOMAIN, + context, + &prefix.selector_bytes(), + )?)); + break; + } + } + } for segment in rest { - let label = match segment { - Segment::Key(key) => Label::Key(key.clone()), - Segment::Index(index) => Label::Index(*index), - // `anchor` cut the path before any wildcard. - Segment::Wildcard => return Err(TermError::JsonSealedRoot), - }; - token = child_token(&token, &label); + prefix = prefix.then(segment.clone()); + gates.push(match segment { + Segment::Index(index) => { + Gate::Slot(u32::try_from(*index).map_err(|_| TermError::JsonTooWide)?) + } + Segment::Key(_) | Segment::Wildcard => Gate::Selector(prf16( + prf, + SELECTOR_DOMAIN, + context, + &prefix.selector_bytes(), + )?), + }); } - Ok(token) + Ok(JsonSubtreeKey { token, gates }) } impl KeysetCipher<'_, K> { @@ -1138,9 +1155,9 @@ impl KeysetCipher<'_, K> { /// The key that walks the subtree under `path`, under the field /// `descriptor` and the index's `options`: the token of the deepest - /// positional node on the path, and, when the path continues with a - /// wildcard below it, the selector of the whole path, which the walk - /// gates on. What `json_tree_subtree` and `json_tree_subtree_at` take. + /// node on the path the client can derive (a chain head, then array + /// positions), and one gate per step below it, which the server walks + /// through. What `json_tree_subtree` and `json_tree_subtree_at` take. /// /// # Errors /// @@ -1150,30 +1167,18 @@ impl KeysetCipher<'_, K> { pub fn json_subtree_key<'c>( &self, path: &JsonPath, - options: JsonOptions, + options: &JsonOptions, descriptor: NonEmpty>, ) -> Pending<'_, JsonSubtreeKey, K> { let context = descriptor.into_prf_context().into_owned(); - let (anchor, cut) = path.anchor(); - let key = token_for(self.prf(), &context, options, &anchor).and_then(|token| { - let selector = cut - .then(|| { - prf16( - self.prf(), - SELECTOR_DOMAIN, - &context, - &path.selector_bytes(), - ) - }) - .transpose()?; - Ok(JsonSubtreeKey { token, selector }) - }); + let key = subtree_key_for(self.prf(), &context, options, path); Pending::ready(self, key.map_err(Error::from)) } } #[cfg(test)] mod tests { + use super::super::hex; use super::*; use serde_json::json; use vitaminc_prf::PrfKeyInit; @@ -1182,7 +1187,14 @@ mod tests { HmacSha256Prf::new(Protected::new([7; 32])) } - fn terms(value: &Value, options: JsonOptions) -> JsonTerms { + fn open_root() -> JsonOptions { + JsonOptions { + sealed_root: false, + ..JsonOptions::new() + } + } + + fn terms(value: &Value, options: &JsonOptions) -> JsonTerms { walk( value, options, @@ -1214,7 +1226,7 @@ mod tests { #[test] fn the_terms_name_a_node_by_its_path() { - let t = terms(&json!({"a": [1, {"b": 2}]}), JsonOptions::default()); + let t = terms(&json!({"a": [1, {"b": 2}]}), &JsonOptions::default()); // $, $.a, $.a[0], $.a[1], $.a[1].b assert_eq!(t.len(), 5); assert!(!t.is_empty()); @@ -1228,7 +1240,7 @@ mod tests { #[test] fn every_element_of_an_array_shares_a_selector_and_values_do_not() { - let t = terms(&json!({"a": [1, 2, 1]}), JsonOptions::default()); + let t = terms(&json!({"a": [1, 2, 1]}), &JsonOptions::default()); let nodes = t.nodes(); assert_eq!(nodes.len(), 5); assert_eq!(nodes[2].s, nodes[3].s); @@ -1239,12 +1251,12 @@ mod tests { #[test] fn the_sealed_root_has_no_token_and_its_fields_are_heads() { - let sealed = terms(&json!({"a": {"b": 1}}), JsonOptions::default()); + let sealed = terms(&json!({"a": {"b": 1}}), &JsonOptions::default()); assert!(sealed.nodes()[0].token.is_none()); assert!(sealed.nodes()[1].token.is_some()); assert!(sealed.nodes()[1].ordinal.is_none(), "a head has no ordinal"); assert!(sealed.nodes()[2].ordinal.is_some()); - let open = terms(&json!({"a": {"b": 1}}), JsonOptions { sealed_root: false }); + let open = terms(&json!({"a": {"b": 1}}), &open_root()); assert!(open.nodes()[0].token.is_some()); assert!(open.nodes()[1].ordinal.is_some()); // The chain below a head is the same hash chain either way, but @@ -1254,44 +1266,81 @@ mod tests { } #[test] - fn a_head_token_is_what_the_subtree_key_derives() { + fn the_subtree_key_derives_what_it_can_and_gates_the_rest() { let context = Context::from_encoded(b"users/profile"); - let value = json!({"a": {"b": [1, {"c": 2}]}}); - for options in [JsonOptions::default(), JsonOptions { sealed_root: false }] { - let t = terms(&value, options); - for node in t.nodes().iter().filter(|n| n.token.is_some()) { - let derived = token_for(&prf(), &context, options, &node.positional).unwrap(); - assert_eq!(Some(derived), node.token, "{}", node.positional); - } + let value = json!({"a": {"b": [1, {"c": 2}]}, "l": [[0], 1]}); + let key = |options: &JsonOptions, path: &str| { + subtree_key_for(&prf(), &context, options, &JsonPath::parse(path).unwrap()) + }; + let at = |t: &JsonTerms, path: &str| { + t.nodes() + .iter() + .find(|n| n.positional == JsonPath::parse(path).unwrap()) + .unwrap() + .clone() + }; + let options = JsonOptions::default(); + let t = terms(&value, &options); + // A head's token, and the positions under it, derive exactly. + for path in ["$.a", "$.l", "$.l[0]", "$.l[0][0]", "$.l[1]"] { + let k = key(&options, path).unwrap(); + assert_eq!(Some(*k.token_bytes()), at(&t, path).token, "{path}"); + assert!(k.gates().is_empty(), "{path}"); } + // The server derives an object member's token the same way, from + // its parent's and its ordinal. + let a = at(&t, "$.a"); + let b = at(&t, "$.a.b"); + assert_eq!( + b.token, + Some(child_token(&a.token.unwrap(), b.ordinal.unwrap())) + ); + // Below a key step, gates: the step's selector, a position, the + // wildcard's selector (every element's). + let k = key(&options, "$.a.b[1].c").unwrap(); + assert_eq!(Some(*k.token_bytes()), a.token); + assert_eq!( + k.gates(), + [hex(&b.s), "#1".to_owned(), hex(&at(&t, "$.a.b[1].c").s)] + ); + let k = key(&options, "$.a.b[*].c").unwrap(); + assert_eq!( + k.gates(), + [ + hex(&b.s), + hex(&at(&t, "$.a.b[0]").s), + hex(&at(&t, "$.a.b[1].c").s) + ] + ); + // Under the sealed root the root has no token; under an open root + // it has, and `$.a` is then one gate below it. assert!(matches!( - token_for(&prf(), &context, JsonOptions::default(), &JsonPath::root()), + key(&JsonOptions::default(), "$"), Err(TermError::JsonSealedRoot) )); - assert!(token_for( - &prf(), - &context, - JsonOptions { sealed_root: false }, - &JsonPath::root() - ) - .is_ok()); + let open = open_root(); + let root = key(&open, "$").unwrap(); + assert!(root.gates().is_empty()); + let a = key(&open, "$.a").unwrap(); + assert_eq!(a.token_bytes(), root.token_bytes()); + assert_eq!(a.gates().len(), 1); } #[test] fn a_root_scalar_carries_its_element_selectors_as_alt() { - let scalar = terms(&json!("a"), JsonOptions::default()); - let array = terms(&json!(["a"]), JsonOptions::default()); + let scalar = terms(&json!("a"), &JsonOptions::default()); + let array = terms(&json!(["a"]), &JsonOptions::default()); assert_eq!( scalar.alt(), Some((array.nodes()[1].s, array.nodes()[1].vs)), "the alt pair is the stored array element's" ); - assert!(terms(&json!({}), JsonOptions::default()).alt().is_none()); + assert!(terms(&json!({}), &JsonOptions::default()).alt().is_none()); } #[test] fn container_elements_group_what_sits_inside_them() { - let t = terms(&json!({"a": [{"x": 1}, 2, [3]]}), JsonOptions::default()); + let t = terms(&json!({"a": [{"x": 1}, 2, [3]]}), &JsonOptions::default()); let nodes = t.nodes(); // $, $.a, $.a[0], $.a[0].x, $.a[1], $.a[2], $.a[2][0] assert!(nodes[2].container_element); @@ -1386,7 +1435,7 @@ mod tests { let value = json!({"a": 1}); let context = Context::from_encoded(b"users/profile"); let sealed = |tag_len: usize| { - let walked = walk(&value, JsonOptions::default(), &prf(), &context).unwrap(); + let walked = walk(&value, &JsonOptions::default(), &prf(), &context).unwrap(); let key = stack_kms::DataKeyWithTag { key: stack_kms::DataKey { iv: [1; 16], @@ -1401,7 +1450,7 @@ mod tests { Uuid::from_u128(1), key, &context, - JsonOptions::default(), + true, ) }; let longest = usize::from(u16::MAX); @@ -1427,7 +1476,7 @@ mod tests { }; assert!(walk( &nested(MAX_DEPTH), - JsonOptions::default(), + &JsonOptions::default(), &prf(), &Context::empty() ) @@ -1435,7 +1484,7 @@ mod tests { assert!(matches!( walk( &nested(MAX_DEPTH + 1), - JsonOptions::default(), + &JsonOptions::default(), &prf(), &Context::empty() ), diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index ebe983d19..9ea0d0c2c 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -7,22 +7,26 @@ //! //! - a **selector** `s`, a PRF of the node's path with every array position //! folded into the wildcard, so every element of an array shares one; -//! - a **value selector** `vs`, a PRF of the node's kind and, for a scalar, -//! its canonical value, bound to `s`; containment is a set comparison -//! over these two; //! - the node's own **ciphertext** `c`, sealed under one ZeroKMS data key //! per document: the node's label, its kind and, for a scalar, its //! value, and for a container its child count. Nothing beneath a node is //! encrypted in that node's entry, so a document's ciphertext is linear //! in its size; -//! - the **token chain** fields `p` and `w`, which let the server walk the -//! subtree under a node from one client-derived token and nothing else, -//! and the **group tags** `n` and `g`, which tell the server which -//! entries sit inside the same array element, so containment agrees with -//! PostgreSQL's own `jsonb @>` across elements; -//! - a leaf ordering term `op` on number and string leaves, the same OPE -//! term the scalar [`Ope`](crate::target::Ope) index derives, keyed by -//! the column. +//! - the **parent tag** `p`, `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`, which +//! lets the server walk the subtree under a node from one client-derived +//! token and nothing else: the ordinals `o` under a node are dense from +//! zero, so a walk probes them in turn until the first miss, and derives +//! each hit's own token as `H("t" ‖ t(parent) ‖ LE32(o))`; +//! - a leaf ordering term `op` on the number and string leaves at the +//! paths the index declares ordered ([`JsonOptions::ordered`]), the +//! same OPE term the scalar [`Ope`](crate::target::Ope) index derives, +//! keyed by the column. No path is ordered by default. +//! +//! Beside the entries the document stores its containment **projection** +//! `q`: one `(s, vs)` pair per node, `vs` being a PRF of the node's kind +//! and, for a scalar, its canonical value, bound to `s`, nested by array +//! element so that PostgreSQL's own `jsonb @>` over two projections is +//! exact containment. The value selectors live there and nowhere else. //! //! By default the root is **sealed** ([`JsonOptions::sealed_root`]): the root //! entry's ciphertext is the whole document, the top-level fields are the @@ -60,18 +64,29 @@ //! `N² / 2^97`, negligible for any document. The ordinal still binds the //! entry to its place through the AAD, so a moved or reordered entry fails //! to open as the design requires. The cost is twelve bytes per entry. -//! - **The ordinal field is `o`, not `i`**, because `i` is the EQL -//! envelope's identifier, which the SQL grafts onto an extracted entry. +//! - **Tokens chain by ordinal, not by label, and nothing stores a token.** +//! The design wraps each child's label-derived token under its parent's +//! (`w`) and stores the ordinal (`o`) so the server can find and unmask +//! it. Here a child's token is a hash of its parent's token and its +//! ordinal, so the server derives it as it walks and both fields go. The +//! ordinals under a node are dense from zero, so the walk needs no +//! fan-out bound either: it probes until the first miss. The price is +//! that the client cannot derive an object member's token (its ordinal +//! is a rank among sibling keys it does not know), so it walks from the +//! nearest token it can derive through one gate per step +//! ([`JsonSubtreeKey`]). //! //! # Byte encodings //! //! The document's wire form is JSON, as the EQL domain stores it: -//! sixteen-byte fields (`s`, `vs`, `p`, `w`, `n`, `g`) and the OPE term `op` +//! sixteen-byte fields (`s`, `p`, and the pairs of `q`) and the OPE term `op` //! in lowercase hex, the ciphertext `c` and the header `h` in standard //! base64. The header's bytes are a frozen layout, [`JsonHeader`]. The //! derivations are versioned by their PAE domain labels, as every other //! term kind's are ([`sem`](crate::sem)): a change to any of them is a new //! version, never an edit in place. +use std::collections::BTreeSet; + use serde::{Deserialize, Serialize}; use uuid::Uuid; use vitaminc_protected::OpaqueDebug; @@ -90,14 +105,15 @@ pub use query::{ pub(crate) use derive::{seal_pending, terms_pending}; -/// A sixteen-byte tag: a selector, a token, a parent tag or a group tag. +/// A sixteen-byte tag: a selector, a token or a parent tag. pub type Tag = [u8; 16]; -/// How a JSON index lays a document out. Part of the index's identity: a -/// stored document and the query probing it must agree on it, so it rides -/// in the data plan beside the index and is recorded in every document's -/// header. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +/// How a JSON index lays a document out, and which of its leaves it +/// orders. Part of the index's identity: a stored document and the query +/// probing it must agree on it, so it rides in the data plan beside the +/// index. The layout is recorded in every document's header; the ordered +/// paths are not, since a stored leaf shows whether it carries a term. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct JsonOptions { /// Seal the root: the root entry's ciphertext is the whole document and /// the top-level fields are the heads of the token chain, so no token @@ -105,11 +121,54 @@ pub struct JsonOptions { /// default. Off, the chain starts at the root and a query may walk the /// whole document from the root's token. pub sealed_root: bool, + /// The paths whose number and string leaves carry an ordering term + /// `op`, and so answer `op::AtPath(op::Lt)` and the other orderings. + /// None by default: an ordering term reveals the order of the leaf + /// across every row that holds one, and costs about eighty bytes a + /// leaf, so a path pays for it only when asked. A wildcard step covers + /// every element of the array (`$.orders[*].total`); a position covers + /// that element alone (`$.scores[0]`). An ordering asked at a path no + /// entry here covers is refused + /// ([`TermError::JsonPathNotOrdered`](crate::sem::TermError::JsonPathNotOrdered)). + pub ordered: BTreeSet, +} + +impl JsonOptions { + /// The defaults: the root sealed, no leaf ordered. + pub const fn new() -> Self { + Self { + sealed_root: true, + ordered: BTreeSet::new(), + } + } + + /// These options with the leaves at `paths` ordered, each path parsed + /// as [`JsonPath::parse`] does. + /// + /// # Errors + /// + /// [`JsonPathError`] for a path that is not one. + pub fn ordering(mut self, paths: I) -> Result + where + I: IntoIterator, + I::Item: AsRef, + { + for path in paths { + let _ = self.ordered.insert(JsonPath::parse(path.as_ref())?); + } + Ok(self) + } + + /// Whether the leaf at `path` carries an ordering term: some ordered + /// path [covers](JsonPath::covers) it. + pub fn orders(&self, path: &JsonPath) -> bool { + self.ordered.iter().any(|ordered| ordered.covers(path)) + } } impl Default for JsonOptions { fn default() -> Self { - Self { sealed_root: true } + Self::new() } } @@ -128,8 +187,9 @@ impl Default for JsonOptions { /// | 38 | `tag_len` | 2 | `u16` little-endian | /// | 40 | ZeroKMS key `tag` | `tag_len` | required to retrieve the key | /// -/// The SQL reads the `iv` at offset 22 to compute parent and group tags; it -/// parses nothing else. The epoch is reserved for bounding a revealed token +/// The SQL reads the `iv` at offset 22 to compute parent tags, and the +/// flags at offset 1 to know whether a walk may fall back to the sealed +/// root; it parses nothing else. The epoch is reserved for bounding a revealed token /// in time (the design's "Epochs"); writing it now keeps that a new value /// rather than a new layout. /// @@ -189,13 +249,6 @@ impl JsonHeader { self.sealed_root } - /// The layout this document was laid out under. - pub fn options(&self) -> JsonOptions { - JsonOptions { - sealed_root: self.sealed_root, - } - } - /// The frozen byte layout; see the type docs. pub fn to_bytes(&self) -> Vec { let mut out = Vec::with_capacity(Self::FIXED_LEN + self.tag.len()); @@ -262,21 +315,14 @@ impl JsonHeader { /// One entry of a stored document: one JSON node. /// -/// Every field but `s`, `vs` and `c` is optional, and which are present -/// says what the node is: `p`, `w` and `o` on a node whose parent has a -/// token (everything below the chain's heads); `n` on an array element -/// that is a container; `g` on every entry inside such an element; `op` on -/// a number or string leaf. +/// `s` and `c` are always present. `p` is present on a node whose parent +/// has a token (everything below the chain's heads); `op` on a number or +/// string leaf at a path the index orders ([`JsonOptions::ordered`]). #[derive(Clone, Debug, PartialEq, Eq)] pub struct JsonEntry { s: Tag, - vs: Tag, c: Vec, - o: Option, p: Option, - w: Option, - n: Option, - g: Option, op: Option>, } @@ -285,37 +331,16 @@ impl JsonEntry { pub fn selector(&self) -> &Tag { &self.s } - /// The value selector. - pub fn value_selector(&self) -> &Tag { - &self.vs - } /// The node's ciphertext, as sealed. pub fn ciphertext(&self) -> &[u8] { &self.c } - /// The node's ordinal under its parent, when its parent has a token. - pub fn ordinal(&self) -> Option { - self.o - } /// The parent tag, when the parent has a token. pub fn parent_tag(&self) -> Option<&Tag> { self.p.as_ref() } - /// The wrapped token, when the parent has a token. - pub fn wrapped_token(&self) -> Option<&Tag> { - self.w.as_ref() - } - /// This node's group tag, when it is an array element that is a - /// container. - pub fn group_tag(&self) -> Option<&Tag> { - self.n.as_ref() - } - /// The enclosing array element's group tag, when this node sits inside - /// an element that is a container. - pub fn enclosing_group_tag(&self) -> Option<&Tag> { - self.g.as_ref() - } - /// The leaf ordering term, on a number or string leaf. + /// The leaf ordering term, on a number or string leaf at an ordered + /// path. pub fn ordering_term(&self) -> Option<&[u8]> { self.op.as_deref() } @@ -327,15 +352,15 @@ impl JsonEntry { /// [`DecryptInto`](crate::DecryptInto) opens. /// /// Serializes to the wire form the EQL domain stores: `{"h": "…", "sv": -/// [{"s": "…", "vs": "…", "c": "…", …}, …], "q": {"sv": [{"s", "vs"}, …], -/// "el": […]}}`, where `q` is the document's own containment -/// [projection](Self::projection), written once so the server never -/// rebuilds it; see the [module docs](self#byte-encodings). +/// [{"s": "…", "c": "…", …}, …], "q": {"sv": [{"s", "vs"}, …], "el": +/// […]}}`, where `q` is the document's containment +/// [projection](Self::projection); see the [module docs](self#byte-encodings). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(try_from = "DocumentWire", into = "DocumentWire")] pub struct JsonDocument { header: JsonHeader, entries: Vec, + projection: JsonProjection, } impl JsonDocument { @@ -349,39 +374,40 @@ impl JsonDocument { &self.entries } + /// The document's containment projection: what a [`JsonQuery`] is + /// matched against, stored as `q`. Computed from the plaintext when the + /// document is sealed and read back as stored: like every search field + /// it is unauthenticated (only `c` is), and a tampered `q` changes + /// which rows a containment query returns and nothing else. + pub fn projection(&self) -> &JsonProjection { + &self.projection + } + /// The children of the node whose token is `token`, as the server - /// finds them: every entry whose parent tag is the tag of `token` at - /// some ordinal, with its own token unwrapped. The ordinals tried are - /// `0..F`, where `F` is one more than the largest ordinal in the - /// document, as the SQL walk tries them. A token that names no node - /// yields nothing. + /// finds them: the entries at the ordinals `0, 1, 2, …` under it, + /// probed until the first miss, each with its own token derived. A + /// token that names no node yields nothing. /// - /// What a `jsonb_array_elements(doc, token)` returns; one level of the + /// What `json_tree_children(doc, token)` returns; one level of the /// walk [`subtree`](Self::subtree) runs to the bottom. pub fn children(&self, token: &Tag) -> Vec<(usize, &JsonEntry, Tag)> { let iv = &self.header.iv; - let fanout = self - .entries - .iter() - .filter_map(|e| e.o) - .max() - .map_or(0, |o| o + 1); - let tags: Vec = (0..fanout) - .map(|o| derive::parent_tag(iv, token, o)) - .collect(); - let mut out: Vec<(usize, &JsonEntry, Tag)> = self - .entries - .iter() - .enumerate() - .filter_map(|(j, e)| { - let (p, w, o) = (e.p.as_ref()?, e.w.as_ref()?, e.o?); - (tags.get(o as usize) == Some(p)).then(|| { - let child = derive::xor(w, &derive::mask(iv, token, &e.c)); - (j, e, child) - }) - }) - .collect(); - out.sort_by_key(|(_, e, _)| e.o); + let mut out = Vec::new(); + // A node has at most as many children as the document has entries, + // so the probe loop is bounded whatever the entries say. + let most = u32::try_from(self.entries.len()).unwrap_or(u32::MAX); + for o in 0..most { + let tag = derive::parent_tag(iv, token, o); + let Some((j, entry)) = self + .entries + .iter() + .enumerate() + .find(|(_, e)| e.p.as_ref() == Some(&tag)) + else { + break; + }; + out.push((j, entry, derive::child_token(token, o))); + } out } @@ -396,6 +422,13 @@ impl JsonDocument { } fn walk<'a>(&'a self, token: &Tag, depth: u32, out: &mut Vec<(usize, &'a JsonEntry, u32)>) { + // A walk finds each entry at most once, so a document whose tags do + // not form a tree (which no sealing produces) ends the walk once it + // has returned as many entries as the document holds, instead of + // looping. + if out.len() >= self.entries.len() { + return; + } for (j, entry, child) in self.children(token) { out.push((j, entry, depth)); self.walk(&child, depth + 1, out); @@ -413,8 +446,16 @@ impl JsonDocument { }) } - pub(crate) fn new(header: JsonHeader, entries: Vec) -> Self { - Self { header, entries } + pub(crate) fn new( + header: JsonHeader, + entries: Vec, + projection: JsonProjection, + ) -> Self { + Self { + header, + entries, + projection, + } } } @@ -485,11 +526,13 @@ where /// [`DecryptInto`](crate::DecryptInto) reassembles into the value /// under that node. /// -/// The walk returns the node's descendants and not the node itself, so the -/// node's kind is read from its children's labels: elements make an array, -/// members an object. A node with no descendants, an empty container or a -/// scalar, yields no rows and so no value; `->` on its selector returns the -/// node itself. +/// The ungated walk returns the node's descendants and not the node +/// itself, so the node's kind is read from its children's labels: elements +/// make an array, members an object. A node with no descendants, an empty +/// container or a scalar, yields no rows and so no value; `->` on its +/// selector returns the node itself. A gated walk, +/// `json_tree_subtree_at(doc, token, gates)`, also returns each node its +/// gates reach, at depth `0`, with that node's descendants beneath it. /// /// Past its budget, a walk over a document with a sealed root returns the /// root entry alone, at depth `0`, instead of a subtree it would be @@ -611,16 +654,14 @@ fn unbase64(text: &str) -> Result, WireError> { #[error("a JSON index wire field does not decode: {0}")] pub struct WireError(&'static str); -/// `h`, `sv` and `q`, as stored. `q` is written from the entries and not -/// read back: a document deserialized from the wire derives its projection -/// from `sv` again, so the field is accepted and ignored on the way in. +/// `h`, `sv` and `q`, as stored. All three are required: the projection +/// `q` is the only place the value selectors live. #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct DocumentWire { h: String, sv: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - q: Option, + q: query::GroupWire, } /// One entry, as stored: tags in hex, the ciphertext in base64. @@ -628,19 +669,10 @@ struct DocumentWire { #[serde(deny_unknown_fields)] struct EntryWire { s: String, - vs: String, c: String, #[serde(default, skip_serializing_if = "Option::is_none")] - o: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] p: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - w: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - n: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - g: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] op: Option, } @@ -648,13 +680,8 @@ impl From<&JsonEntry> for EntryWire { fn from(e: &JsonEntry) -> Self { Self { s: hex(&e.s), - vs: hex(&e.vs), c: base64(&e.c), - o: e.o, p: e.p.as_ref().map(|t| hex(t)), - w: e.w.as_ref().map(|t| hex(t)), - n: e.n.as_ref().map(|t| hex(t)), - g: e.g.as_ref().map(|t| hex(t)), op: e.op.as_deref().map(hex), } } @@ -666,13 +693,8 @@ impl TryFrom for JsonEntry { fn try_from(w: EntryWire) -> Result { Ok(Self { s: unhex16(&w.s)?, - vs: unhex16(&w.vs)?, c: unbase64(&w.c)?, - o: w.o, p: w.p.as_deref().map(unhex16).transpose()?, - w: w.w.as_deref().map(unhex16).transpose()?, - n: w.n.as_deref().map(unhex16).transpose()?, - g: w.g.as_deref().map(unhex16).transpose()?, op: w.op.as_deref().map(unhex).transpose()?, }) } @@ -683,7 +705,7 @@ impl From for DocumentWire { Self { h: base64(&doc.header.to_bytes()), sv: doc.entries.iter().map(EntryWire::from).collect(), - q: Some(query::GroupWire::from(&doc.projection())), + q: query::GroupWire::from(&doc.projection), } } } @@ -698,7 +720,12 @@ impl TryFrom for JsonDocument { w.sv.into_iter() .map(JsonEntry::try_from) .collect::>()?; - Ok(Self { header, entries }) + let projection = JsonProjection::try_from(w.q)?; + Ok(Self { + header, + entries, + projection, + }) } } @@ -784,9 +811,8 @@ mod tests { assert_eq!(header.iv(), &[9; 16]); assert_eq!(header.tag(), &[1, 2, 3]); assert!(!header.sealed_root()); - assert_eq!(header.options(), JsonOptions { sealed_root: false }); let sealed = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![], true); - assert_eq!(sealed.options(), JsonOptions { sealed_root: true }); + assert!(sealed.sealed_root()); } #[test] diff --git a/packages/stack-encrypt/src/json/path.rs b/packages/stack-encrypt/src/json/path.rs index 87ce2c960..965b89acc 100644 --- a/packages/stack-encrypt/src/json/path.rs +++ b/packages/stack-encrypt/src/json/path.rs @@ -17,7 +17,7 @@ use std::fmt; use std::str::FromStr; /// One step of a [`JsonPath`]. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Segment { /// An object member, by key. Key(String), @@ -28,7 +28,7 @@ pub enum Segment { } /// A path into a JSON document, from the root (`$`) down. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] pub struct JsonPath { segments: Vec, } @@ -152,6 +152,41 @@ impl JsonPath { !self.segments.iter().any(|s| matches!(s, Segment::Wildcard)) } + /// Whether this path, read as a pattern, covers `path`: the same + /// steps, except that a wildcard here covers any array step there, a + /// position or the wildcard. A position here covers that position + /// alone, not the wildcard, since the wildcard names elements the + /// position does not. How the index's ordered paths + /// ([`JsonOptions::ordered`](super::JsonOptions::ordered)) are matched + /// against a node's path when a document is sealed, and against a + /// query's path when an ordering is asked. + /// + /// ``` + /// use stack_encrypt::json::JsonPath; + /// + /// let wild = JsonPath::parse("$.orders[*].total").unwrap(); + /// assert!(wild.covers(&JsonPath::parse("$.orders[3].total").unwrap())); + /// assert!(wild.covers(&JsonPath::parse("$.orders[*].total").unwrap())); + /// let third = JsonPath::parse("$.orders[3].total").unwrap(); + /// assert!(third.covers(&JsonPath::parse("$.orders[3].total").unwrap())); + /// assert!(!third.covers(&JsonPath::parse("$.orders[2].total").unwrap())); + /// assert!(!third.covers(&JsonPath::parse("$.orders[*].total").unwrap())); + /// assert!(!wild.covers(&JsonPath::parse("$.orders[*]").unwrap())); + /// ``` + pub fn covers(&self, path: &JsonPath) -> bool { + self.segments.len() == path.segments.len() + && self + .segments + .iter() + .zip(&path.segments) + .all(|(pattern, step)| match (pattern, step) { + (Segment::Key(a), Segment::Key(b)) => a == b, + (Segment::Wildcard, Segment::Index(_) | Segment::Wildcard) => true, + (Segment::Index(a), Segment::Index(b)) => a == b, + _ => false, + }) + } + /// The values at this path in `value`, in document order: at most one /// for a positional path, one per element a wildcard reaches, none when /// the path names nothing. A key step into an array, or a position @@ -186,21 +221,6 @@ impl JsonPath { Self { segments } } - /// The longest prefix with no wildcard, and whether anything was cut. - pub(crate) fn anchor(&self) -> (Self, bool) { - let end = self - .segments - .iter() - .position(|s| matches!(s, Segment::Wildcard)) - .unwrap_or(self.segments.len()); - ( - Self { - segments: self.segments[..end].to_vec(), - }, - end < self.segments.len(), - ) - } - /// The canonical bytes a selector is derived over: every array step, /// positional or wildcard, is the one array tag, so every element of an /// array shares its parent's selector. A key is length-prefixed, so two @@ -457,16 +477,6 @@ mod tests { ); } - #[test] - fn the_anchor_is_the_prefix_before_the_first_wildcard() { - let (anchor, cut) = JsonPath::parse("$.a[*].x").unwrap().anchor(); - assert_eq!(anchor, JsonPath::parse("$.a").unwrap()); - assert!(cut); - let (anchor, cut) = JsonPath::parse("$.a[1].x").unwrap().anchor(); - assert_eq!(anchor.to_string(), "$.a[1].x"); - assert!(!cut); - } - #[test] fn a_needle_holds_the_value_at_the_path() { let needle = JsonPath::parse("$.a[*].x") diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index c5bac8c53..299332535 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -2,9 +2,9 @@ //! //! A containment needle is the **element-nested projection** of the //! needle's terms: the selector pairs `{s, vs}` of every node, grouped by -//! the array element they sit in. The server projects a stored document the -//! same way from its group tags, and native `jsonb @>` over the two -//! projections is exact containment: each needle element has to fit inside +//! the array element they sit in. A stored document carries its own +//! projection, built the same way, as `q`, which the server reads, and +//! native `jsonb @>` over the two projections is exact containment: each needle element has to fit inside //! one stored element, which is PostgreSQL's own rule for arrays. The //! design's "The preferred form" records why no re-check is needed. use std::fmt; @@ -14,7 +14,7 @@ use serde_json::Value; use super::derive::{JsonTerms, TermNode}; use super::path::JsonPath; -use super::{hex, JsonDocument, JsonEntry, JsonOptions, Tag}; +use super::{hex, JsonDocument, JsonOptions, Tag}; use crate::op::{ Answers, AtPath, Between, ContainedBy, Contains, DefaultIndex, Eq, Gt, Gte, Input, LabelledTarget, Lt, Lte, One, Pair, Pathed, QueryTerm, QueryToken, Supports, @@ -32,8 +32,8 @@ struct Pair16 { /// The element-nested projection of a document: the selector pairs that /// sit outside any array element, and one nested projection per array -/// element that is a container. What the server computes from a stored -/// document's group tags (`json_tree_query`), and what a containment +/// element that is a container. What a stored document carries as `q`, +/// which the server reads (`json_tree_query`), and what a containment /// needle is. #[derive(Clone, Debug, PartialEq, Eq, Default)] pub struct JsonProjection { @@ -95,8 +95,9 @@ impl JsonQuery { } /// The group of the nodes that sit directly in `parent` (`None` is the top -/// level), with the container elements in it nested as groups. -fn group_of(nodes: &[TermNode], parent: Option) -> JsonProjection { +/// level), with the container elements in it nested as groups. What a +/// needle is, and what a document writes to `q` when it is sealed. +pub(crate) fn group_of(nodes: &[TermNode], parent: Option) -> JsonProjection { let belongs = |index: usize, node: &TermNode| { if node.container_element { Some(index) @@ -120,34 +121,6 @@ fn group_of(nodes: &[TermNode], parent: Option) -> JsonProjection { } } -impl JsonDocument { - /// The stored document's projection, from its group tags: what a - /// [`JsonQuery`] is matched against. The document writes it to the wire - /// as `q`, beside `sv`, and the SQL's `json_tree_query` reads it from - /// there; a document deserialized from the wire derives it again from - /// the entries, so a stored `q` is never trusted over them. - pub fn projection(&self) -> JsonProjection { - fn belongs(e: &JsonEntry) -> Option<&Tag> { - e.n.as_ref().or(e.g.as_ref()) - } - fn group(entries: &[JsonEntry], parent: Option<&Tag>) -> JsonProjection { - JsonProjection { - sv: entries - .iter() - .filter(|e| belongs(e) == parent) - .map(|e| Pair16 { s: e.s, vs: e.vs }) - .collect(), - el: entries - .iter() - .filter(|e| e.n.is_some() && e.g.as_ref() == parent) - .map(|e| group(entries, e.n.as_ref())) - .collect(), - } - } - group(&self.entries, None) - } -} - #[derive(Clone, Serialize, Deserialize)] struct PairWire { s: String, @@ -185,6 +158,20 @@ impl From<&JsonProjection> for GroupWire { } } +/// A projection serializes as a stored `q` and a needle do: `{"sv": [{"s", +/// "vs"}, …], "el": [group, …]}`. +impl Serialize for JsonProjection { + fn serialize(&self, s: S) -> Result { + GroupWire::from(self).serialize(s) + } +} + +impl<'de> Deserialize<'de> for JsonProjection { + fn deserialize>(d: D) -> Result { + JsonProjection::try_from(GroupWire::deserialize(d)?).map_err(serde::de::Error::custom) + } +} + impl TryFrom for JsonProjection { type Error = super::WireError; fn try_from(w: GroupWire) -> Result { @@ -326,13 +313,35 @@ impl Serialize for JsonSelector { } } -/// The key of a subtree walk: the token of a positional node, and, when -/// the walked path continues with a wildcard below it, the selector of the -/// whole path the walk gates on. Serializes as `{"t": "…", "s": "…"}`. +/// One gate of a subtree walk: what the entry at one depth below the +/// walk's token must be for the walk to descend through it. #[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Gate { + /// The selector of the path down to that depth: a key step, or a + /// wildcard, whose selector every element carries. + Selector(Tag), + /// An array position: the one ordinal probed at that depth. + Slot(u32), +} + +impl Gate { + /// The gate as `json_tree_subtree_at` takes it: the selector's hex, or + /// `#n` for a position. + fn text(&self) -> String { + match self { + Gate::Selector(s) => hex(s), + Gate::Slot(n) => format!("#{n}"), + } + } +} + +/// The key of a subtree walk: the token of the deepest node on the path +/// the client can derive, and one gate per step below it. Serializes as +/// `{"t": "…", "g": ["…", "#0", …]}`, `g` omitted when empty. +#[derive(Clone, PartialEq, Eq)] pub struct JsonSubtreeKey { pub(crate) token: Tag, - pub(crate) selector: Option, + pub(crate) gates: Vec, } impl JsonSubtreeKey { @@ -348,10 +357,12 @@ impl JsonSubtreeKey { &self.token } - /// The selector the walk gates on, when the path has a wildcard below - /// the token's node. - pub fn selector(&self) -> Option { - self.selector.map(JsonSelector) + /// The gates the walk descends through below the token, one per step, + /// as `json_tree_subtree_at` takes them: a selector's hex, or `#n` for + /// an array position. Empty when the token names the walked node + /// itself, which `json_tree_subtree` takes. + pub fn gates(&self) -> Vec { + self.gates.iter().map(Gate::text).collect() } } @@ -359,7 +370,7 @@ impl JsonSubtreeKey { impl fmt::Debug for JsonSubtreeKey { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("JsonSubtreeKey") - .field("gated", &self.selector.is_some()) + .field("gates", &self.gates.len()) .finish_non_exhaustive() } } @@ -369,12 +380,12 @@ impl Serialize for JsonSubtreeKey { #[derive(Serialize)] struct Wire { t: String, - #[serde(skip_serializing_if = "Option::is_none")] - s: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + g: Vec, } Wire { t: hex(&self.token), - s: self.selector.as_ref().map(|t| hex(t)), + g: self.gates(), } .serialize(s) } @@ -572,9 +583,21 @@ fn ordered(value: &Value) -> Result<(), Error> { } } +/// Whether the index orders the leaves at `path`: some path in its +/// `ordered` option covers it. +fn orders(index: &Json, path: &JsonPath) -> Result<(), Error> { + if index.options.orders(path) { + Ok(()) + } else { + Err(TermError::JsonPathNotOrdered.into()) + } +} + /// The orderings at a path: the path's selector and the value's leaf /// ordering term. Refused, when the query is awaited, for a value that is -/// neither a number nor a string. +/// neither a number nor a string, and for a path the index does not order +/// ([`JsonOptions::ordered`]): no stored leaf there carries a term, so the +/// comparison would match no row. macro_rules! orderings { ($($op:ident),+ $(,)?) => {$( impl Supports, (P, V)> for Json @@ -585,7 +608,8 @@ macro_rules! orderings { type Query = JsonLeafQuery; type Verifier = (); fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { - let _ = parse_path(path.as_ref())?; + let path = parse_path(path.as_ref())?; + orders(self, &path)?; ordered(&value.clone().into()) } fn verifier(&self, _: &(P, V)) {} @@ -619,7 +643,8 @@ where type Query = RangeQuery; type Verifier = (); fn check(&self, (path, (lo, hi)): &(P, (V, V))) -> Result<(), Error> { - let _ = parse_path(path.as_ref())?; + let path = parse_path(path.as_ref())?; + orders(self, &path)?; ordered(&lo.clone().into())?; ordered(&hi.clone().into()) } @@ -652,9 +677,9 @@ where /// plan that declares the index carries them, and a query through it takes /// them from there. impl Json { - /// The index's layout options. - pub fn options(&self) -> JsonOptions { - self.options + /// The index's options: its layout, and the paths it orders. + pub fn options(&self) -> &JsonOptions { + &self.options } } @@ -701,16 +726,28 @@ mod tests { ); let key = JsonSubtreeKey { token: [0xcd; 16], - selector: Some([0xab; 16]), + gates: vec![Gate::Selector([0xab; 16]), Gate::Slot(3)], }; assert_eq!(key.token(), "cd".repeat(16)); assert_eq!(key.token_bytes(), &[0xcd; 16]); - assert_eq!(key.selector(), Some(selector)); + assert_eq!(key.gates(), ["ab".repeat(16), "#3".to_owned()]); + assert_eq!( + serde_json::to_value(&key).unwrap(), + serde_json::json!({"t": "cd".repeat(16), "g": ["ab".repeat(16), "#3"]}) + ); assert_eq!( format!("{key:?}"), - "JsonSubtreeKey { gated: true, .. }", + "JsonSubtreeKey { gates: 2, .. }", "a token is a capability: it never prints" ); + let bare = JsonSubtreeKey { + token: [0xcd; 16], + gates: Vec::new(), + }; + assert_eq!( + serde_json::to_value(&bare).unwrap(), + serde_json::json!({"t": "cd".repeat(16)}) + ); } #[test] @@ -725,9 +762,52 @@ mod tests { #[test] fn the_index_reports_its_options() { - let open = JsonOptions { sealed_root: false }; - assert_eq!(Json::new(open).options(), open); - assert_eq!(Json::default().options(), JsonOptions::default()); + let open = JsonOptions { + sealed_root: false, + ..JsonOptions::new() + }; + assert_eq!(Json::new(open.clone()).options(), &open); + assert_eq!(Json::default().options(), &JsonOptions::default()); + } + + #[test] + fn a_projection_round_trips_through_its_wire_form() { + let projection = JsonProjection { + sv: vec![Pair16 { + s: [1; 16], + vs: [2; 16], + }], + el: vec![JsonProjection { + sv: vec![Pair16 { + s: [3; 16], + vs: [4; 16], + }], + el: Vec::new(), + }], + }; + let wire = serde_json::to_value(&projection).unwrap(); + assert_eq!( + wire, + serde_json::json!({ + "sv": [{"s": "01".repeat(16), "vs": "02".repeat(16)}], + "el": [{"sv": [{"s": "03".repeat(16), "vs": "04".repeat(16)}], "el": []}], + }) + ); + let back: JsonProjection = serde_json::from_value(wire).unwrap(); + assert_eq!(back, projection); + assert!( + serde_json::from_value::(serde_json::json!({"sv": [], "el": []})) + .unwrap() + .el + .is_empty() + ); + assert!( + serde_json::from_value::( + serde_json::json!({"sv": [{"s": "zz", "vs": "00"}], "el": []}) + ) + .is_err(), + "a pair that is not two tags is refused" + ); } #[test] diff --git a/packages/stack-encrypt/src/plan/chain.rs b/packages/stack-encrypt/src/plan/chain.rs index 21cf04262..2096fc562 100644 --- a/packages/stack-encrypt/src/plan/chain.rs +++ b/packages/stack-encrypt/src/plan/chain.rs @@ -9,12 +9,12 @@ use uuid::Uuid; use super::build::{FieldPlan, FieldsBuilder, IntoLabel, Opens, PlanContext, Runs}; use super::field_ref::FieldRef; -use super::value::{Indexed, Typed, ValueLayout, ValuePlan, ValueShape}; +use super::value::{Indexed, Laid, Typed, ValueLayout, ValuePlan, ValueShape}; use super::PlanError; use crate::op::{Answers, DefaultIndex, Input, LabelledTarget, Operator, QueryToken, Supports}; use crate::target::{ ciphertext, Borrowed, CallerContext, DeclaredContext, DecryptField, Decryptable, EncryptFrom, - Index, Indexes, Pending, PendingFuture, Select, Whole, + Index, Indexes, Layout, Pending, PendingFuture, Select, Whole, }; use crate::{Error, KeysetCipher, Label, LabelError, NonEmpty, StackCipher, StackCipherText}; @@ -777,12 +777,30 @@ where } } +/// A plan laid out by value answers the operators its layout's index +/// supports, through that index, under the plan's options. +impl Chooses for ValuePlan> +where + Op: Operator, + In: Input, + S: 'static, + X: Layout, + X::Index: Supports + Default, + Laid: ValueShape, +{ + type Via = X::Index; + fn choose(&self, call: Option