From c165a6975d3c622de629e3167759ab7e830a6a38 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 14 Oct 2026 02:56:25 +0000 Subject: [PATCH] Raise snapshot field cap to 64 and refuse over-cap at mutate. Default MEMNET_MAX_FIELDS is 64. Mutate and snapshot save share one effective-field count so a CREATE/SET/MERGE that could not save is refused with limit_exceeded|fields n/max and writes nothing. snapshot_unsaveable stays the fail-closed save backstop (MN-REQ-05.4). Co-authored-by: chouswei --- CHANGELOG.md | 1 + docs/cap-contract.md | 8 +- docs/operations/admin-usage-report.md | 2 +- docs/operations/one-session-per-document.md | 12 +- parts/common/memnet/memnet/config.py | 2 +- parts/common/memnet/memnet/mutate_gate.py | 7 + parts/common/memnet/memnet/snapshot.py | 75 ++++++++-- scripts/probe_doc_gate_readiness.py | 42 ++++-- sysml-models/models/deploy.sysml | 10 ++ sysml-models/models/requirements.sysml | 38 ++++- sysml-models/models/verify.sysml | 67 ++++++++- tests/cap_contract_lib.py | 2 +- tests/doc_gate_lib.py | 11 +- tests/test_cap_contract.py | 2 +- tests/test_doc_gate_readiness.py | 36 +++-- tests/test_engine_roundtrip_where_acl.py | 6 +- tests/test_expire_save_keep.py | 6 +- tests/test_field_cap.py | 158 ++++++++++++++++++++ tests/test_sysml_engine_bugs.py | 16 +- 19 files changed, 427 insertions(+), 74 deletions(-) create mode 100644 tests/test_field_cap.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7355d074..cae4bb9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. ## [Unreleased] ### Changed +- **Honesty `c` — fields-per-tag cap 64; mutate refuses over-cap (MN-REQ-05.4)** — Default `MEMNET_MAX_FIELDS` is 64. Mutate and snapshot save share one effective-field count (declared SCHEMA fields + undeclared extras on any row of that tag + the `id` column save appends). A CREATE / SET / MERGE that would exceed the cap refuses `@ERR: limit_exceeded|fields n/max` and writes nothing. Snapshot load of a SCHEMA over 64 refuses the same code. `snapshot_unsaveable` stays the fail-closed save backstop. Verify: MN-VER-05-S02 (`tests/test_field_cap.py`). - **Invent only — ClusterRoute vs SliceHandCarry (#191 / #47 cousin)** — `MemNetTwoMoves` outside `MemNetSystem` (`MN-REQ-06.9` + `MN-REQ-06.10` / `MN-VER-06-S08`). ClusterRoute = where the session lives (`MemNetLanMcpFront`; one owner; `pin_map` / `find` SHALL NOT span backends). SliceHandCarry = explicit copy into another session (`export_pin_map` or `session_save` → LAN file copy → dest import/`session_load`; `import_slice` same-serve only). Not a live hop. `import_slice(from_url)` not shipped. tip≠face. `inventOnly=true`; `implemented=false`; no engine code; no SemVer bump. Wire: [`docs/operations/cluster-route-vs-slice-hand-carry.md`](docs/operations/cluster-route-vs-slice-hand-carry.md). - **Invent only — LAN MCP front over several serves (#191)** — `MemNetLanMcpFront` outside `MemNetSystem` (`MN-REQ-06.9` / `MN-VER-06-S07`). One MCP catalogue, N LAN `memnet serve` backends; `SessionOwnerRegistry` is owner (explicit pin allowed; silent hash is not sole routing). One owner per session; `pin_map` / `find` SHALL NOT span backends. Cousin of #47 (peer sid handoff), not the same invent. tip≠face. `inventOnly=true`; `implemented=false`; no engine code; no SemVer bump. Wire: [`docs/operations/memnet-lan-mcp-front.md`](docs/operations/memnet-lan-mcp-front.md). diff --git a/docs/cap-contract.md b/docs/cap-contract.md index 1a29a55a..e7ded338 100644 --- a/docs/cap-contract.md +++ b/docs/cap-contract.md @@ -142,9 +142,9 @@ Path-B ingest commits with `allow_new_relation=True`. | Limit | Default | Knob | Wire | |-------|---------|------|------| | User tags | 64 | `MEMNET_MAX_TAGS` | `@ERR: limit_exceeded\|tags {n}/{max}` | -| Fields per tag | 32 | `MEMNET_MAX_FIELDS` | `@ERR: limit_exceeded\|fields {n}/{max}` | +| Fields per tag | 64 | `MEMNET_MAX_FIELDS` | `@ERR: limit_exceeded\|fields {n}/{max}` | -Raised at map load (`memnet/tag_map.py`). Session open fails; nothing is stored. +SCHEMA register refuses a declared map over the cap (`memnet/tag_map.py`). Mutate uses the same cap on the **effective** field set: declared SCHEMA fields, plus undeclared extras on any row of that tag, plus the `id` column snapshot save appends. Over-cap CREATE/SET/MERGE refuses `limit_exceeded|fields {n}/{max}` and writes nothing. Snapshot save uses that same helper (`memnet.snapshot.effective_snapshot_fields`); it is never stricter than a mutate that succeeded. Snapshot load of a SCHEMA over the cap refuses the same `fields {n}/{max}` code. `snapshot_unsaveable` stays the fail-closed save backstop when RAM still exceeds the cap. ## Pipe value / line bytes @@ -161,9 +161,9 @@ Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, Mutate stdin, leftover pipe batches, and GQL statement breaks use that same LF-only split (one optional trailing CR is stripped). A raw VT, FF, FS, GS, RS, NEL, LS, or PS inside a value stays inside the value. It does not start another statement. An embedded CR is not a break. A raw LF still separates statements. This matches snapshot load (#202): the separator is kept literally, not refused. -**Undeclared properties.** GQL mutate may store keys that are absent from the live tag SCHEMA (GraphElement extras; Path-B locators such as `qname`). Those keys stay in RAM. Snapshot save persists them by widening the **snapshot** SCHEMA (live session SCHEMA is unchanged). The widen keys extras by the canonical SCHEMA tag, so a CREATE label whose spelling differs in case from the SCHEMA tag still persists. After load, the restored map includes the extra columns. Extras on fixed tags `EDG` / `LAW`, or a widened SCHEMA over `max_fields`, refuse `snapshot_unsaveable` and write no file. +**Undeclared properties.** GQL mutate may store keys that are absent from the live tag SCHEMA (GraphElement extras; Path-B locators such as `qname`). Those keys stay in RAM. Snapshot save persists them by widening the **snapshot** SCHEMA (live session SCHEMA is unchanged). The widen keys extras by the canonical SCHEMA tag, so a CREATE label whose spelling differs in case from the SCHEMA tag still persists. After load, the restored map includes the extra columns. Extras on fixed tags `EDG` / `LAW` refuse `snapshot_unsaveable` and write no file. A mutate that would widen a tag over `max_fields` refuses `limit_exceeded|fields {n}/{max}` and writes nothing. Save uses the same effective-field count; `snapshot_unsaveable` wrapping `fields|{n}/{max}` remains the backstop if RAM still exceeds the cap. -A live SCHEMA that omits `id` is accepted at open (`id` need not be present or first). Save still succeeds: the snapshot SCHEMA appends an `id` column and the row gets a minted nickname. The live map is unchanged until that snapshot is loaded. A property key that does not survive the SCHEMA whitespace split (for example `foo bar`) cannot reload, so save refuses `snapshot_unsaveable` and writes no file. A 32-field map with no `id` becomes 33 columns and refuses `fields|33/32`. +A live SCHEMA that omits `id` is accepted at open (`id` need not be present or first). Save still succeeds: the snapshot SCHEMA appends an `id` column and the row gets a minted nickname. The live map is unchanged until that snapshot is loaded. A property key that does not survive the SCHEMA whitespace split (for example `foo bar`) cannot reload, so save refuses `snapshot_unsaveable` and writes no file. A 64-field map with no `id` becomes 65 columns and refuses `fields|65/64`. **Labelled MATCH and edge create.** `MATCH (n:Label {…}) SET` is intended to match. The label is case-folded to the SCHEMA tag (`cst`, `Cst`, and `CST` are the same tag). A label that is a different known tag and matches nothing is `@ERR: not_found`, not a silent success. `MATCH (a) MATCH (b) CREATE (a)-[:rel]->(b)` is `@ERR: parse_error` (`unsupported MATCH continuation`). The comma form `MATCH (a), (b) CREATE (a)-[:rel]->(b)` with `--allow-new-relation` creates the edge. The MATCH-MATCH form is not implemented. diff --git a/docs/operations/admin-usage-report.md b/docs/operations/admin-usage-report.md index 0cdbcf9d..2b341dc8 100644 --- a/docs/operations/admin-usage-report.md +++ b/docs/operations/admin-usage-report.md @@ -84,7 +84,7 @@ A refuse MUST NOT look like a complete empty report (`ok` JSON). If a measured f "max_relations": 200, "max_law": 100, "max_tags": 64, - "max_fields": 32, + "max_fields": 64, "max_value_bytes": 4096, "max_line_bytes": 32768, "max_batch_lines": 1000, diff --git a/docs/operations/one-session-per-document.md b/docs/operations/one-session-per-document.md index b1f6f365..5b1bbeb7 100644 --- a/docs/operations/one-session-per-document.md +++ b/docs/operations/one-session-per-document.md @@ -49,7 +49,7 @@ Client helper: `memnet.serve.send_command(args, stdin=…, host=…, port=…)`. `session save --file` writes `# memnet-snapshot-v1` (`@SNAP` version 2) through a temp file created `0600`, fsynced, then renamed over the target (overwrite, atomic). MemNet does **not** make that file write-once (no `O_EXCL` on the target, no immutable flag). The caller or the filesystem can. -Anything mutate accepts must save and reload as the same property values. Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, FS/GS/RS, NEL, LS, PS) plus `\` and `|`; load splits records on LF only. A 16 KiB string is refused at GQL CREATE/SET with `@ERR: limit_exceeded|value_bytes 16384/4096` (one decoded cap with leftover pipe and snapshot load). Unicode, `|`, quotes, CR, and newlines on a value under the cap round-trip. Undeclared RAM keys persist by widening the snapshot SCHEMA; extras over `max_fields` refuse `snapshot_unsaveable`. +Anything mutate accepts must save and reload as the same property values. Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, FS/GS/RS, NEL, LS, PS) plus `\` and `|`; load splits records on LF only. A 16 KiB string is refused at GQL CREATE/SET with `@ERR: limit_exceeded|value_bytes 16384/4096` (one decoded cap with leftover pipe and snapshot load). Unicode, `|`, quotes, CR, and newlines on a value under the cap round-trip. Undeclared RAM keys persist by widening the snapshot SCHEMA. A mutate that would exceed `max_fields` (default 64) refuses `@ERR: limit_exceeded|fields n/max` and writes nothing. Save uses that same effective-field count; `snapshot_unsaveable` stays the backstop. Expire: with save-on-expire and a dir, TTL drop writes `{dir}/{sid}.snap` (do not log the name). Next use: `@ERR: session_expired|snap_available`. Restore: `session load --session ` (no `--file`). If that write cannot complete (`snapshot_unsaveable`, unwritable disk or directory, or any other save failure), MemNet does **not** drop the session. It stays in RAM, still counts against `MEMNET_MAX_SESSIONS`, and every sweep or access that retries expiry emits `@WRN: expire_snapshot_failed|{code}`. Access after TTL is `@ERR: session_expired|overdue` until an explicit `session save` succeeds or `session close`. `session expire-status`, `session list`, and the admin usage report expose `@STAT: expire_snapshot_failed|n|` / `sessions.expire_snapshot_failed`. When save-on-expire is off, TTL still drops RAM. @@ -81,7 +81,7 @@ python scripts/probe_doc_gate_readiness.py --out /opt/cursor/artifacts/doc-gate- | E13 | 16 KiB strings with LaTeX / quotes / newline / `\|` / CJK are refused at GQL CREATE/SET (`value_bytes 16384/4096`). Pipe leftover, GQL mutate, and snapshot load share decoded value 4096. `line_bytes` 32768 is leftover-pipe / snapshot escaped line. GQL string escapes: `\\ \' \" \n \r \t`. | | E14 | List literals store as JSON strings and emit as GQL lists. `'k' IN p.citeKeys` honours membership on MATCH…SET. Locators are `KEY=VAL` equality on the JSON string; leftover `read list --where` can glob that string. | | E17 | `WHERE n.value CONTAINS` is honoured on SET/DELETE. RETURN → `product_gate`. Unique MATCH miss → `not_found` (nothing SET). SET of \|Q\|>1 after WHERE is still `cue_conflict` (unique-SET law). `STARTS WITH` / `ENDS WITH` / `=~` same. Keyword: `find`/`pin_map --keyword` (casefold). See paragraph below. | -| E18 | Snapshot `value_bytes` 4096 is the **decoded** field after `split_payload` (`>` not `>=`); `join_payload` expansion of splitlines separators / `\\` / `\|` is not the value cap. `line_bytes` 32768 is the escaped/raw snapshot line. SCHEMA `max_fields=32`. Tab, CR, and LF round-trip. Undeclared extras persist unless they exceed `max_fields`. Live table below. | +| E18 | Snapshot `value_bytes` 4096 is the **decoded** field after `split_payload` (`>` not `>=`); `join_payload` expansion of splitlines separators / `\\` / `\|` is not the value cap. `line_bytes` 32768 is the escaped/raw snapshot line. SCHEMA `max_fields=64`. Tab, CR, and LF round-trip. Undeclared extras persist unless they exceed `max_fields` (mutate refuses `limit_exceeded|fields n/max`). Live table below. | Second RSS fixture: 3000 nodes, no edges, 1500 of them with 2/3/4 KiB text (about 4.4 MiB of UTF-8 payload, not 1 MiB). Measure process RSS the same way as the 1800-part fixture. Short fat churn is on (`--fat-churn`, default 8); 110 cycles of this fixture is not the default. @@ -96,7 +96,7 @@ E16 (on the 10000 fulldoc session, this VM `Intel(R) Xeon(R) Processor` 4-core K **note:** MemNet has **no** native delete-refused-while-referenced check. `DETACH DELETE` of a node with inbound edges exits 0 and leaves dangling edges. The gate must refuse from the reverse lookup (`## Truncation truncated=true M=400 omitted=2604 reason=max_rows` on the hub). `MATCH ()-[r {id}]-() DELETE r` honours relationship DELETE. `MATCH (n WHERE true)-[r {id}]->() DELETE r` remains a valid spelling. -E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_payload` (`check_value_bytes`: decoded length `>` cap, so 4096 passes). `join_payload` expansion of splitlines separators / `\\` / `|` is not the value cap (4000 `\\` or `|` emit 8000 escaped bytes, snap line ~8021, still loads). `parse_line` measures the escaped/raw line vs `line_bytes` 32768 first; save verify uses the same check. SCHEMA register vs `max_fields=32`. Snapshot save widens SCHEMA for undeclared RAM keys; leftover `emit_record` still writes SCHEMA columns. Loopback CREATE → save → load into a fresh session → `pin_map` cue. Binary search skipped (4000 exact for `\\` and `|`). +E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_payload` (`check_value_bytes`: decoded length `>` cap, so 4096 passes). `join_payload` expansion of splitlines separators / `\\` / `|` is not the value cap (4000 `\\` or `|` emit 8000 escaped bytes, snap line ~8021, still loads). `parse_line` measures the escaped/raw line vs `line_bytes` 32768 first; save verify uses the same check. SCHEMA register vs `max_fields=64`. Mutate and snapshot save share one effective-field count (declared + extras + appended `id`). Snapshot save widens SCHEMA for undeclared RAM keys; leftover `emit_record` still writes SCHEMA columns. Loopback CREATE → save → load into a fresh session → `pin_map` cue. Binary search skipped (4000 exact for `\\` and `|`). | Case | Wire shape | Save / load | Exact? | |------|------------|-------------|--------| @@ -108,9 +108,11 @@ E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_ | E18a 4096 (a–e) | same shapes; CJK is 1365 × `测` + 1 X | 0 / 0 all five | **yes** (`\\`/`\|` snap line 8215; quotes/CJK 4119) | | E18b tab mid/end | `CREATE (:USR {id: 'USR_tabm', … value: 'ab\\tcd'})` / `'ab\\t'` | 0 / 0 | **yes** (byte-exact) | | E18b CR mid/end | `… value: 'ab\\rcd'` / `'ab\\r'` | save 0 / load 0 | **yes** (byte-exact; LF-only record split) | -| E18c SCHEMA 64/128 | `SCHEMA WIDE ; fields=id p000 …` (64 / 128 names) | open 1 | `@ERR: limit_exceeded\|fields 64/32` and `128/32` | +| E18c SCHEMA 64 | `SCHEMA WIDE ; fields=id p000 …` (64 names) | open 0 | SCHEMA 64 opens (`max_fields=64`, `>` not `>=`) | +| E18c SCHEMA 128 | `SCHEMA WIDE ; fields=id p000 …` (128 names) | open 1 | `@ERR: limit_exceeded\|fields 128/64` | | E18c SCHEMA 32 | `SCHEMA PRT ; fields=id p00 … p30` | save 0 / load 0 | yes | -| E18c 64/128 extras on USR | CREATE 60 / 124 keys beyond 4-field SCHEMA | save 1 | RAM extras yes; save refuses `snapshot_unsaveable` (`fields\|n/32`). Extras that fit persist by widening snapshot SCHEMA. | +| E18c 64 extras on USR | CREATE 60 keys beyond 4-field SCHEMA | save 0 / load 0 | At cap: mutate accepts; extras persist by widening snapshot SCHEMA | +| E18c 128 extras on USR | CREATE 124 keys beyond 4-field SCHEMA | create 1 | Mutate refuses `@ERR: limit_exceeded\|fields 128/64`; nothing written | | E18c 8 × 4000 ASCII | `CREATE (:FAT {id: 'FAT_8x4000', p000: <4000 A>, … p007: <4000 A>})` | 0 / 0 | **yes** (snap line 32024 < 32768; all eight fields exact) | E17: **yes.** GQL `WHERE n.value CONTAINS '…'` is honoured on SET/DELETE. `MATCH … WHERE … RETURN n` → `@ERR: product_gate|agent surface forbids RETURN …`. Unique MATCH miss → `@ERR: not_found` and nothing SET. SET of `|Q|>1` after WHERE is still `@ERR: cue_conflict|SET Q =…` (unique-SET law, not a dropped WHERE). Inline `MATCH (n WHERE n.value CONTAINS '…')` → `@ERR: parse_error|unsupported MATCH shape`. Bare WHERE without SET/RETURN → `@ERR: parse_error|unsupported MATCH continuation`. `STARTS WITH` / `ENDS WITH` / `=~` are honoured the same way. Keyword substring: `query find --keyword` / `pin_map --keyword` (casefold across all fields, hard `--limit` / `--max-rows`). leftover `read list --where value=*测例*` works on a small graph; on this fulldoc it is `@ERR: response_too_large|response 4659616 bytes exceeds cap 4194304`. Substitute latency (n=200, `find --limit 50`, warm 10000-row session, this VM): common `测例` (1500 USR hits, 50 returned) p50 **67.282** / p95 **84.020** / max **94.672** ms; rare `Part 1500` (1 hit) p50 **51.183** / p95 **69.090** / max **85.424** ms. diff --git a/parts/common/memnet/memnet/config.py b/parts/common/memnet/memnet/config.py index 9359af1c..78a8684a 100644 --- a/parts/common/memnet/memnet/config.py +++ b/parts/common/memnet/memnet/config.py @@ -70,7 +70,7 @@ def __init__(self) -> None: self.max_law = _env_int("MEMNET_MAX_LAW", 100) self.max_relations = _env_int("MEMNET_MAX_RELATIONS", 200) self.max_tags = _env_int("MEMNET_MAX_TAGS", 64) - self.max_fields = _env_int("MEMNET_MAX_FIELDS", 32) + self.max_fields = _env_int("MEMNET_MAX_FIELDS", 64) self.max_value_bytes = _env_int("MEMNET_MAX_VALUE_BYTES", 4096) self.max_line_bytes = _env_int("MEMNET_MAX_LINE_BYTES", 32768) self.max_batch_lines = _env_int("MEMNET_MAX_BATCH_LINES", 1000) diff --git a/parts/common/memnet/memnet/mutate_gate.py b/parts/common/memnet/memnet/mutate_gate.py index 318dc088..da87e0aa 100644 --- a/parts/common/memnet/memnet/mutate_gate.py +++ b/parts/common/memnet/memnet/mutate_gate.py @@ -626,6 +626,7 @@ def _endpoint_changes_for(oid: str) -> list[tuple[str, str]]: orig_nick = source_backup.fields.get("id") or "" if orig_nick != new_id: applied_renames.append((old.hid, new_id, False, source_backup, ep_changes)) + self._refuse_over_field_cap() self.ss.mark_written() except MemNetError: # Undo renames first (commit order: drops → upserts → renames). @@ -683,6 +684,11 @@ def _endpoint_changes_for(oid: str) -> list[tuple[str, str]]: dialect="gql", ) + def _refuse_over_field_cap(self) -> None: + from memnet.snapshot import refuse_mutate_field_cap + + refuse_mutate_field_cap(self.ss) + def _item_to_record(self, it: NodeRec | EdgeRec) -> Record: if isinstance(it, EdgeRec): return self._edge_to_record(it) @@ -991,6 +997,7 @@ def _commit_records( warnings.append( f"mission_settled|{rec.id or rec.tag}|next read use query pin-map from cue" ) + self._refuse_over_field_cap() self.ss.mark_written() except MemNetError: for rid in added: diff --git a/parts/common/memnet/memnet/snapshot.py b/parts/common/memnet/memnet/snapshot.py index cd038faa..04f5cda7 100644 --- a/parts/common/memnet/memnet/snapshot.py +++ b/parts/common/memnet/memnet/snapshot.py @@ -76,8 +76,11 @@ def _append_id_column(tag: str, fields: list[str]) -> list[str]: return [*fields, "id"] -def _snapshot_emit_tag_map(ss: SessionStore) -> TagMap: - """Widen SCHEMA with undeclared RAM keys so extras round-trip (MN-REQ-01.9).""" +def collect_undeclared_extras(ss: SessionStore) -> dict[str, list[str]]: + """Undeclared RAM keys per canonical tag, first-seen order (MN-REQ-01.9). + + Fixed-tag extras fail closed (``snapshot_unsaveable``), same as save. + """ extras: dict[str, list[str]] = {} for rid in ss.store.write_order: rec = ss.store._by_hid.get(rid) @@ -98,26 +101,70 @@ def _snapshot_emit_tag_map(ss: SessionStore) -> TagMap: ) extra.append(key) known.add(key) - tags: dict[str, TagDef] = {} - max_fields = ss.caps.max_fields + return extras + + +def effective_snapshot_fields(ss: SessionStore) -> dict[str, list[str]]: + """Field lists snapshot save would emit (MN-REQ-05.4). + + Declared SCHEMA fields, plus undeclared extras on any row of that tag, + plus ``id`` when save appends it. Mutate and save MUST share this helper. + """ + extras = collect_undeclared_extras(ss) + fields_by_tag: dict[str, list[str]] = {} for tag, td in ss.tag_map.tags.items(): - fields = _append_id_column(tag, list(td.fields) + extras.get(tag, [])) - if len(fields) > max_fields: - raise MemNetError( - "snapshot_unsaveable", - f"{tag} nick=- fields|{len(fields)}/{max_fields}", - ) - tags[tag] = TagDef(tag=td.tag, fields=fields, kind=td.kind) + fields_by_tag[tag] = _append_id_column(tag, list(td.fields) + extras.get(tag, [])) for tag, extra in extras.items(): - if tag in tags or not extra: + if tag in fields_by_tag or not extra: + continue + fields_by_tag[tag] = _append_id_column(tag, ["id"] + extra) + return fields_by_tag + + +def first_over_field_cap( + ss: SessionStore, + *, + skip_fixed: bool = False, +) -> tuple[str, int, int] | None: + """First tag whose effective field count exceeds ``max_fields``, else None. + + Mutate skips fixed tags (EDG/LAW): their declared width is engine-defined + and is not a write the caller made. Save still checks every tag. + """ + max_fields = ss.caps.max_fields + for tag, fields in effective_snapshot_fields(ss).items(): + if skip_fixed and tag in FIXED_TAGS: continue - fields = _append_id_column(tag, ["id"] + extra) + if len(fields) > max_fields: + return tag, len(fields), max_fields + return None + + +def refuse_mutate_field_cap(ss: SessionStore) -> None: + """Mutate: refuse when a user tag's effective fields could not save.""" + over = first_over_field_cap(ss, skip_fixed=True) + if over is None: + return + _tag, n, cap = over + raise MemNetError("limit_exceeded", f"fields|{n}/{cap}") + + +def _snapshot_emit_tag_map(ss: SessionStore) -> TagMap: + """Widen SCHEMA with undeclared RAM keys so extras round-trip (MN-REQ-01.9).""" + field_lists = effective_snapshot_fields(ss) + max_fields = ss.caps.max_fields + tags: dict[str, TagDef] = {} + for tag, fields in field_lists.items(): if len(fields) > max_fields: raise MemNetError( "snapshot_unsaveable", f"{tag} nick=- fields|{len(fields)}/{max_fields}", ) - tags[tag] = TagDef(tag=tag, fields=fields, kind="node") + td = ss.tag_map.get(tag) + if td is not None: + tags[tag] = TagDef(tag=td.tag, fields=fields, kind=td.kind) + else: + tags[tag] = TagDef(tag=tag, fields=fields, kind="node") return TagMap(tags=tags) diff --git a/scripts/probe_doc_gate_readiness.py b/scripts/probe_doc_gate_readiness.py index 3ca19bfd..a9183528 100644 --- a/scripts/probe_doc_gate_readiness.py +++ b/scripts/probe_doc_gate_readiness.py @@ -2025,12 +2025,17 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: tab_ok = bool(b_rows["tab_mid"].get("exact") and b_rows["tab_end"].get("exact")) cr_ok = bool(b_rows["cr_mid"].get("exact") and b_rows["cr_end"].get("exact")) nl_ok = bool(b_rows.get("nl_mid", {}).get("exact")) - schema_64_refused = schema_open["64"]["open_exit"] != 0 + schema_64_ok = schema_open["64"]["open_exit"] == 0 schema_128_refused = schema_open["128"]["open_exit"] != 0 - extras_ram = bool(width64.get("ram_has_extras") and width128.get("ram_has_extras")) - extras_save_refused = width64.get("save_exit") not in (0, None) and width128.get( - "save_exit" - ) not in (0, None) + extras_64_ok = ( + width64.get("create_exit") == 0 + and width64.get("ram_has_extras") + and width64.get("save_exit") == 0 + and width64.get("loaded_has_extras") + ) + extras_128_refused = width128.get("create_exit") not in (0, None) + extras_128_wire = "\n".join(width128.get("create_err") or []) + extras_128_loud = "limit_exceeded" in extras_128_wire and "fields" in extras_128_wire fat_ok = bool(fat_row.get("exact")) decoded = bool(caps.get("value_bytes_on_decoded_field") and caps.get("value_bytes_gt_not_ge")) @@ -2038,14 +2043,16 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: gaps.append("code citation did not show decoded value_bytes with `>`") if not a_exact_4000: gaps.append("one or more 4000-byte (a–e) cases failed round-trip") - if not schema_64_refused or not schema_128_refused: - gaps.append("SCHEMA 64/128 did not refuse max_fields") + if not schema_64_ok or not schema_128_refused: + gaps.append("SCHEMA 64 did not open or SCHEMA 128 did not refuse max_fields=64") if not tab_ok: gaps.append("tab did not round-trip") if not cr_ok: gaps.append("CR did not round-trip") - if not extras_save_refused: - gaps.append("64/128 extras on 4-field USR did not refuse at save (max_fields)") + if not extras_64_ok: + gaps.append("64 extras on 4-field USR did not save/load at max_fields=64") + if not extras_128_refused or not extras_128_loud: + gaps.append("128 extras on 4-field USR did not refuse at mutate (max_fields)") predicted = ( decoded @@ -2053,9 +2060,11 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: and a_exact_4096 and tab_ok and cr_ok - and schema_64_refused + and schema_64_ok and schema_128_refused - and extras_save_refused + and extras_64_ok + and extras_128_refused + and extras_128_loud ) if predicted: verdict = "yes" @@ -2069,8 +2078,8 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: "(check_value_bytes uses `>` not `>=`). join_payload escapes splitlines " "separators plus `\\` and `|`; that expansion is not the value cap. " "line_bytes 32768 is the escaped/raw snapshot line. SCHEMA register vs " - "max_fields=32. Snapshot save widens SCHEMA for undeclared RAM keys; " - "extras over max_fields refuse snapshot_unsaveable.", + "max_fields=64. Mutate and snapshot save share one effective-field count; " + "extras over max_fields refuse limit_exceeded|fields n/max at mutate.", "E18e CJK 4000: " + e18_cjk_composition(4000, han=E18_HAN) + ".", "E18e CJK 4096: " + e18_cjk_composition(4096, han=E18_HAN) + ".", "CREATE wires use gql_str (\\\\ \\' \\\" \\n \\r \\t). Proof numbers omit blobs.", @@ -2095,9 +2104,10 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: "tab_roundtrip": tab_ok, "cr_roundtrip": cr_ok, "nl_roundtrip": nl_ok, - "schema_64_128_refused": schema_64_refused and schema_128_refused, - "extras_in_ram": extras_ram, - "extras_save_refused_over_max_fields": extras_save_refused, + "schema_64_open": schema_64_ok, + "schema_128_refused": schema_128_refused, + "extras_64_saved": extras_64_ok, + "extras_128_mutate_refused": extras_128_refused and extras_128_loud, "fat_8x4000_exact": fat_ok, }, wires=[w for w in wires if w], diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index 03dc4143..b175ba58 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -174,12 +174,17 @@ package MemNet { attribute maxValueBytesDefault : Integer = 4096; attribute valueBytesOnDecodedRaw : Boolean = true; attribute gqlMutateEnforcesValueBytes : Boolean = true; + attribute envMaxFields : String = "MEMNET_MAX_FIELDS"; + attribute maxFieldsDefault : Integer = 64; + attribute gqlMutateEnforcesMaxFields : Boolean = true; + attribute saveUsesSameFieldCountAsMutate : Boolean = true; attribute lineBytesOnEmittedLine : Boolean = true; attribute sessionSaveLoadCloseAclWho : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_1_StoreResourceCaps; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_2_QueryFanoutCaps; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; + satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_4_ConsistentFieldCap; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; satisfy MN_REQ_00_MissionBridge::MN_REQ_10_LlmPropertiesAndLimits::MN_REQ_10_2_PinMapMustFitContext; satisfy MN_REQ_00_MissionBridge::MN_REQ_12_MultitaskMode::MN_REQ_12_7_NoAssumeAclReserveIngest; @@ -441,12 +446,14 @@ package MemNet { attribute whereTrueEdgeDeleteKept : Boolean = true; attribute matchMapEqualityKept : Boolean = true; attribute gqlValueBytesHardCap : Boolean = true; + attribute gqlFieldsHardCap : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_1_AddFailsIfExists; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_2_UpdateFailsIfAbsent; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_3_NoSilentUpsert; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_4_HonourWherePredicate; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; + satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_4_ConsistentFieldCap; satisfy MN_REQ_00_MissionBridge::MN_REQ_02_MemoryNetGraph::MN_REQ_02_7_SchemaValidatedIngest; satisfy MN_REQ_00_MissionBridge::MN_REQ_02_MemoryNetGraph::MN_REQ_02_8_IdNotRequiredOnWire; satisfy MN_REQ_00_MissionBridge::MN_REQ_12_MultitaskMode::MN_REQ_12_7_NoAssumeAclReserveIngest; @@ -1158,6 +1165,8 @@ package MemNet { attribute persistUndeclaredProperties : Boolean = true; attribute idColumnWidenedWhenAbsent : Boolean = true; attribute lineBytesOnEmittedLine : Boolean = true; + attribute saveNeverStricterThanMutate : Boolean = true; + attribute sharedEffectiveFieldCount : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; @@ -1165,6 +1174,7 @@ package MemNet { satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_3_SessionTtlAndCaps; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; + satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_4_ConsistentFieldCap; satisfy MN_REQ_00_MissionBridge::MN_REQ_11_SnapshotInterop::MN_REQ_11_4_DistinctFromSessionSnapshot; part snapshotAcl : SnapshotAcl; diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index 98580e7c..4edb6a25 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -33,6 +33,7 @@ MN-REQ-01.10 CapsPolicy who-check on session save / load-into-ACL / close MN-REQ-03.4 MATCH WHERE SET/DELETE SHALL honour the predicate or refuse MN-REQ-05.3 One decoded value_bytes cap on pipe mutate, GQL mutate, snapshot load + MN-REQ-05.4 One effective-field cap (default 64) on SCHEMA register, mutate, snapshot save/load; save never stricter than write MN-REQ-06.11 Admin-only serve usage report (counts/caps; opaque session alias; not agent MCP) MN-REQ-06.12 Product gateway on memnet-mcp: per-product route to owning serve (not mn_tip_) MN-REQ-06.13 Serve request isolation: each command's stdout, stderr, and exit_code @@ -245,8 +246,18 @@ package MemNetRequirements { id column and a minted nickname. Live session SCHEMA is unchanged until load. Fixed tags EDG/LAW SHALL NOT widen; extras there SHALL fail - closed. Save SHALL fail closed (snapshot_unsaveable - naming the row) rather than write an unloadable file, + closed. A CREATE, SET, or MERGE that would make a tag's + effective field set (declared SCHEMA fields, undeclared + extras on any row of that tag, and the id column save + appends) exceed max_fields SHALL refuse + limit_exceeded|fields n/max and SHALL NOT write + (MN-REQ-05.4). Mutate and snapshot save SHALL share one + effective-field count so save is never stricter than a + mutate that succeeded. Default max_fields is 64 + (MEMNET_MAX_FIELDS). Snapshot load SHALL refuse a SCHEMA + over that cap with the same fields n/max code. Save SHALL + fail closed (snapshot_unsaveable naming the row) as a + backstop rather than write an unloadable file, including when extras would exceed max_fields or the escaped line exceeds line_bytes. Expire-save that cannot write (snapshot_unsaveable or any other save failure, @@ -731,6 +742,26 @@ package MemNetRequirements { */ attribute requirementId : String = "MN-REQ-05.3"; } + + requirement def MN_REQ_05_4_ConsistentFieldCap { + doc /* + The fields-per-tag cap SHALL be one hard cap on SCHEMA + register, GQL mutate, leftover pipe mutate, and snapshot + save/load. The effective count is declared SCHEMA fields + plus undeclared extras stored on any row of that tag, + plus the id column snapshot save appends when the live + SCHEMA omitted it. Default 64. Knob MEMNET_MAX_FIELDS. + Over-cap mutate SHALL refuse limit_exceeded|fields n/max + and SHALL NOT accept the write. Save SHALL use the same + count (one shared helper) so anything mutate accepts can + save and reload. Snapshot load SHALL refuse a SCHEMA over + the cap with that same fields n/max code. + snapshot_unsaveable remains the fail-closed save backstop + when RAM still exceeds the cap (MN-REQ-01.9). Verify: + MN-VER-05-S02 (tests/test_field_cap.py). + */ + attribute requirementId : String = "MN-REQ-05.4"; + } } // ----- Process / transport boundary ----- @@ -2278,6 +2309,8 @@ package MemNetRequirements { : MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_2_QueryFanoutCaps; requirement consistentValueByteCapReq : MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; + requirement consistentFieldCapReq + : MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_4_ConsistentFieldCap; requirement processBoundaryReq : MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary; @@ -2535,6 +2568,7 @@ package MemNetRequirements { end #derive ::> storeResourceCapsReq; end #derive ::> queryFanoutCapsReq; end #derive ::> consistentValueByteCapReq; + end #derive ::> consistentFieldCapReq; } #derivation connection deriveProcessBoundaryLeaves { diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index 37189041..4dcc39f5 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -77,10 +77,12 @@ Expire session_save (MN-VER-01-S03): configurable; default off; RAM drops; file until user drop; load restores; not Neo4j. Snapshot lossless round-trip (MN-VER-01-S04): emit escapes LF/CR; - fail-closed save; 0.19.18 snapshots still load. + fail-closed save; 0.19.18 snapshots still load; mutate field cap. Session save/load/close ACL who (MN-VER-01-S05) when session ACL on. Honour WHERE or refuse (MN-VER-03-S01). Consistent decoded value_bytes cap (MN-VER-05-S01). + Consistent effective-field cap (MN-VER-05-S02): default 64; + mutate refuses over-cap; save uses the same helper. */ package MemNetVerification { private import ScalarValues::*; @@ -2128,13 +2130,16 @@ package MemNetVerification { verification def MN_VER_01_S04_SnapshotLosslessRoundTrip { doc /* - MN-REQ-01.9 / MN-REQ-05.3 — snapshot emit escapes splitlines - separators (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus pipe/backslash - losslessly; undeclared RAM properties persist by snapshot SCHEMA - widen; line_bytes is the escaped emitted line at save and load; - save fails closed; expire-save writes no unloadable file and - keeps RAM on any save failure until explicit save or close; - 0.19.18 snapshots still load; value_bytes is decoded raw UTF-8. + MN-REQ-01.9 / MN-REQ-05.3 / MN-REQ-05.4 — snapshot emit escapes + splitlines separators (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus + pipe/backslash losslessly; undeclared RAM properties persist by + snapshot SCHEMA widen; line_bytes is the escaped emitted line at + save and load; mutate refuses a tag whose effective fields + exceed max_fields (limit_exceeded|fields n/max) so save is never + stricter than write; save fails closed; expire-save writes no + unloadable file and keeps RAM on any save failure until explicit + save or close; 0.19.18 snapshots still load; value_bytes is + decoded raw UTF-8. */ attribute verificationId : String = "MN-VER-01-S04"; @@ -2143,6 +2148,7 @@ package MemNetVerification { objective snapshotLosslessRoundTrip { verify snapshotLosslessRoundTripReq; verify consistentValueByteCapReq; + verify consistentFieldCapReq; require constraint { system.core.transport.inProcess.memory.sessions.snap .losslessPropertyRoundTrip == true @@ -2188,6 +2194,14 @@ package MemNetVerification { .valueBytesOnDecodedRaw == true and system.core.transport.inProcess.memory.sessions.caps .gqlMutateEnforcesValueBytes == true + and system.core.transport.inProcess.memory.sessions.caps + .gqlMutateEnforcesMaxFields == true + and system.core.transport.inProcess.memory.sessions.caps + .saveUsesSameFieldCountAsMutate == true + and system.core.transport.inProcess.memory.sessions.snap + .saveNeverStricterThanMutate == true + and system.core.transport.inProcess.memory.sessions.recallCommit + .commit.mutate.gqlFieldsHardCap == true } } } @@ -2266,6 +2280,41 @@ package MemNetVerification { } } + verification def MN_VER_05_S02_ConsistentFieldCap { + doc /* + MN-REQ-05.4 / MN-REQ-01.9 — one effective-field cap; default 64; + knob MEMNET_MAX_FIELDS; mutate refuses over-cap with + limit_exceeded|fields n/max and writes nothing; save uses the + same helper (save never stricter than write); snapshot load of + a SCHEMA over the cap refuses loudly; snapshot_unsaveable stays + the save backstop. Tests: tests/test_field_cap.py. + */ + attribute verificationId : String = "MN-VER-05-S02"; + + subject system : MemNetSystem; + + objective consistentFieldCap { + verify consistentFieldCapReq; + verify snapshotLosslessRoundTripReq; + require constraint { + system.core.transport.inProcess.memory.sessions.caps + .envMaxFields == "MEMNET_MAX_FIELDS" + and system.core.transport.inProcess.memory.sessions.caps + .maxFieldsDefault == 64 + and system.core.transport.inProcess.memory.sessions.caps + .gqlMutateEnforcesMaxFields == true + and system.core.transport.inProcess.memory.sessions.caps + .saveUsesSameFieldCountAsMutate == true + and system.core.transport.inProcess.memory.sessions.snap + .saveNeverStricterThanMutate == true + and system.core.transport.inProcess.memory.sessions.snap + .sharedEffectiveFieldCount == true + and system.core.transport.inProcess.memory.sessions.recallCommit + .commit.mutate.gqlFieldsHardCap == true + } + } + } + verification def MN_VER_06_S06_StorageRoles { doc /* storageRole only where bytes outlive the call: working_memory, @@ -2363,6 +2412,8 @@ package MemNetVerification { : MN_VER_03_S01_HonourWherePredicate; verification consistentValueByteCapVerify : MN_VER_05_S01_ConsistentValueByteCap; + verification consistentFieldCapVerify + : MN_VER_05_S02_ConsistentFieldCap; verification storageRolesVerify : MN_VER_06_S06_StorageRoles; verification lanMcpFrontSeveralServesVerify : MN_VER_06_S07_LanMcpFrontSeveralServes; diff --git a/tests/cap_contract_lib.py b/tests/cap_contract_lib.py index 4d62137d..798b456d 100644 --- a/tests/cap_contract_lib.py +++ b/tests/cap_contract_lib.py @@ -449,7 +449,7 @@ def case_tag_and_field_caps() -> list[Case]: Case( name="fields_per_tag", kind="hard_refuse", - default="32", + default="64", knob="MEMNET_MAX_FIELDS", library_code=exc.code, library_message=exc.message, diff --git a/tests/doc_gate_lib.py b/tests/doc_gate_lib.py index ff372bb5..cb677fe3 100644 --- a/tests/doc_gate_lib.py +++ b/tests/doc_gate_lib.py @@ -527,7 +527,7 @@ def snapshot_value_cap_report() -> dict[str, Any]: return { "value_bytes_default": 4096, "line_bytes_default": 32768, - "max_fields_default": 32, + "max_fields_default": 64, "value_bytes_on_decoded_field": "value_utf8_len" in tag and "max_value_bytes" in tag, "value_bytes_gt_not_ge": "n > caps.max_value_bytes" in tag, "decoded_after_split_payload": ( @@ -549,7 +549,7 @@ def snapshot_value_cap_report() -> dict[str, Any]: "save_atomic_write_no_value_check": "atomic_write_bytes(" in snap, "config_value": '_env_int("MEMNET_MAX_VALUE_BYTES", 4096)' in cfg, "config_line": '_env_int("MEMNET_MAX_LINE_BYTES", 32768)' in cfg, - "config_fields": '_env_int("MEMNET_MAX_FIELDS", 32)' in cfg, + "config_fields": '_env_int("MEMNET_MAX_FIELDS", 64)' in cfg, "code_path": ( "session_save -> snapshot_text -> emit_record -> join_payload " "(escapes splitlines separators plus \\\\ and |). session_load -> " @@ -557,7 +557,7 @@ def snapshot_value_cap_report() -> dict[str, Any]: "vs max_line_bytes, then split_payload unescape, then " "validate_values decoded utf-8 vs max_value_bytes (`>` not `>=`). " "Undeclared RAM keys widen snapshot SCHEMA. SCHEMA register vs " - "max_fields." + "max_fields. Mutate uses the same effective-field count as save." ), } @@ -790,7 +790,8 @@ def e18_largest_roundtrip( def e18_instance_width(svc: ServeProc, tmp: Path, n_props: int) -> dict[str, Any]: """n_props keys on 4-field USR SCHEMA. - Extras stay in RAM; save widens SCHEMA or refuses max_fields. + Extras at or under max_fields persist; over-cap mutate refuses + limit_exceeded|fields n/max and writes nothing. """ nid = f"USR_w{n_props}" extra_n = max(0, n_props - 4) @@ -806,6 +807,8 @@ def e18_instance_width(svc: ServeProc, tmp: Path, n_props: int) -> dict[str, Any public = e18_public_rt(row) public["n_requested"] = n_props public["extra_n"] = extra_n + public["create_exit"] = row.get("create_exit") + public["create_err"] = row.get("create_err") public["ram_has_extras"] = row.get("ram_has_extras") public["loaded_has_extras"] = row.get("loaded_has_extras", False) public["save_exit"] = row.get("save_exit") diff --git a/tests/test_cap_contract.py b/tests/test_cap_contract.py index b415e4f3..b1b6dfbf 100644 --- a/tests/test_cap_contract.py +++ b/tests/test_cap_contract.py @@ -23,7 +23,7 @@ def test_defaults_match_library(): assert d["max_relations"] == 200 assert d["max_law"] == 100 assert d["max_tags"] == 64 - assert d["max_fields"] == 32 + assert d["max_fields"] == 64 assert d["max_value_bytes"] == 4096 assert d["max_line_bytes"] == 32768 assert d["max_batch_lines"] == 1000 diff --git a/tests/test_doc_gate_readiness.py b/tests/test_doc_gate_readiness.py index c788f905..a665f0ea 100644 --- a/tests/test_doc_gate_readiness.py +++ b/tests/test_doc_gate_readiness.py @@ -758,7 +758,7 @@ def test_snapshot_value_cap_is_decoded_field(): report = snapshot_value_cap_report() assert report["value_bytes_default"] == 4096 assert report["line_bytes_default"] == 32768 - assert report["max_fields_default"] == 32 + assert report["max_fields_default"] == 64 assert report["value_bytes_on_decoded_field"] is True assert report["value_bytes_gt_not_ge"] is True assert report["decoded_after_split_payload"] is True @@ -784,17 +784,20 @@ def test_e18_cjk_blob_composition(): assert "U+6D4B" in e18_cjk_composition(4000) -def test_e18_schema_64_and_128_refuse_fields(doc_serve: ServeProc): +def test_e18_schema_64_opens_and_128_refuses_fields(doc_serve: ServeProc): sid64, r64 = doc_serve.try_open_session(map_lines=e18_schema_fields(64)) - assert sid64 is None - joined64 = "\n".join(err_lines(r64.stderr)) - assert "limit_exceeded" in joined64 - assert "fields" in joined64 - assert "64/32" in joined64 + assert sid64 is not None, redact(r64.stderr) + doc_serve.close(sid64) + sid65, r65 = doc_serve.try_open_session(map_lines=e18_schema_fields(65)) + assert sid65 is None + joined65 = "\n".join(err_lines(r65.stderr)) + assert "limit_exceeded" in joined65 + assert "fields" in joined65 + assert "65/64" in joined65 sid128, r128 = doc_serve.try_open_session(map_lines=e18_schema_fields(128)) assert sid128 is None - assert "128/32" in "\n".join(err_lines(r128.stderr)) - assert_sid_free(redact(r64.stderr), redact(r128.stderr)) + assert "128/64" in "\n".join(err_lines(r128.stderr)) + assert_sid_free(redact(r64.stderr), redact(r65.stderr), redact(r128.stderr)) def test_e18_tab_survives_cr_breaks(doc_serve: ServeProc, tmp_path: Path): @@ -845,11 +848,18 @@ def test_e18_8x4000_and_ram_extras(doc_serve: ServeProc, tmp_path: Path): assert fat["snap_line_utf8"] is not None assert fat["snap_line_utf8"] < 32768 wide = e18_instance_width(doc_serve, tmp_path, 64) + assert wide["create_exit"] == 0, wide assert wide["ram_has_extras"] is True - assert wide["save_exit"] != 0 - joined = "\n".join(wide.get("save_err") or []) - assert "snapshot_unsaveable" in joined or "fields|" in joined - assert wide["loaded_has_extras"] is False + assert wide["save_exit"] == 0, wide + assert wide["load_exit"] == 0, wide + assert wide["loaded_has_extras"] is True + over = e18_instance_width(doc_serve, tmp_path, 128) + assert over["create_exit"] != 0 + joined = "\n".join(over.get("create_err") or []) + assert "limit_exceeded" in joined + assert "fields" in joined + assert over["ram_has_extras"] is not True + assert over["loaded_has_extras"] is False fit = e18_instance_width(doc_serve, tmp_path, 8) assert fit["ram_has_extras"] is True assert fit["save_exit"] == 0, fit diff --git a/tests/test_engine_roundtrip_where_acl.py b/tests/test_engine_roundtrip_where_acl.py index 7a329c95..9b26da66 100644 --- a/tests/test_engine_roundtrip_where_acl.py +++ b/tests/test_engine_roundtrip_where_acl.py @@ -166,9 +166,13 @@ def test_snapshot_refuses_extras_over_max_fields(memnet_temp, tmp_path: Path, mo caps=Caps(), ) MutateGate(ss).apply( - ["CREATE (:CST {id: 'N_F', name: 'a', role: 'b', extra1: 'x', extra2: 'y'})"], + ["CREATE (:CST {id: 'N_F', name: 'a', role: 'b'})"], mode="add", ) + rec = ss.store.get("N_F") + assert rec is not None + rec.fields["extra1"] = "x" + rec.fields["extra2"] = "y" path = tmp_path / "fields.snap" with pytest.raises(MemNetError) as ei: write_snapshot(ss, path) diff --git a/tests/test_expire_save_keep.py b/tests/test_expire_save_keep.py index 33fc7798..cacd93b8 100644 --- a/tests/test_expire_save_keep.py +++ b/tests/test_expire_save_keep.py @@ -171,8 +171,10 @@ def test_save_on_expire_off_still_drops(memnet_temp, schema_file, monkeypatch): set_now_override(None) -def _extras_create(n: int = 40) -> str: - extras = ", ".join(f"x{i:02d}: 'v'" for i in range(n)) +def _extras_create() -> str: + """Mutate-legal row that save refuses (escaped line over line_bytes).""" + blob = "|" * 4000 + extras = ", ".join(f"x{i:02d}: '{blob}'" for i in range(12)) return f"CREATE (:N {{id: 'N01', name: 'n', {extras}}})" diff --git a/tests/test_field_cap.py b/tests/test_field_cap.py new file mode 100644 index 00000000..a5dc8980 --- /dev/null +++ b/tests/test_field_cap.py @@ -0,0 +1,158 @@ +"""MN-REQ-05.4 / MN-VER-05-S02 — effective field cap (default 64).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from memnet.config import Caps +from memnet.exceptions import MemNetError +from memnet.mutate_gate import MutateGate +from memnet.output import format_err +from memnet.session import open_session +from memnet.snapshot import ( + _snapshot_emit_tag_map, + effective_snapshot_fields, + first_over_field_cap, + load_snapshot, + write_snapshot, +) + +_PRT_MAP = ["SCHEMA PRT ; fields=id name qname path sysml_kind session grain recycle"] +_CST_MAP = ["SCHEMA CST ; fields=id name role"] +_PRT_DECLARED = 8 +_CST_DECLARED = 3 + + +def _extras(n: int, *, start: int = 0) -> dict[str, str]: + return {f"x{i:03d}": f"v{i:03d}" for i in range(start, start + n)} + + +def _gql_props(fields: dict[str, str]) -> str: + return ", ".join(f"{k}: '{v}'" for k, v in fields.items()) + + +def _create(kind: str, fields: dict[str, str]) -> str: + return f"CREATE (:{kind} {{{_gql_props(fields)}}})" + + +def test_prt_38_fields_saves_and_reloads(memnet_temp, tmp_path: Path): + ss = open_session(map_lines=list(_PRT_MAP)) + extras = _extras(38 - _PRT_DECLARED) + fields = { + "id": "PRT_38", + "name": "wide", + "qname": "N", + "path": "p", + "sysml_kind": "part", + "session": "", + "grain": "", + "recycle": "", + **extras, + } + MutateGate(ss).apply([_create("PRT", fields)], mode="add") + rec = ss.store.get("PRT_38") + assert rec is not None + for key, val in extras.items(): + assert rec.fields.get(key) == val + eff = effective_snapshot_fields(ss)["PRT"] + assert len(eff) == 38 + path = tmp_path / "prt38.snap" + write_snapshot(ss, path) + loaded = load_snapshot(path) + got = loaded.store.get("PRT_38") + assert got is not None + for key, val in extras.items(): + assert got.fields.get(key) == val + assert got.fields.get("name") == "wide" + assert "x000" in loaded.tag_map.get("PRT").fields + assert len(loaded.tag_map.get("PRT").fields) == 38 + + +def test_mutate_refuses_over_cap_writes_nothing(memnet_temp): + cap = Caps().max_fields + over_n = cap - _CST_DECLARED + 1 + ss = open_session(map_lines=list(_CST_MAP)) + over = {"id": "N_OVER", "name": "a", "role": "b", **_extras(over_n)} + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply([_create("CST", over)], mode="add") + assert ei.value.code == "limit_exceeded" + assert f"fields|{_CST_DECLARED + over_n}/{cap}" in ei.value.message + assert format_err(ei.value.code, ei.value.message) == ( + f"@ERR: limit_exceeded|fields {_CST_DECLARED + over_n}/{cap}" + ) + assert ss.store.get("N_OVER") is None + merge_fields = {"id": "N_MG", "name": "a", "role": "b", **_extras(over_n)} + with pytest.raises(MemNetError) as ei2: + MutateGate(ss).apply( + [f"MERGE (:CST {{{_gql_props(merge_fields)}}})"], + mode="mutate", + ) + assert ei2.value.code == "limit_exceeded" + assert "fields|" in ei2.value.message + assert ss.store.get("N_MG") is None + assert ss.store.get("N_OVER") is None + + +def test_mutate_set_past_cap_writes_nothing(memnet_temp): + cap = Caps().max_fields + ss = open_session(map_lines=list(_CST_MAP)) + at_cap = {"id": "N_SET", "name": "a", "role": "b", **_extras(cap - _CST_DECLARED)} + MutateGate(ss).apply([_create("CST", at_cap)], mode="add") + rec = ss.store.get("N_SET") + assert rec is not None + assert rec.fields.get("x000") == "v000" + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply( + ["MATCH (n:CST {id: 'N_SET'}) SET n.x061 = 'nope'"], + mode="mutate", + ) + assert ei.value.code == "limit_exceeded" + assert f"fields|{cap + 1}/{cap}" in ei.value.message + rec = ss.store.get("N_SET") + assert rec is not None + assert rec.fields.get("name") == "a" + assert rec.fields.get("x000") == "v000" + assert rec.fields.get("x061") is None + + +def test_save_cap_not_stricter_than_mutate(memnet_temp, tmp_path: Path): + ss = open_session(map_lines=list(_CST_MAP)) + cap = ss.caps.max_fields + assert cap == Caps().max_fields == 64 + fields = {"id": "N_MAX", "name": "a", "role": "b", **_extras(cap - _CST_DECLARED)} + MutateGate(ss).apply([_create("CST", fields)], mode="add") + rec = ss.store.get("N_MAX") + assert rec is not None + eff = effective_snapshot_fields(ss) + assert len(eff["CST"]) == cap + assert first_over_field_cap(ss) is None + emit = _snapshot_emit_tag_map(ss) + assert list(emit.tags["CST"].fields) == list(eff["CST"]) + path = tmp_path / "at_cap.snap" + write_snapshot(ss, path) + loaded = load_snapshot(path) + got = loaded.store.get("N_MAX") + assert got is not None + assert got.fields.get("x000") == "v000" + last = f"x{cap - _CST_DECLARED - 1:03d}" + assert got.fields.get(last) == f"v{cap - _CST_DECLARED - 1:03d}" + assert len(loaded.tag_map.get("CST").fields) == cap + + +def test_load_snapshot_over_64_fails_loudly(memnet_temp, tmp_path: Path, monkeypatch): + monkeypatch.setenv("MEMNET_MAX_FIELDS", "80") + names = ["id"] + [f"p{i:03d}" for i in range(64)] + assert len(names) == 65 + ss = open_session( + map_lines=[f"SCHEMA WIDE ; fields={' '.join(names)}"], + caps=Caps(), + ) + path = tmp_path / "over.snap" + write_snapshot(ss, path) + monkeypatch.setenv("MEMNET_MAX_FIELDS", "64") + with pytest.raises(MemNetError) as ei: + load_snapshot(path, caps=Caps()) + assert ei.value.code == "limit_exceeded" + assert "fields|65/64" in ei.value.message diff --git a/tests/test_sysml_engine_bugs.py b/tests/test_sysml_engine_bugs.py index 8a55530f..3a7866fc 100644 --- a/tests/test_sysml_engine_bugs.py +++ b/tests/test_sysml_engine_bugs.py @@ -13,13 +13,20 @@ def test_requirements_ids_present(): text = REQ.read_text(encoding="utf-8") - for nid in ("MN-REQ-01.9", "MN-REQ-01.10", "MN-REQ-03.4", "MN-REQ-05.3"): + for nid in ( + "MN-REQ-01.9", + "MN-REQ-01.10", + "MN-REQ-03.4", + "MN-REQ-05.3", + "MN-REQ-05.4", + ): assert nid in text assert "snapshot_unsaveable" in text assert "expire_snapshot_failed" in text assert "SHALL NOT drop RAM" in text assert "unsupported_predicate" in text assert "MEMNET_MAX_VALUE_BYTES" in text + assert "MEMNET_MAX_FIELDS" in text assert "str.splitlines()" in text assert "widening the" in text assert "escaped/raw leftover-pipe" in text @@ -27,6 +34,7 @@ def test_requirements_ids_present(): assert "sessionLifecycleAclWhoReq" in text assert "honourWherePredicateReq" in text assert "consistentValueByteCapReq" in text + assert "consistentFieldCapReq" in text def test_deploy_and_verify_trail_model(): @@ -35,6 +43,11 @@ def test_deploy_and_verify_trail_model(): assert "failClosedUnsaveable" in deploy assert "honourWhereOrRefuse" in deploy assert 'envMaxValueBytes : String = "MEMNET_MAX_VALUE_BYTES"' in deploy + assert 'envMaxFields : String = "MEMNET_MAX_FIELDS"' in deploy + assert "maxFieldsDefault : Integer = 64" in deploy + assert "gqlMutateEnforcesMaxFields" in deploy + assert "saveNeverStricterThanMutate" in deploy + assert "gqlFieldsHardCap" in deploy assert "sessionSaveLoadCloseAclWho" in deploy assert "acceptsCaller" in deploy assert "persistUndeclaredProperties" in deploy @@ -47,6 +60,7 @@ def test_deploy_and_verify_trail_model(): assert "MN-VER-01-S05" in ver assert "MN-VER-03-S01" in ver assert "MN-VER-05-S01" in ver + assert "MN-VER-05-S02" in ver assert "persistUndeclaredProperties" in ver assert "splitlinesSeparatorsEscaped" in ver assert "lineBytesOnEmittedLine" in ver