From fc6e303f1d0a66226cd2d55fe385684af5e14030 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 20:34:51 -0700 Subject: [PATCH 01/15] fix(differential,guards): port the #360 mc and #367 de rules to the 1.4 ledger 'Mc Donald' and 'Sir de Mesnil' fell to the fields-only catch-all not because nothing described them but because this ledger never got the rules expected_since_2.0.0.toml and expected_since_2.1.0.toml both carry. Both ported verbatim; the #360 rule alone drops `_ambiguities`, which cannot enter a diff below baseline 2.0. _CROSS_RULE_WINNERS recorded the absorption for three names; two of its rows are now false and go, and its comment says so. The catch-all is down to 12 from 14. Gate green at 1.4.0, 2.0.0 and 2.1.0. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 31 ++++++------ tools/differential/expected_since_1.4.0.toml | 50 +++++++++++++++++++- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index facc1a4b..a93790cd 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1460,6 +1460,10 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('family', 'middle', 'suffix'), "a564b97f7162"), "fix(#360) ste moved into the never-given particles with mc": _Claim(1, ('family', 'given'), "e62caedec864"), + "fix(#360) mc moved into the never-given particles, so it folds into the family": + _Claim(1, ('family', 'given'), "ee4339908f4d"), + "fix(#367) a title no longer displaces a leading never-given particle": + _Claim(1, ('family', 'given'), "db724fb9c779"), "fix(#445) a maiden marker makes the lone name word the family": _Claim(7, ('family', 'given', 'maiden', 'middle'), "3de9ef12b4a8"), }, @@ -1776,21 +1780,20 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: #: a diff shape that shifted is a finding, not a number to update. _CROSS_RULE_WINNERS: dict[str, dict[tuple[str, tuple[str, ...]], str]] = { "expected_since_1.4.0.toml": { - # Three behaviours that got named rules in the 2.0 and 2.1 - # ledgers but not this one, so here they fall to the fields-only - # catch-all. Recorded because the assignment is incidental - # rather than argued: 'Berg, Jan vd' is reachable by three - # named rules and wins none of them (their `fields` do not - # match), while 'Mc Donald' and 'Sir de Mesnil' are reachable - # by NO named rule at all. Being absorbed by the catch-all is - # the recoverable direction, but a rule added later that takes - # one of them silently would change what the 1.4 summary - # attributes -- and AGENTS.md names that summary as the source - # for the release log's Behavior Changes section. #380, #360 - # and #367 respectively. + # One behaviour that got a named rule in the 2.0 and 2.1 ledgers + # but not this one, so here it falls to the fields-only + # catch-all. The other two -- 'Mc Donald' (#360) and + # 'Sir de Mesnil' (#367) -- had their rules ported into this + # ledger by #451, which is why those rows are gone. + # Recorded because the assignment is incidental rather than + # argued: 'Berg, Jan vd' is reachable by three named rules and + # wins none of them (their `fields` do not match). Being + # absorbed by the catch-all is the recoverable direction, but a + # rule added later that takes it silently would change what the + # 1.4 summary attributes -- and AGENTS.md names that summary as + # the source for the release log's Behavior Changes section. + # #380. ("Berg, Jan vd", ("family", "suffix")): "fix(suffix-routing)", - ("Mc Donald", ("family", "given")): "fix(suffix-routing)", - ("Sir de Mesnil", ("family", "given")): "fix(suffix-routing)", ("Andrews, M.D.", ("given", "suffix")): "fix(comma-family)", ("田中, 太郎さん", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", ("김, 민준씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index b94aa73c..b1e0c1d7 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1135,6 +1135,54 @@ issue = "fix(#360) ste moved into the never-given particles with mc" name_regex = "(?i)^ste\\s+\\S+$" fields = ["given", "family"] +[[change]] +issue = "fix(#360) mc moved into the never-given particles, so it folds into the family" +# 'Mc Donald': mc became a never-given particle, so the leading token +# stops being read as the given name and folds into the family instead +# ('Mc' + 'Donald' -> family 'Mc Donald'). The twin of the ste rule +# above, and written beside it for that reason. +# +# In `(?i)^mc\s+\S+$` the `mc` is a LITERAL anchored at the start, +# followed by whitespace and then exactly one token to the string's +# end. `\S+$`, not a `\b`, is what closes it: the pattern carries no +# word boundary at all. 'McDonald, Ronald' and 'Mcintyre Smith Jr.' +# fail on the whitespace, having none after 'mc'; 'Ste Marie' and +# 'Los Santos' fail earlier still, opening with neither letter. None +# of the four is this rule's to claim, whatever else explains it -- +# and if one ever needs explaining, it needs a rule of its own. Do +# not widen this one to reach it: a rule claiming names its prose +# does not describe is what #451 is retiring. +# +# Ported verbatim from expected_since_2.0.0.toml, minus `_ambiguities`: +# below baseline 2.0 there is no v2 surface, so no diff can carry that +# pseudo-field and naming it would narrow by something unreachable. +# Until this port the name fell to the fields-only catch-all, which +# _CROSS_RULE_WINNERS recorded rather than closed (#451). +name_regex = "(?i)^mc\\s+\\S+$" +fields = ["given", "family"] + +[[change]] +issue = "fix(#367) a title no longer displaces a leading never-given particle" +# 'Sir de Mesnil': the title stops pushing the particle out of the +# leading position, so the particle run reads as the family rather +# than the given name. The `de` sibling of the two #367 `van` rules +# below. +# +# `(?i)^sir\s+de\b` is anchored at the start, and its `\b` is real +# work rather than decoration: it is what keeps 'Sir Denis Smith' and +# 'Sir del Rio' out. The roster's own probes fail earlier -- 'John Sir +# de Mesnil' and 'Smith, Sir de Vaux' on the anchor, 'sir' opening +# neither string, and 'Sir Smith' on the literal 'de'. None of the +# three is this rule's to claim, whatever else explains it, and a name +# of that shape that ever needs explaining wants its own rule rather +# than a widening of this one (#451). +# +# Ported verbatim from expected_since_2.0.0.toml, whose copy already +# reads fields = ["given", "family"] and carries nothing to drop -- +# unlike the #360 rule above, this one never claimed `_ambiguities`. +name_regex = "(?i)^sir\\s+de\\b" +fields = ["given", "family"] + [[change]] issue = "fix(#424) an unlisted abbreviation is as transparent as a listed title to the leading particle" # 'Esq. van Gogh' (corpus_issues): rules.md#P4 -- "An unlisted @@ -1438,7 +1486,7 @@ issue = "fix(#410) a title and one name word name the family, whatever annotatio # they were falling to the fields-only fix(suffix-routing) below, on a # rule whose every other name moves a trailing token into `suffix`, # while these move no suffix at all. Same surgery #372 did on the same -# catch-all, and its heading returns to 14 once this rule takes them. +# catch-all, whose heading returned to 14 once this rule took them. # # THREE names here, where the 2.0.0 and 2.1.0 ledgers classify four: # at this baseline 'Dr. Smith née Jones' also moves `middle` and From 7cc282f94a005eda6dc4251dcfb28ec28774512a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 21:05:22 -0700 Subject: [PATCH 02/15] fix(differential,guards): name the A2 emptying instead of a comma rule '(' was the fields-only catch-all's and '.,' was claimed by `fix(comma-family) lone post-comma piece routes to suffix/title, not first`, on nothing but that rule's Latin-range comma regex reaching it. Both names are one behaviour: v1 kept punctuation-only input as a name, 2.0 empties it (decisions.md#A2). The rule is written above that comma rule on purpose -- same tier, so file order decides -- and _CROSS_RULE_WINNERS pins the reassignment, since nothing but that order keeps it. `^\W*$` is A2's own predicate rather than a hand-drawn punctuation class: _assemble.py empties on `not any(c.isalnum() ...)`, and \w and isalnum diverge on exactly one codepoint in all of Unicode (U+005F, which A2 empties and this rule does not claim). The rule is therefore a strict subset of the behaviour it names, by construction. Catch-all 12 -> 11. Gate green at all three baselines. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 9 ++++++ tools/differential/expected_since_1.4.0.toml | 34 ++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index a93790cd..97c7bdae 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1344,6 +1344,8 @@ def _claim(rule: dict) -> _Claim: #: both is growth into names the rule genuinely describes. _CORPUS_CLAIMS: dict[str, dict[str, _Claim]] = { "expected_since_1.4.0.toml": { + "fix(A2) content-free input names nobody, so every role empties": + _Claim(5, ('given',), "1af8d718688b"), "fix(#335) a marker-led clause leaves the one name word its bare reading": _Claim(1, ('maiden', 'nickname'), "c09cc7dba88b"), "fix(#434) a multi-word maiden marker takes the maiden name": @@ -1849,6 +1851,13 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: "fix(cjk-glued-honorific-peel)", ("김민준 씨.", ("family", "given", "suffix")): "fix(cjk-honorific-suffix)", + # '.,' moved off `fix(comma-family) lone post-comma piece + # routes to suffix/title, not first`, whose Latin-range comma + # regex reaches it, onto the A2 rule that describes it (#451). + # Recorded because only file order separates the two rules -- + # they are in the same tier and both reach the name -- and a + # later edit that moves either one silently hands it back. + (".,", ("given",)): "fix(A2) content-free input names nobody, so every role empties", }, } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index b1e0c1d7..62a22ef3 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -220,6 +220,40 @@ issue = "fix(credential-pair-order) a split credential and a suffix render in wr name_regex = "(?i)^[\\u0000-\\u024f]*,\\s*jr\\.?\\s+ph\\.?\\s*d\\.?\\s*$" fields = ["suffix"] +[[change]] +issue = "fix(A2) content-free input names nobody, so every role empties" +# '(' / '.,': v1 kept a punctuation-only string as a name -- parse('.') +# read first='.' -- and 2.0 empties input with no name content in it. +# decisions.md#A2. Five corpus names are content-free ('', '()', ',', +# '(', '.,'); two of them diff at this baseline and the other three +# were already empty on both sides. +# +# Written ABOVE `fix(comma-family) lone post-comma piece routes to +# suffix/title, not first` deliberately -- that rule, well down the +# file, and NOT the comma-family rule sitting immediately below this +# one, which is anchored to a literal name and cannot reach '.,' at +# all. Both carry a name_regex so they share a tier, where +# _sorted_rules' stable sort leaves file order deciding, and that +# rule's Latin-range comma regex reaches '.,' -- which is how a comma +# rule came to explain an emptying that has nothing to do with commas. +# (Its regex was a bare ',' until #372; the Latin anchoring it has +# now reaches '.,' just the same. That rule's own comment owns the +# story of what the anchoring keeps off it, and the count with it -- +# restating either here is how one copy goes stale while the other +# does not.) _CROSS_RULE_WINNERS pins the reassignment. +# +# `\W` rather than a punctuation class, because it IS A2's own +# predicate: _assemble.py decides the emptying with `not +# any(c.isalnum() ...)`, and across all of Unicode `\w` and isalnum +# diverge on exactly one codepoint -- U+005F, which A2 empties and +# this rule does not claim. So the rule is a strict SUBSET of the +# behavior it names, by construction rather than by measurement: it +# cannot claim a name A2 leaves alone. Being Unicode-aware, it also +# puts any name carrying a letter or digit in any script outside +# itself, which is why it matches none of validate_rules' sentinels. +name_regex = "^\\W*$" +fields = ["given"] + [[change]] issue = "fix(comma-family) a comma followed only by titles keeps the given/family split" # 'Bob Jones, author' / 'Bob Jones, compositeur': the comma is followed From 1b55eb12afa23cf1d5197bc20a17ff5ceaf376c5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 21:19:22 -0700 Subject: [PATCH 03/15] fix(differential,guards): name the emoji token division MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'John😀Smith' was the fields-only catch-all's last {family, given} name. 1.4 dropped the emoji and read one token; 2.x divides on it, a reading rules.md#T1 already documents alongside its keep-emoji policy twin. The emoji must stand between two non-space characters, which is what "divides a token" means. A bare character class was the first draft: measured, it reaches four corpus names, and the three it adds are byte-identical on both sides today -- a rule that claims them pre-excuses a regression on shapes nobody has looked at, which is the #372 failure mode the fix(#379) comment names. The class itself is the astral half of _tokenize.py's _EMOJI_RANGES, not a single span to U+1FAFF. That first draft was the same #372 shape one level down: it covered 1072 codepoints the tokenizer does NOT divide on, so the rule's own prose was false for every one of them and a {given, family} diff with another cause would have classified here. The hand copy is pinned by a new guard, which fails on the draft it replaces. The BMP half is deliberately not copied -- no corpus name reaches it through the token anchor, and it keeps the pattern free of \uXXXX-\uXXXX span text. Catch-all 11 -> 10, and every name left on it moves {family, suffix}. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 53 ++++++++++++++++++++ tools/differential/expected_since_1.4.0.toml | 36 +++++++++++++ 2 files changed, 89 insertions(+) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 97c7bdae..2f870b21 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -530,6 +530,57 @@ def test_nickname_delimiter_sets_are_deliberate() -> None: "passing vacuously") +def test_the_emoji_boundary_rule_copies_the_dividing_ranges() -> None: + """The emoji rule's character class is a HAND COPY of the ranges + the tokenizer actually divides on, and this is what keeps the two + in step. + + The rule promises "an emoji inside a token divides it". Only + _EMOJI_RANGES decides that, so a class wider than those ranges + makes the promise false for every codepoint in the gap -- and the + rule would then claim a {given, family} diff whose cause is + something else entirely. The first draft had exactly that bug: one + \\U0001F300-\\U0001FAFF span, covering U+1F650-U+1F67F and + U+1F700-U+1FAFF, where the parser leaves the token whole. + + Only the ASTRAL half is copied. The BMP half (U+2600-U+26FF, + U+2700-U+27BF) is out because no corpus name reaches it through + the rule's \\S...\\S anchor, and a rule should be no wider than the + diffs it must explain -- so this asserts a SUBSET relationship in + that direction, not equality. What it refuses is the other + direction: a class reaching a codepoint the tokenizer does not + divide on. + """ + from nameparser._pipeline._tokenize import _EMOJI_RANGES + + divides = {c for lo, hi in _EMOJI_RANGES for c in range(lo, hi + 1)} + found = 0 + for ledger in _LEDGERS: + for rule in _rules(ledger): + if "emoji-boundary" not in rule["issue"]: + continue + found += 1 + pattern = rule["name_regex"] + claimed = {c for c in range(0x1F000, 0x20000) + if re.search(pattern, f"a{chr(c)}b")} + stray = sorted(claimed - divides) + assert not stray, ( + f"{ledger.name}: {rule['issue']!r} claims " + f"{len(stray)} codepoint(s) the tokenizer does not " + f"divide on, e.g. {[hex(c) for c in stray[:3]]}. Its " + f"prose says an emoji inside a token divides it, which " + f"is false for those -- so a diff with another cause " + f"would classify here as intended. Copy from " + f"_EMOJI_RANGES rather than widening the span.") + assert claimed, ( + f"{ledger.name}: {rule['issue']!r} claims no astral " + f"codepoint at all; the class or the anchor is broken " + f"and the rule can explain nothing") + assert found, ( + "no emoji-boundary rule in any ledger; this pin is passing " + "vacuously") + + def test_cjk_corpus_matches_the_case_table() -> None: """corpus_cjk.jsonl is GENERATED, not curated (#295): every distinct case-table text bearing a codepoint the script table @@ -1424,6 +1475,8 @@ def _claim(rule: dict) -> _Claim: _Claim(11, ('given', 'middle'), "1eaed91fc574"), "fix(#272/#308) nakaguro division and a glued hangul honorific in one name": _Claim(1, ('family', 'given', 'middle', 'suffix'), "2fbf1a94f122"), + "fix(emoji-boundary) an emoji inside a token divides it": + _Claim(1, ('family', 'given'), "efa60ca42d4a"), "fix(nickname-typographic-pairs) two typographic quote spans read as one nickname set": _Claim(1, ('family', 'given', 'middle', 'nickname'), "3cf566c78800"), "fix(#411) the bound-given reserve stops counting words the maiden name takes": diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 62a22ef3..b174d571 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -534,6 +534,42 @@ issue = "fix(#272/#308) nakaguro division and a glued hangul honorific in one na name_regex = "^\\S*\u00b7\\S*\uc528$" fields = ["given", "middle", "family", "suffix"] +[[change]] +issue = "fix(emoji-boundary) an emoji inside a token divides it" +# 'John😀Smith': 1.4 dropped the emoji and read one token +# (first='JohnSmith'); 2.x treats it as a division, so the name reads +# given 'John', family 'Smith'. rules.md#T1 documents both this and the +# keep-emoji policy reading. +# +# The emoji must stand BETWEEN two non-space characters, which is the +# behavior: it divides a token. A bare character class was the first +# draft and is rejected -- measured, it reaches four corpus names, and +# the three it adds carry a spaced or trailing emoji, are byte-identical +# on both sides today, and would be pre-excused by a rule that claims +# them. Pre-claiming shapes it has never seen is the #372 failure mode +# the fix(#379) comment names. +# +# The class is the ASTRAL HALF of _tokenize.py's _EMOJI_RANGES, hand +# copied and pinned by test_the_emoji_boundary_rule_copies_the_dividing_ranges. +# A single \U0001F300-\U0001FAFF span was the first draft and is wrong +# in the dangerous direction: it covers U+1F650-U+1F67F and +# U+1F700-U+1FAFF, where the parser does NOT divide, so the rule's own +# prose is false for about a thousand codepoints it would claim. +# 'John\U0001FA70Smith' reads given 'John\U0001FA70Smith' with no family -- +# a future {given, family} diff on such a name, with some other cause, +# would classify here as intended. That is the same #372 shape this +# comment invokes against the bare class. +# +# The BMP half (U+2600-U+26FF, U+2700-U+27BF) is deliberately left out +# rather than copied: no corpus name reaches it once the token anchor +# is on, and a rule should be no wider than the diffs it must explain. +# A name that needs it will arrive unclassified and be read once. That +# omission also keeps the pattern free of \uXXXX-\uXXXX span text, so +# _SPAN_BEARING_RULES has nothing to declare about a rule that is not +# script-scoped. +name_regex = "\\S[\\U0001F300-\\U0001F64F\\U0001F680-\\U0001F6FF]\\S" +fields = ["given", "family"] + [[change]] issue = "fix(nickname-typographic-pairs) two typographic quote spans read as one nickname set" # 'Hans „Erster“ und “Zweiter” Müller': 2.0 extracts typographically From 8b06f488bf42f1cc83ecb68b9f62b7c21be5394d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 21:32:41 -0700 Subject: [PATCH 04/15] fix(differential,guards): stop the catch-all declaring a role it never explains Measured: every name still on the fields-only rule moves {family, suffix}. It declared `given` as well, so it pre-claimed a shape it has never explained -- the fix(#424) defect decisions.md#H1 records, a rule matching by superset while the diff beneath it shrinks. _CORPUS_CLAIMS' `names` stays 1090 because the rule still has no name_regex. That is the blindness #451 is about; it closes when the rule goes. The comment loses its LIVE count and keeps its historical ones. "Four corpus names, and the prose fits all four" was written when the rule explained four and read the same while it explained fourteen, and nothing recomputes it; 'Johnson PhD' had likewise stopped being a corpus name. #372's 25-and-21 stay, because a past-tense count does not go stale, and they are the record of the same surgery this issue is finishing. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 2 +- tools/differential/expected_since_1.4.0.toml | 45 +++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 2f870b21..e3d23e4e 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1442,7 +1442,7 @@ def _claim(rule: dict) -> _Claim: "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(279, ('family', 'given'), "28a62b622a48"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": - _Claim(1090, ('family', 'given', 'suffix'), "89e0b6d7f4c8"), + _Claim(1090, ('family', 'suffix'), "89e0b6d7f4c8"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": _Claim(0, ('suffix',), "e3b0c44298fc"), "ambiguous-surname-acronym data change: parenthesized (MA)/(DO) now stays nickname": diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index b174d571..b043071a 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -632,23 +632,38 @@ fields = ["middle", "family"] [[change]] issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" -# 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone -# trailing suffix to family/first (no comma present); 2.0 keeps -# recognized suffixes in `suffix`. Four corpus names, and the prose -# fits all four. -# -# It claimed 25 until #372, on nothing but being the only fields-only -# rule in any ledger: it sorts after every name_regex rule, so it -# takes whatever nothing narrower named. The other 21 have their own -# rules now -- fix(cjk-glued-honorific-peel) and the widened +# 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone trailing suffix +# to family/first (no comma present); 2.0 keeps recognized suffixes in +# `suffix`. +# +# STILL THE ONLY RULE IN ANY LEDGER WITH NO name_regex, and #451 is +# retiring it. Read that issue before widening anything here. What is +# left on it after the named rules #451 has added so far: names in +# several behaviour families, all moving {family, suffix} and no more, +# which is what this `fields` now says. It declared `given` as well +# until #451 measured that no name it explains moves that role -- the +# fix(#424) shape decisions.md#H1 records, a rule matching by superset +# while the diff beneath it shrinks, with nothing able to see it. +# +# The same surgery has been done to it twice. It claimed 25 until +# #372, on nothing but being the only fields-only rule in any ledger: +# it sorts after every name_regex rule, so it takes whatever nothing +# narrower named. #372 gave 21 of those their own rules -- +# fix(cjk-glued-honorific-peel) and the widened # fix(cjk-honorific-suffix) below for #312's honorifics, # fix(comma-precomma-family) above for the three names that move no -# suffix at all. -# -# Being last is the point of this rule and worth keeping. What is not -# worth keeping is it being the only explanation a name ever gets: -# when it grows, the question is which narrower rule is missing. -fields = ["given", "family", "suffix"] +# suffix at all -- and #451 is doing the rest. Those figures are +# history and stay written; a count of what the rule explains NOW is +# not, because nothing recomputes it. The last one read "Four corpus +# names, and the prose fits all four" while the rule explained +# fourteen. Run tools/differential/compare.py --baseline 1.4.0 for +# the live number. +# +# Being last is the point of this rule and worth keeping until it is +# gone. What is not worth keeping is it being the only explanation a +# name ever gets: when it grows, the question is which narrower rule +# is missing. +fields = ["family", "suffix"] [[change]] issue = "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole" From 525f743b6a0ffe54d440b70c249747dd795b8f54 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 21:41:08 -0700 Subject: [PATCH 05/15] fix(differential,guards): port the #380 vd rule to the 1.4 ledger 'Berg, Jan vd' fell to the fields-only catch-all not because nothing described it but because this ledger never got the rule expected_since_2.0.0.toml and expected_since_2.1.0.toml both already carry. Ported verbatim (issue text, regex, fields {family, suffix}). fix(#379)'s comment in both the 1.4.0 and 2.1.0 ledgers promised that a vd-shaped name would "arrive unexplained and be read once" if the corpus ever grew one. That promise was already false: the name is in corpus_issues.jsonl and corpus_rules.jsonl, and in 1.4.0 the fields- only catch-all had been quietly absorbing it, not leaving it unclassified; in 2.1.0 the name never had a gap to fall into at all, because fix(#380) already lived in that same file. Both files are corrected to say what actually happened, worded to each baseline's own measurement rather than copied from one to the other. _CROSS_RULE_WINNERS drops the 'Berg, Jan vd' row and the comment block above it -- the last of the three names that comment described, the other two ported in an earlier #451 commit. All three now have named rules in this ledger. Catch-all 10 -> 9. Gate green at 1.4.0, 2.0.0 and 2.1.0. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 16 +----- tools/differential/expected_since_1.4.0.toml | 60 +++++++++++++++++--- tools/differential/expected_since_2.1.0.toml | 20 ++++--- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index e3d23e4e..43392c68 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1419,6 +1419,8 @@ def _claim(rule: dict) -> _Claim: _Claim(5, ('family', 'given', 'maiden', 'middle'), "bc0e10dd7ec8"), "fix(#379) a tussenvoegsel after a family comma attaches to the family": _Claim(13, ('family', 'middle'), "973617235cda"), + "fix(#380) a trailing vd after a family comma is the tussenvoegsel, not a post-nominal": + _Claim(2, ('family', 'suffix'), "ec0d45289dc1"), "fix(comma-family) lone post-comma piece routes to suffix/title, not first": _Claim(279, ('given', 'suffix', 'title'), "28a62b622a48"), "fix(comma-family) a comma followed only by titles keeps the given/family split": @@ -1835,20 +1837,6 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: #: a diff shape that shifted is a finding, not a number to update. _CROSS_RULE_WINNERS: dict[str, dict[tuple[str, tuple[str, ...]], str]] = { "expected_since_1.4.0.toml": { - # One behaviour that got a named rule in the 2.0 and 2.1 ledgers - # but not this one, so here it falls to the fields-only - # catch-all. The other two -- 'Mc Donald' (#360) and - # 'Sir de Mesnil' (#367) -- had their rules ported into this - # ledger by #451, which is why those rows are gone. - # Recorded because the assignment is incidental rather than - # argued: 'Berg, Jan vd' is reachable by three named rules and - # wins none of them (their `fields` do not match). Being - # absorbed by the catch-all is the recoverable direction, but a - # rule added later that takes it silently would change what the - # 1.4 summary attributes -- and AGENTS.md names that summary as - # the source for the release log's Behavior Changes section. - # #380. - ("Berg, Jan vd", ("family", "suffix")): "fix(suffix-routing)", ("Andrews, M.D.", ("given", "suffix")): "fix(comma-family)", ("田中, 太郎さん", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", ("김, 민준씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index b043071a..d16b3554 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -602,14 +602,19 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # Fields are {middle, family}: the given name does not move. `suffix` # is deliberately NOT here, though the #380 half of the change does # move a token out of it ('Berg, Jan vd' was suffix 'vd' at every -# baseline and is now family 'vd Berg'). No corpus name exercises that -# shape, and adding `suffix` to claim it costs three shapes this rule -# has no business explaining -- {family,suffix} taken from -# fix(suffix-routing), and {middle,suffix} and {middle,family,suffix} -# taken from UNEXPLAINED, the last two being diffs in which `family` -# does not move at all. A rule that pre-claims shapes it has never seen -# is the #372 failure mode. If a vd-shaped name ever enters the corpus -# it should arrive unexplained and be read once. +# baseline and is now family 'vd Berg'). Adding `suffix` to claim it +# costs three shapes this rule has no business explaining -- +# {family,suffix} taken from fix(suffix-routing), and {middle,suffix} +# and {middle,family,suffix} taken from UNEXPLAINED, the last two +# being diffs in which `family` does not move at all. A rule that +# pre-claims shapes it has never seen is the #372 failure mode. +# +# 'Berg, Jan vd' entered corpus_issues.jsonl and corpus_rules.jsonl +# after this was written, and it did NOT arrive unexplained -- the +# fields-only catch-all took it, and _CROSS_RULE_WINNERS recorded the +# absorption rather than closing it. It has fix(#380)'s own rule now +# (#451, below). The declining was right; what was missing was the +# rule beside it. # # Latin-anchored over the WHOLE name, not just the trailing word: with # a leading .* this absorbed non-Latin comma names whose diff has @@ -630,6 +635,45 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" name_regex = "(?i)^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*\\s(de|del|den|der|di|do|dos|du|la|le|los|mc|van|vd|von|zu)(\\s+(de|del|den|der|di|do|dos|du|la|le|los|mc|van|vd|von|zu))*\\s*$" fields = ["middle", "family"] +[[change]] +issue = "fix(#380) a trailing vd after a family comma is the tussenvoegsel, not a post-nominal" +# 'Berg, Jan vd': the tree reads it given 'Jan', family 'vd Berg', +# suffix '' -- the trailing 'vd' attaches to the family the comma +# already named, the same P6 precedence fix(#379) states above for the +# leading-particle shapes. Whether 1.4 itself read the trailing word +# as `suffix` is outside what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 release +# from a cached environment); the diff this rule explains at this +# baseline moves exactly {family, suffix}, which is consistent with +# that reading and is what the fields below claim. +# +# decisions.md#vocabulary-collisions C-ii records the word itself: +# "vd is the live case: never-given particle AND credential acronym +# (the British Volunteer Decoration), neither of them a name. +# Decision: the Dutch van der reading, as the more common." #380 is +# named there as the trailing-orphan half that decision unblocks. +# +# The {family, suffix} companion to fix(#379) above, which claims +# {middle, family} and deliberately declined `suffix` -- that +# comment's own accounting of what claiming it would have cost stands +# as written; this rule is the piece it left out, not a widening of +# it. +# +# Ported verbatim from expected_since_2.0.0.toml, issue text and regex +# included, so the _MUST_NOT_MATCH "fix(#380)" key that already guards +# the 2.x copies guards this one too. Latin-anchored over the whole +# name for fix(#379)'s reason: with a leading .* it would absorb +# non-Latin comma names whose diff has another cause. +# +# The regex also reaches 'Berg, abdul vd', which fix(comma-precomma-family) +# above claims on {family, given} -- a different diff shape, so this +# rule's {family, suffix} fields never match it and it is not claimed +# here. That reach is real, so 'Berg, abdul vd' is NOT a +# _MUST_NOT_MATCH probe for this rule: that roster tests the regex, +# not classify(). +name_regex = "(?i)^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*\\svd\\s*$" +fields = ["family", "suffix"] + [[change]] issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" # 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone trailing suffix diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index f1d50efa..e1dbd085 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -698,14 +698,18 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # Fields are {middle, family}: the given name does not move. `suffix` # is deliberately NOT here, though the #380 half of the change does # move a token out of it ('Berg, Jan vd' was suffix 'vd' at every -# baseline and is now family 'vd Berg'). No corpus name exercises that -# shape, and adding `suffix` to claim it costs three shapes this rule -# has no business explaining -- {family,suffix} taken from -# fix(suffix-routing), and {middle,suffix} and {middle,family,suffix} -# taken from UNEXPLAINED, the last two being diffs in which `family` -# does not move at all. A rule that pre-claims shapes it has never seen -# is the #372 failure mode. If a vd-shaped name ever enters the corpus -# it should arrive unexplained and be read once. +# baseline and is now family 'vd Berg'). Adding `suffix` to claim it +# costs three shapes this rule has no business explaining -- +# {family,suffix} taken from fix(suffix-routing), and {middle,suffix} +# and {middle,family,suffix} taken from UNEXPLAINED, the last two +# being diffs in which `family` does not move at all. A rule that +# pre-claims shapes it has never seen is the #372 failure mode. +# +# 'Berg, Jan vd' is a corpus name, and it did not arrive unexplained +# here either: fix(#380) above already claims {family, suffix} for +# exactly this shape, in this same ledger, so there was never a gap +# for a future rule to fill at this baseline the way there was at +# 1.4.0's. # # Latin-anchored over the WHOLE name, not just the trailing word: with # a leading .* this absorbed non-Latin comma names whose diff has From 8237b548fe52e587c4b039619867a2b526c0c06f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 21:44:04 -0700 Subject: [PATCH 06/15] fix(differential): drop four false claims the 2.x ledgers copied from the 1.4 one Found reviewing the #380 port. All three ledgers carry the same fix(#379) comment, and three of its sentences are true only in the 1.4.0 file they were written for: - "No corpus name exercises that shape" -- 'Berg, Jan vd' has been in corpus_issues.jsonl and corpus_rules.jsonl since #414's harvest. - "{family,suffix} taken from fix(suffix-routing)" -- there is no rule of that name in either 2.x ledger. It is the 1.4 catch-all, and the shape is taken by fix(#380), which sits ABOVE fix(#379) in both files. - "If a vd-shaped name ever enters the corpus it should arrive unexplained and be read once" -- fix(#380) claims it, so it never could. The fourth is in the Latin-anchoring paragraph: a hypothetical Cyrillic comma name "now falls to the fields-only catch-all". Neither 2.x ledger has a fields-only rule -- each says so twenty lines earlier -- so such a name would arrive UNEXPLAINED, which is the right outcome and the better story. Measured: no Cyrillic comma name is in any corpus, so the sentence was always about a shape nobody has seen. Same defect class as #451 itself: a comment asserting something the ledger it sits in cannot support. Comment-only; all three gates unmoved (1.4.0 229/0, 2.0.0 194/0, 2.1.0 102/0) and 147 guards green. The 1.4.0 copy of the catch-all sentence is still TRUE and is left alone. It stops being true when #451 deletes the rule, which that commit has to handle. Refs #451 Co-Authored-By: Claude Opus 5 --- tools/differential/expected_since_2.0.0.toml | 32 +++++++++++++------- tools/differential/expected_since_2.1.0.toml | 11 ++++--- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 82d7dd0d..43cc22c2 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -792,22 +792,32 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # Fields are {middle, family}: the given name does not move. `suffix` # is deliberately NOT here, though the #380 half of the change does # move a token out of it ('Berg, Jan vd' was suffix 'vd' at every -# baseline and is now family 'vd Berg'). No corpus name exercises that -# shape, and adding `suffix` to claim it costs three shapes this rule -# has no business explaining -- {family,suffix} taken from -# fix(suffix-routing), and {middle,suffix} and {middle,family,suffix} -# taken from UNEXPLAINED, the last two being diffs in which `family` -# does not move at all. A rule that pre-claims shapes it has never seen -# is the #372 failure mode. If a vd-shaped name ever enters the corpus -# it should arrive unexplained and be read once. +# baseline and is now family 'vd Berg'). Adding `suffix` to claim it +# costs three shapes this rule has no business explaining -- +# {family,suffix} taken from fix(#380) above, and {middle,suffix} and +# {middle,family,suffix} taken from UNEXPLAINED, the last two being +# diffs in which `family` does not move at all. A rule that pre-claims +# shapes it has never seen is the #372 failure mode. +# +# 'Berg, Jan vd' is a corpus name, and it did not arrive unexplained +# here: fix(#380) above already claims {family, suffix} for exactly +# this shape, in this same ledger, so there was never a gap for a +# future rule to fill at this baseline the way there was at 1.4.0's. +# The two sentences this replaces -- "No corpus name exercises that +# shape" and "If a vd-shaped name ever enters the corpus it should +# arrive unexplained" -- were copied from the 1.4 ledger and were +# false here from the day fix(#380) was written above them (#451). # # Latin-anchored over the WHOLE name, not just the trailing word: with # a leading .* this absorbed non-Latin comma names whose diff has # another cause -- a Cyrillic 'Smith, de la' moves {family} because the # particle's RENDERING ORDER inside the family changed, which no rule -# in this ledger names, and it classified here. (It now falls to the -# fields-only catch-all, so the absorption moved rather than vanished; -# worth a narrower rule if such a name ever appears.) The class is the +# in this ledger names, and it classified here. (No such name is in +# any corpus, so this is what WOULD happen. It would now arrive +# UNEXPLAINED and block the gate until someone read it, which is the +# right outcome: this ledger has no fields-only rule to absorb it. +# The claim that it "falls to the fields-only catch-all" was copied +# from the 1.4 ledger, where there is one -- #451.) The class is the # neighbours' and compare.py's own: U+0000-U+024F, which stops at # Latin Extended-B, so a diacritic-bearing 'Nguyen, Van de la' in NFD # or Latin Extended Additional is outside it. diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index e1dbd085..78686367 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -700,7 +700,7 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # move a token out of it ('Berg, Jan vd' was suffix 'vd' at every # baseline and is now family 'vd Berg'). Adding `suffix` to claim it # costs three shapes this rule has no business explaining -- -# {family,suffix} taken from fix(suffix-routing), and {middle,suffix} +# {family,suffix} taken from fix(#380) above, and {middle,suffix} # and {middle,family,suffix} taken from UNEXPLAINED, the last two # being diffs in which `family` does not move at all. A rule that # pre-claims shapes it has never seen is the #372 failure mode. @@ -715,9 +715,12 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # a leading .* this absorbed non-Latin comma names whose diff has # another cause -- a Cyrillic 'Smith, de la' moves {family} because the # particle's RENDERING ORDER inside the family changed, which no rule -# in this ledger names, and it classified here. (It now falls to the -# fields-only catch-all, so the absorption moved rather than vanished; -# worth a narrower rule if such a name ever appears.) The class is the +# in this ledger names, and it classified here. (No such name is in +# any corpus, so this is what WOULD happen. It would now arrive +# UNEXPLAINED and block the gate until someone read it, which is the +# right outcome: this ledger has no fields-only rule to absorb it. +# The claim that it "falls to the fields-only catch-all" was copied +# from the 1.4 ledger, where there is one -- #451.) The class is the # neighbours' and compare.py's own: U+0000-U+024F, which stops at # Latin Extended-B, so a diacritic-bearing 'Nguyen, Van de la' in NFD # or Latin Extended Additional is outside it. From 7b2b2b68eb3b4d602d5b2933e426c49a872885a1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 22:19:23 -0700 Subject: [PATCH 07/15] fix(differential,guards): give the nickname-plus-suffix name its own rule "'Smitty' Jones Jr." was one of the fields-only fix(suffix-routing) catch-all's nine names at 1.4.0. It is three tokens -- a quoted nickname, one bare name word, one suffix word -- so it will not fit the two-token suffix rules #451 is landing in a later task; it needs its own rule now. The tree reads it given 'Jones', family '', suffix 'Jr.', which is decisions.md#N3's lone-word nickname rule at work: the diff against 1.4.0 moves exactly {family, suffix}, consistent with 1.4 having routed the trailing suffix word into `family` the way the general suffix-routing note describes, though 1.4's own internals are outside what this ledger's worker can check directly. The comment deliberately does not settle which of two readings is right for a nickname-led name with a trailing suffix: decisions.md#H1 records that N3 and H1 count a suffix in opposite directions, and calls that question undecided. This rule classifies the 1.4-to-2.x movement only. Declares the suffix alternation in _LATIN_ALTERNATION_SOURCES (SUFFIX_WORDS, all six members genuine entries), adds a _MUST_NOT_MATCH probe set (two-token names and the H1 title shape), and records the _CORPUS_CLAIMS entry. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 11 +++++++ tools/differential/expected_since_1.4.0.toml | 32 ++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 43392c68..42bebb71 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -677,6 +677,12 @@ def test_cjk_corpus_matches_the_case_table() -> None: # missing is fix(#399)'s or fix(#412)'s "fix(#418) accepted": ("Jane née Jr Jones", "Jane née y Jones", "Jane van der Berg née Jr Jones"), + # decisions.md#H1's own nickname-plus-title-plus-name shape, kept + # out on purpose: this rule is anchored to a trailing suffix word, + # not a title, and a two-token name (with or without the suffix) + # has no third token for the regex to require. + "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": + ("'Smitty' Jones", "Jones Jr.", "'Smitty' Dr. Jones"), } @@ -978,6 +984,9 @@ class _LatinCopy(NamedTuple): covers=frozenset({"de", "del", "den", "der", "di", "do", "dos", "du", "la", "le", "los", "mc", "van", "vd", "von", "zu"})), + "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": + _LatinCopy(vocabulary=SUFFIX_WORDS, + covers=frozenset({"jr", "sr", "ii", "iii", "iv", "v"})), } #: Alternations that copy no vocabulary, so discovery must not demand a @@ -1523,6 +1532,8 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('family', 'given'), "db724fb9c779"), "fix(#445) a maiden marker makes the lone name word the family": _Claim(7, ('family', 'given', 'maiden', 'middle'), "3de9ef12b4a8"), + "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": + _Claim(1, ('family', 'suffix'), "570f265a2f46"), }, "expected_since_2.0.0.toml": { "fix(#335) a marker-led clause leaves the one name word its bare reading": diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index d16b3554..0caef9f2 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1789,3 +1789,35 @@ issue = "fix(#445) a maiden marker makes the lone name word the family" # above listed as the checklist to re-measure. name_regex = "(?i)^(?:Janey n[ée]e Jones|Jane n[ée]e Jones J\\. V|Jane n[ée]e Jones Smith|Jane n[ée]e and Jones Smith|John n[ée]e Jones Smith V|Smith n[ée]e Jones|Smith n[ée]e Jones PhD)$" fields = ["given", "middle", "family", "maiden"] + +[[change]] +issue = "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`" +# "'Smitty' Jones Jr.": the diff against 1.4.0 moves exactly {family, +# suffix} -- `given` is 'Jones' at both baselines, which is what the +# N3 lone-word nickname rule already gives it once the nickname and +# suffix are set aside. Whether 1.4 itself read the trailing word as +# `family` is outside what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 +# release from a cached environment); the diff shape is consistent +# with that reading, and 2.x reads this name given 'Jones', family +# '', suffix 'Jr.'. decisions.md#N3. +# +# Its own rule because it is three tokens -- a quoted nickname, one +# bare name word, one suffix word -- and #451 is retiring the +# fields-only fix(suffix-routing) catch-all above by giving its names +# narrower rules; this name was one of that catch-all's nine at the +# start of this task. No two-token suffix rule exists in this ledger +# yet to hold it either way: #451 is landing those in a later task, +# and this shape -- three tokens, a leading nickname -- would not fit +# one regardless. +# +# Note what is NOT settled here: decisions.md#H1 records that N3 and +# H1 count a trailing suffix in opposite directions -- H1's +# title-plus-one-word rule declines the moment a suffix stands beside +# it, while "N3 counts a suffix the other way (`'Smitty' Jones Jr.` +# reads given 'Jones', family '')" -- and H1 states plainly that +# "which of the two readings is right for a nickname-led name is not +# decided here". This rule classifies only the 1.4-to-2.x movement +# measured above; it does not endorse either reading. +name_regex = "(?i)^['\"‘“][^'\"’”]+['\"’”]\\s+\\S+\\s+(jr|sr|ii|iii|iv|v)\\.?$" +fields = ["family", "suffix"] From 67ec9cb8e3fb7c17ed04e6ef506b2415a2cae26b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 22:28:09 -0700 Subject: [PATCH 08/15] fix(differential,guards): give the two NOT-WANTED regressions their own rules 'Aishwarya Rai' and 'Carod i' were two of the fields-only fix(suffix-routing) catch-all's eight names at 1.4.0, and unlike the other six their readings are open bugs, not accepted behavior: #342 (OPEN, bug, milestone v2.2) reports that a bare trailing 'Rai' -- a common South Asian surname -- is read as a post-nominal credential instead, emptying `family`; #397 (OPEN, enhancement, milestone v2.2) reports the same shape for a trailing Catalan/Polish linking 'i', read as a Roman-numeral generation marker instead of joining the family the way Spanish 'y' already does. Both get literal-anchored rules rather than a [[never]] exclusion. An excluded name reports UNEXPLAINED and exits non-zero, which would redden the 1.4.0 gate and block every release until both bugs land -- and neither has a chosen fix yet (#342 lists two candidate options that trade different corpus names; #397 leaves open whether 'i' joins CONJUNCTIONS at all). Both rules classify "because the cause is known and recorded, NOT because the reading is wanted", the exact wording decisions.md#P5's 2026-08-22 #369 entry attributes to that #369 ledger's own now-rewritten rules. Each rule carries a DELETE-THIS-RULE-when-fixed instruction. Once the fix lands the rule stops matching any diff, and the gate's own dormancy check (not a declared `dormant`, the general one that catches any rule gone quiet) prints it as 'EXPLAINED NOTHING' -- the reminder is mechanical, not just a comment. Catch-all 8 -> 6. Adds both _MUST_NOT_MATCH probe sets (three-token neighbours and #397's two no-diff longer names) and both _CORPUS_CLAIMS entries. Gate green at 1.4.0 (229/0), 2.0.0 (194/0) and 2.1.0 (102/0); 147 guards green. Refs #451, #342, #397 Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 12 +++ tools/differential/expected_since_1.4.0.toml | 91 ++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 42bebb71..9450251a 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -683,6 +683,14 @@ def test_cjk_corpus_matches_the_case_table() -> None: # has no third token for the regex to require. "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": ("'Smitty' Jones", "Jones Jr.", "'Smitty' Dr. Jones"), + # #451's two NOT-WANTED rules, literal-anchored to one corpus name + # apiece: a three-token name with a rootname before or after is a + # different diff shape (or, for 'Carod i Rovira' and 'Lluis Carod + # i', no diff at all -- #397's still-open enhancement, not a + # 1.4-to-2.x regression), and 'Rai'/'Jane Rai Smith' have no + # 'aishwarya' to anchor on. + "fix(#342)": ("Aishwarya Rai Bachchan", "Rai", "Jane Rai Smith"), + "fix(#397)": ("Carod i Rovira", "Josep Carod i Rovira", "Lluis Carod i"), } @@ -1454,6 +1462,10 @@ def _claim(rule: dict) -> _Claim: _Claim(279, ('family', 'given'), "28a62b622a48"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(1090, ('family', 'suffix'), "89e0b6d7f4c8"), + "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost": + _Claim(1, ('family', 'suffix'), "694fd06a2e9a"), + "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost": + _Claim(1, ('family', 'suffix'), "498602f3cfd0"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": _Claim(0, ('suffix',), "e3b0c44298fc"), "ambiguous-surname-acronym data change: parenthesized (MA)/(DO) now stays nickname": diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 0caef9f2..873688b6 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -709,6 +709,97 @@ issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix sta # is missing. fields = ["family", "suffix"] +[[change]] +issue = "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost" +# 'Aishwarya Rai': the tree reads given 'Aishwarya', family '', suffix +# 'Rai'. 'rai' is a bare entry in SUFFIX_ACRONYMS +# (nameparser/config/suffixes.py -- not SUFFIX_ACRONYMS_AMBIGUOUS, +# not SUFFIX_WORDS), so the credential reading outranks the surname +# reading with nothing to arbitrate them. +# #342 (OPEN, milestone v2.2, bug) reports this: Rai, in the issue's +# own words, is "a common surname across Hindi- and Bengali-speaking +# regions," and the bulk 2019-12-11 wikipedia post-nominal import +# (af5bdab) that added it was never reviewed against that collision. +# +# Whether 1.4 itself read the trailing word as `family` is outside +# what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 release +# from a cached environment); the diff this rule explains at this +# baseline moves exactly {family, suffix}, which is consistent with +# that reading and is what the fields below claim. +# +# Classified because the cause is known and recorded, NOT because the +# reading is wanted -- decisions.md#P5's 2026-08-22 #369 entry +# attributes that exact wording, quoted verbatim, to the previous +# fix(#369) ledger rules (since rewritten, because #369 itself was +# fixed). A [[never]] exclusion was considered and rejected: an +# excluded name reports UNEXPLAINED and exits non-zero, which would +# redden the 1.4.0 gate and block every release until #342 lands -- +# and #342 lists two candidate fixes that trade different corpus +# names against each other, so which one lands is not yet decided. +# +# DELETE THIS RULE when #342 lands. The gate then reports it as +# 'EXPLAINED NOTHING', which is the reminder. +# +# Literal-anchored to the one corpus name that reaches this shape. A +# rule keyed on the SUFFIX_ACRONYMS collision itself, rather than the +# literal name, would claim every trailing acronym-shaped surname and +# pre-excuse regressions on names nobody has looked at. +name_regex = "(?i)^aishwarya\\s+rai$" +fields = ["family", "suffix"] + +[[change]] +issue = "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost" +# 'Carod i': the tree reads given 'Carod', family '', suffix 'i'. 'i' +# is a bare entry in SUFFIX_WORDS (nameparser/config/suffixes.py -- +# the Roman-numeral generation markers i/ii/iii/iv/v, not +# SUFFIX_ACRONYMS), so a two-word name ending in it reads as a +# name plus a trailing generation rather than a two-word family link. +# #397 (OPEN, milestone v2.2, enhancement) reports that Catalan and +# Polish link two surnames with 'i' functioning as a conjunction, the +# way Spanish already links them with 'y' ("Juan Garcia y Lopez" reads +# family 'Garcia y Lopez'); nameparser's CONJUNCTIONS vocabulary does +# not include 'i', so nothing there joins the link, and SUFFIX_WORDS' +# numeral reading is what actually claims the trailing word instead. +# +# Whether 1.4 itself read the trailing word as `family` is outside +# what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 release +# from a cached environment); the diff this rule explains at this +# baseline moves exactly {family, suffix}, which is consistent with +# that reading and is what the fields below claim. +# +# Classified because the cause is known and recorded, NOT because the +# reading is wanted -- the same standing fix(#342) above claims, from +# decisions.md#P5's 2026-08-22 #369 entry. A [[never]] exclusion was +# considered and rejected for the same reason as fix(#342) above: an +# excluded name reports UNEXPLAINED and exits non-zero, and #397's own +# "What needs deciding" section leaves open both whether 'i' joins +# CONJUNCTIONS at all and how the join's rootname carve-out should +# treat a word that is both a conjunction and suffix vocabulary -- +# unresolved, so the fix is not yet in hand. +# +# DELETE THIS RULE when #397 lands. The gate then reports it as +# 'EXPLAINED NOTHING', which is the reminder. +# +# Literal-anchored to the one corpus name that reaches this shape, for +# fix(#342) above's reason: a rule keyed on "two-word name, trailing +# single letter" would pre-excuse every generation-marker regression, +# not just this one. Measured, the four longer corpus names in the +# same family produce no diff at this baseline at all, so none of them +# reaches classify() and none needs a rule here: 'Carod i Rovira', +# 'Josep Carod i Rovira' and 'Josep Lluis Carod i Rovira' read 'i' into +# `middle` with `family` already the tail word ('Rovira'), and 'Lluis +# Carod i' reads it into `suffix` with `family` already populated +# ('Carod') -- unlike 'Carod i' above, none of the four has an empty +# `family`, and all four read this way at every baseline this ledger's +# worker can reach. That is #397's open enhancement (the family link +# is still one word short of what a Catalan reader wants), not a +# 1.4-to-2.x regression, so it is out of scope for a ledger that only +# records diffs FROM 1.4.0. +name_regex = "(?i)^carod\\s+i$" +fields = ["family", "suffix"] + [[change]] issue = "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole" # Only fires when a custom suffix delimiter is configured (Policy / From ffd242a612c62596b303ed8fdd2c77fd836a30b7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 22:56:09 -0700 Subject: [PATCH 09/15] fix(differential,guards): retire the fields-only catch-all for four bounded rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last six names on the fields-only fix(suffix-routing) rule get rules that name them, and the catch-all is deleted. No ledger has a fields-only rule now -- 1.4.0 70 rules, 2.0.0 58, 2.1.0 51, and 0 with no name_regex in any of them -- which is what #451 was filed for. Where the six went: 'John V', 'Mohamad X' -> a trailing roman numeral 'Smith Jr.' -> the trailing suffix word jr 'Donald mc', 'QC MP' -> a trailing credential acronym 'Jack M.A.' -> the dotted M.A. spelling (ma-do) Four rather than one, and the split is FORCED rather than chosen. _LatinCopy carries exactly one `vocabulary` and test_latin_alternations_mean_something_the_vocabulary_ships asserts exactly one roster key per alternation, so `jr` (SUFFIX_WORDS) and `mc|mp` (SUFFIX_ACRONYMS) cannot share an alternation. `m\.?a\.?` cannot be an alternation member at all: measured, it matches the corpus fragment 'M.A.', which _normalize leaves as 'm.a', not a SUFFIX_ACRONYMS entry -- the same finding the ambiguous-surname-acronym rule already records. And the numeral reaches `suffix` through no wordlist at all: `x` is in none of the suffix sets, so _NOT_A_VOCABULARY_COPY is where its alternation is declared, being _ROMAN's body with the empty alternative dropped (proved equivalent on every string of length 1..4 over IVXivx; the empty case is a latent widening removed, not a bug fixed). All four are appended at the END of the file. The catch-all sorted last because it had no name_regex; with regexes these join the regex tier where file order decides ties, so appending them reproduces its position exactly and no rule above loses a contest. Verified before and after: 'Carod i' stays with fix(#397) and '田中さん II' with fix(cjk-glued-honorific-peel), and _CROSS_RULE_WINNERS now pins both. Deleting the rule falsified prose in the same file, and the sweep found more than the one sentence this task set out to fix. The fix(#379) Latin-anchoring paragraph now reads as its 2.0.0 and 2.1.0 copies have since 8237b54: a hypothetical Cyrillic comma name would arrive UNEXPLAINED and block the gate, which is the right outcome. Twelve other references to the catch-all are corrected -- past-tense history kept, dangling "below" pointers dropped, and three sentences that were already false before this commit rewritten against fresh measurements: two counterfactuals that stopped holding when the catch-all lost `given` earlier in this bundle, and _EXCLUSION_EFFECT's docstring figure, which named a one-tuple where the answer was a three-tuple. fix(suffix-routing) leaves that roster's own absorbed_by tuple, a shrink rather than a growth. Gate green at 1.4.0 (229/0), 2.0.0 and 2.1.0; 147 guards green; tools/differential/compare.py untouched. Refs #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_differential.py | 8 +- tests/v2/test_ledger_guards.py | 153 +++++++- tools/differential/expected_since_1.4.0.toml | 376 ++++++++++++++----- 3 files changed, 421 insertions(+), 116 deletions(-) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index 3ac3f844..501ecb27 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -862,10 +862,12 @@ def test_validate_exclusions_accepts_the_shipped_entries() -> None: def test_classify_refuses_an_excluded_shape() -> None: """The whole point: an excluded name reports UNEXPLAINED however - many rules would otherwise claim it. Two do, for the shape this + many rules would otherwise claim it. Two did, for the shape this was built for -- fix(comma-family) on file order, and the - fields-only fix(suffix-routing) which has no name_regex at all and - so reaches every name.""" + fields-only fix(suffix-routing) which had no name_regex at all and + so reached every name. #451 deleted that second one, and no ledger + has a fields-only rule now; the fixture below keeps one because + the behaviour it pins is compare.classify's, not any ledger's.""" rules = [{"issue": "broad", "name_regex": ","}, {"issue": "broader", "fields": ["given", "suffix"]}] never = [{"why": "parity", "name_regex": r"(?i)\bph\.\s*d\.\s*$", diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 9450251a..634c6ad4 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -49,7 +49,8 @@ class declares, which members an alternation offers. Those are exact from nameparser.config.particles import PARTICLES from nameparser.config.titles import GIVEN_NAME_TITLES, TITLES from nameparser.config.suffixes import ( - GLUED_HONORIFICS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_WORDS) + GLUED_HONORIFICS, SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, + SUFFIX_WORDS) from ._differential_fixtures import ( _CORPUS_NAMES, _LEDGERS, _TOOLS, _UNCLASSIFIED_NAMES, _claimed, @@ -691,6 +692,34 @@ def test_cjk_corpus_matches_the_case_table() -> None: # 'aishwarya' to anchor on. "fix(#342)": ("Aishwarya Rai Bachchan", "Rai", "Jane Rai Smith"), "fix(#397)": ("Carod i Rovira", "Josep Carod i Rovira", "Lluis Carod i"), + # #451's four replacements for the fields-only catch-all. Each is + # anchored to a two-token name, so the probes are a third token and + # each other's vocabulary: the four exist BECAUSE one rule could not + # carry all six names, and a rule that quietly grew to reach a + # sibling's name would put the split back where it started. + # + # 'Carod i' and '田中さん II' are deliberately NOT probes for the + # numeral rule: its regex really does reach both, and rules above it + # win them -- 'Carod i' on file order, '田中さん II' on the subset + # test, its diff moving {given, suffix} where the numeral rule + # declares {family, suffix}. _CROSS_RULE_WINNERS pins both instead; + # this roster tests the regex, not classify(). + "fix(suffix-routing) a two-token name ending in a roman numeral keeps it in `suffix`": + ("Mohamad X Surname", "Smith Jr.", "Donald mc", "Aishwarya Rai"), + "fix(suffix-routing) a two-token name ending in the suffix word jr keeps it in `suffix`": + ("John Smith Jr.", "Smith Jr. PhD", "John V", "Jack Ma"), + # 'Mc Donald' is the leading-particle shape fix(#360) claims and + # 'Berg, Jan vd' the comma shape P6 gives to fix(#380); this rule is + # the third corner, trailing and comma-less, and must not reach + # either of the others. + "fix(suffix-routing) a two-token name ending in a credential acronym keeps it in `suffix`": + ("Mc Donald", "Berg, Jan vd", "John Smith MP", "Jack Ma"), + # decisions.md#ma-do turns on the BARE spelling keeping its surname + # while the dotted one reads as a credential, so both spellings of + # the bare one are probes here -- and 'Jack Ma' is a probe for the + # acronym rule above as well, since 'ma' is acronym vocabulary too. + "fix(suffix-routing) the dotted M.A. spelling reads as a credential (ma-do)": + ("Jack Ma", "Jack MA", "John Smith M.A."), } @@ -995,6 +1024,21 @@ class _LatinCopy(NamedTuple): "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": _LatinCopy(vocabulary=SUFFIX_WORDS, covers=frozenset({"jr", "sr", "ii", "iii", "iv", "v"})), + # #451's credential-acronym rule, one of the four that replaced the + # fields-only fix(suffix-routing) catch-all. Two members, because + # 'Donald mc' and 'QC MP' are the two corpus names in that shape; + # partial for fix(#379)'s reason, SUFFIX_ACRONYMS running to + # hundreds of entries that nobody writes after a bare name word. + # + # This is the roster entry that FORCED #451's split into four rules + # rather than one: the sibling rule's `jr` is a SUFFIX_WORDS entry, + # _LatinCopy carries exactly one `vocabulary`, and the test below + # asserts exactly one roster key per alternation -- so an + # alternation spanning both vocabularies could not be pinned + # against either, and had to become two rules. + "fix(suffix-routing) a two-token name ending in a credential acronym keeps it in `suffix`": + _LatinCopy(vocabulary=SUFFIX_ACRONYMS, + covers=frozenset({"mc", "mp"})), } #: Alternations that copy no vocabulary, so discovery must not demand a @@ -1033,6 +1077,26 @@ class _LatinCopy(NamedTuple): frozenset({"Janey n[ée]e Jones", "Jane n[ée]e Jones J\\. V", "Jane n[ée]e Jones Smith", "John n[ée]e Jones Smith Ma", "Smith n[ée]e Jones", "Smith n[ée]e Jones PhD"}), + # #451's roman-numeral rule copies _ROMAN -- the pattern in + # nameparser/_pipeline/_vocab.py that _vocab.is_trailing_numeral_suffix + # matches a final piece against -- and NOT a wordlist. There is no + # vocabulary here to drift from: 'x' is in none of SUFFIX_WORDS, + # SUFFIX_ACRONYMS or their ambiguous subsets, and 'Mohamad X' moves + # to `suffix` all the same, which is the whole reason the rule + # cannot be an entry in _LATIN_ALTERNATION_SOURCES. + # + # The members are _ROMAN's body with the EMPTY alternative removed + # (I{0,3} -> I{1,3}, plus the bare V that would otherwise be lost). + # Measured over every string of length 1..4 from IVXivx, the two + # accept the same strings apart from the empty one. Nothing pins + # the ledger's copy against _ROMAN: test_regex_sync pins _ROMAN + # against _config.REGEXES["roman_numeral"], which is the parser's + # two copies of it, and a hand copy in a toml is outside that pair. + # What holds this one is _CORPUS_CLAIMS' reach and digest plus the + # _MUST_NOT_MATCH probes, the same wall every other literal rule + # here stands behind. This roster records only that a wordlist is + # not what is copied. + frozenset({"X", "IX", "IV", "V?I{1,3}", "V"}), }) def _unjustified_reach(name_regex: str, members: set[str]) -> list[str]: @@ -1460,8 +1524,6 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('family', 'given', 'middle'), "d8ee9cd5da5f"), "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(279, ('family', 'given'), "28a62b622a48"), - "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": - _Claim(1090, ('family', 'suffix'), "89e0b6d7f4c8"), "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost": _Claim(1, ('family', 'suffix'), "694fd06a2e9a"), "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost": @@ -1546,6 +1608,21 @@ def _claim(rule: dict) -> _Claim: _Claim(7, ('family', 'given', 'maiden', 'middle'), "3de9ef12b4a8"), "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": _Claim(1, ('family', 'suffix'), "570f265a2f46"), + # #451's four replacements for the fields-only catch-all, whose + # own entry recorded the whole 1090-name corpus -- a rule with no + # name_regex reaches everything, so its reach could never move and + # this roster could not see names arrive on it. The numeral and jr + # rules reach more names than they explain, and rules above them + # take the surplus, which is why the four are last in the file; + # the acronym and M.A. rules reach exactly what they explain. + "fix(suffix-routing) a two-token name ending in a roman numeral keeps it in `suffix`": + _Claim(4, ('family', 'suffix'), "fc52089dfa8e"), + "fix(suffix-routing) a two-token name ending in the suffix word jr keeps it in `suffix`": + _Claim(5, ('family', 'suffix'), "602e2d83a23b"), + "fix(suffix-routing) a two-token name ending in a credential acronym keeps it in `suffix`": + _Claim(2, ('family', 'suffix'), "ed72c9672214"), + "fix(suffix-routing) the dotted M.A. spelling reads as a credential (ma-do)": + _Claim(1, ('family', 'suffix'), "17379620526b"), }, "expected_since_2.0.0.toml": { "fix(#335) a marker-led clause leaves the one name word its bare reading": @@ -1895,7 +1972,25 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: ("MD, PHD", ("family", "given", "suffix", "title")): "fix(#296) a credential-only comma string reads a name and " "its postnominal", - ("Smith Jr.", ("family", "suffix")): "fix(suffix-routing)", + ("Smith Jr.", ("family", "suffix")): + "fix(suffix-routing) a two-token name ending in the suffix " + "word jr keeps it in `suffix`", + # #451's two contests with the numeral rule, whose regex reaches + # both of these names from the very end of the file. Only the + # first is decided by order: 'Carod i' diffs {family, suffix}, + # which both rules declare, so nothing but _sorted_rules' + # stability inside the name_regex tier keeps it with fix(#397). + # '田中さん II' diffs {given, suffix}, which the numeral rule's + # `fields` cannot admit at any position. Both are recorded + # because a later edit that moves either rule, or widens the + # numeral rule's `fields`, would take one silently -- exactly the + # absorption #451 exists to end. + ("Carod i", ("family", "suffix")): + "fix(#397) NOT WANTED: a trailing Catalan/Polish linking " + "'i' is read as a generation marker and the family is lost", + ("田中さん II", ("given", "suffix")): + "fix(cjk-glued-honorific-peel) glued honorific peels into " + "suffix", # the glued/spaced boundary. 'Andersonさん' and '김민준씨' left # suffix-routing for a rule that names them; '김민준 씨.' is # spaced and stays on the spaced rule, which #372 taught to @@ -1905,10 +2000,13 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: # SUFFIX_WORDS: 양 is absent from the glued set, so no rule # NAMED for honorifics can claim a suffix diff on a given name # that merely ends in one. The fields-only fix(suffix-routing) - # still would -- measured -- which is unchanged by #372 and is + # still would -- measured -- which was unchanged by #372 and was # the residual cost of having a last-resort tier at all. Being # absorbed by the catch-all is recoverable; being labelled - # 'recognized honorific' by a specific rule is not. + # 'recognized honorific' by a specific rule is not. #451 deleted + # that catch-all, so there is no last-resort tier left in any + # ledger and the residual cost went with it: all four rules that + # replaced it carry a name_regex. ("Andersonさん", ("given", "suffix")): "fix(cjk-glued-honorific-peel)", ("김민준씨", ("family", "given", "suffix")): @@ -1929,12 +2027,16 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: def test_the_recorded_rule_still_wins_each_contested_name() -> None: """Who explains what, which nothing else here asks. - Measured: narrowing fix(cjk-comma-compound)'s script class sends - three of these names to fix(suffix-routing) -- a fields-only - catch-all whose prose is about two-token Latin names -- and the - gate still reports 108 intentional / 0 unexplained. Reach is - per-rule, the total is per-corpus; neither notices a name changing - hands. + Measured when this was written: narrowing fix(cjk-comma-compound)'s + script class sent three of these names to fix(suffix-routing) -- a + fields-only catch-all whose prose was about two-token Latin names -- + and the gate still reported 108 intentional / 0 unexplained. Reach + is per-rule, the total is per-corpus; neither notices a name + changing hands. #451 has since deleted that catch-all, so the same + narrowing would now send those three to UNEXPLAINED and the gate + would say so -- but only because a fields-only rule happened to be + what absorbed them. File order inside the name_regex tier hides a + handover just as completely, which is what the #451 rows below pin. A failure here is not necessarily a regression: it can equally mean a rule was narrowed correctly and its names found a better home. It @@ -2011,12 +2113,19 @@ class _Excluded(NamedTuple): # fix(comma-precomma-family) JOINED this tuple in #372, # it did not replace anything: it claims the {given, # family} readings, which it legitimately describes for a - # Latin comma name, while fix(suffix-routing) still - # claims the readings outside its two fields. The - # exclusion refuses the name before any of the three is - # reached; this records what would happen without it. - ("fix(comma-family)", "fix(comma-precomma-family)", - "fix(suffix-routing)")), + # Latin comma name, while fix(suffix-routing) claimed the + # readings outside its two fields. The exclusion refuses + # the name before either is reached; this records what + # would happen without it. + # + # fix(suffix-routing) LEFT the tuple in #451, which + # deleted the fields-only catch-all of that name: the + # four rules that replaced it are two-token literals and + # none of them reaches a trailing 'Ph. D.'. A tuple that + # SHRANK is the safe direction -- one fewer rule stands + # ready to claim a protected reading -- which is why this + # roster's message warns only about growth. + ("fix(comma-family)", "fix(comma-precomma-family)")), '(^|[\\w.]\\s+)[("\'][^)"\']+[)"\'](\\s+\\w|\\s*$)': # 51 -> 54 as rules.md gained the bracketed Polish examples # (#434): 'Maria Kowalska (z domu Nowak)', 'Maria Kowalska @@ -2129,7 +2238,13 @@ def test_a_fields_narrowing_actually_narrows_something() -> None: recorded pin, which is then asked about every reading rather than the three the key covers, and sees rules claim them. Measured: deleting this entry's `fields` grows its `absorbed_by` from () to - ('fix(suffix-routing)',). + ('fix(comma-family)', 'fix(comma-precomma-family)'). Re-measured + for #451, and the figure it replaces was wrong in both halves: it + read ('fix(suffix-routing)',) where the answer was a three-tuple + with those two comma rules in it, and #451's deletion of the + fields-only fix(suffix-routing) catch-all then took the third + entry away. The point the sentence is making is unchanged -- the + hypothetical is still caught, by more rules than it named. Measured: deleting `fields = ["nickname", "middle"]` from the ASCII-pairs entry passes every other check in this tree. The entry diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 873688b6..a8595a92 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -272,9 +272,14 @@ issue = "fix(comma-family) a comma followed only by titles keeps the given/famil # 'John Smith, Mr.' and 'John Smith, Mr. Jr.': the rules.md#C1 examples # of the rule above (first 'John Smith' -> given 'John', family # 'Smith'; the title stays where it was; with the postnominal, v1's -# first 'Jr.' is the suffix). Without this rule the second lands on -# the fields-only fix(suffix-routing) catch-all below. Its own literal rather than -# an alternation with 'Bob Jones', which would be no vocabulary. +# first 'Jr.' is the suffix). Without this rule the second landed on +# the fields-only fix(suffix-routing) catch-all, back when that +# catch-all declared `given` as well; once #451 narrowed it to +# {family, suffix} a {given, family, suffix} diff stopped being a +# subset of it, and #451 has now deleted it outright. Measured both +# before and after that deletion: with this rule removed the diff is +# UNEXPLAINED. Its own literal rather than an alternation with 'Bob +# Jones', which would be no vocabulary. name_regex = "(?i)^john\\s+smith,\\s*mr\\.?(\\s+jr\\.?)?$" fields = ["given", "family", "suffix"] @@ -397,9 +402,9 @@ issue = "fix(comma-precomma-family) pre-comma run reads as family, not given" # exactly {given, family}. # # Its own rule because these have nothing to do with suffix routing: -# they were falling to the fields-only fix(suffix-routing) below, on a -# rule whose every other name moves a trailing token into `suffix`, -# while these move no suffix at all (#372). +# they were falling to the fields-only fix(suffix-routing) catch-all +# (deleted in #451), on a rule whose every other name moves a trailing +# token into `suffix`, while these move no suffix at all (#372). # # Distinct from fix(comma-family) above by fields, not by order: that # rule is the lone post-comma piece moving into suffix/title and its @@ -604,10 +609,18 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # move a token out of it ('Berg, Jan vd' was suffix 'vd' at every # baseline and is now family 'vd Berg'). Adding `suffix` to claim it # costs three shapes this rule has no business explaining -- -# {family,suffix} taken from fix(suffix-routing), and {middle,suffix} +# {family,suffix} taken from fix(#380) below, and {middle,suffix} # and {middle,family,suffix} taken from UNEXPLAINED, the last two # being diffs in which `family` does not move at all. A rule that # pre-claims shapes it has never seen is the #372 failure mode. +# The first name read fix(suffix-routing) until #451 deleted that +# catch-all, and it was already the wrong name by then: measured, a +# {family, suffix} diff on 'Berg, Jan vd' goes to fix(#380), which +# sits BELOW this rule, so claiming `suffix` here would take it from +# there. On a name this regex reaches that fix(#380)'s does not -- +# 'Vega, Juan de la' -- the same diff went to the catch-all and +# arrives UNEXPLAINED now. Either way the decision below stands; only +# the rule the shape would be taken FROM has changed. # # 'Berg, Jan vd' entered corpus_issues.jsonl and corpus_rules.jsonl # after this was written, and it did NOT arrive unexplained -- the @@ -620,9 +633,13 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # a leading .* this absorbed non-Latin comma names whose diff has # another cause -- a Cyrillic 'Smith, de la' moves {family} because the # particle's RENDERING ORDER inside the family changed, which no rule -# in this ledger names, and it classified here. (It now falls to the -# fields-only catch-all, so the absorption moved rather than vanished; -# worth a narrower rule if such a name ever appears.) The class is the +# in this ledger names, and it classified here. (No such name is in +# any corpus, so this is what WOULD happen. It would now arrive +# UNEXPLAINED and block the gate until someone read it, which is the +# right outcome: this ledger has no fields-only rule to absorb it any +# more. The claim that it "falls to the fields-only catch-all" was +# true until #451 deleted that rule, and this file now says what its +# 2.0.0 and 2.1.0 copies have said since 8237b54.) The class is the # neighbours' and compare.py's own: U+0000-U+024F, which stops at # Latin Extended-B, so a diacritic-bearing 'Nguyen, Van de la' in NFD # or Latin Extended Additional is outside it. @@ -674,41 +691,6 @@ issue = "fix(#380) a trailing vd after a family comma is the tussenvoegsel, not name_regex = "(?i)^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*\\svd\\s*$" fields = ["family", "suffix"] -[[change]] -issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" -# 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone trailing suffix -# to family/first (no comma present); 2.0 keeps recognized suffixes in -# `suffix`. -# -# STILL THE ONLY RULE IN ANY LEDGER WITH NO name_regex, and #451 is -# retiring it. Read that issue before widening anything here. What is -# left on it after the named rules #451 has added so far: names in -# several behaviour families, all moving {family, suffix} and no more, -# which is what this `fields` now says. It declared `given` as well -# until #451 measured that no name it explains moves that role -- the -# fix(#424) shape decisions.md#H1 records, a rule matching by superset -# while the diff beneath it shrinks, with nothing able to see it. -# -# The same surgery has been done to it twice. It claimed 25 until -# #372, on nothing but being the only fields-only rule in any ledger: -# it sorts after every name_regex rule, so it takes whatever nothing -# narrower named. #372 gave 21 of those their own rules -- -# fix(cjk-glued-honorific-peel) and the widened -# fix(cjk-honorific-suffix) below for #312's honorifics, -# fix(comma-precomma-family) above for the three names that move no -# suffix at all -- and #451 is doing the rest. Those figures are -# history and stay written; a count of what the rule explains NOW is -# not, because nothing recomputes it. The last one read "Four corpus -# names, and the prose fits all four" while the rule explained -# fourteen. Run tools/differential/compare.py --baseline 1.4.0 for -# the live number. -# -# Being last is the point of this rule and worth keeping until it is -# gone. What is not worth keeping is it being the only explanation a -# name ever gets: when it grows, the question is which narrower rule -# is missing. -fields = ["family", "suffix"] - [[change]] issue = "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost" # 'Aishwarya Rai': the tree reads given 'Aishwarya', family '', suffix @@ -1011,8 +993,8 @@ issue = "fix(cjk-glued-honorific-peel) glued honorific peels into suffix" # '김민준씨' -> 김/민준/씨, '田中さん' -> 田中/さん, 'Andersonさん' -> # Anderson/さん: #312's peel, on a name with no space before the # honorific and no comma anywhere. 17 corpus names, which fell to the -# fields-only fix(suffix-routing) above until #372 gave them a rule -# that names them. +# fields-only fix(suffix-routing) catch-all (deleted in #451) until +# #372 gave them a rule that names them. # # The regex is this file's first hand copy of GLUED_HONORIFICS rather # than SUFFIX_WORDS, and that narrower vocabulary is the whole reason @@ -1046,8 +1028,9 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # honorifics as suffix vocabulary, so a trailing 先生/씨/様 moves to # `suffix` where 1.4 read it as a name part -- and the remaining # wholly-CJK name simultaneously takes the family-first flip, so a -# three-token name moves `middle` too, past the fields-only -# suffix-routing rule. Keyed on the honorific alternation itself +# three-token name moves `middle` too, which is what carried it past +# the fields-only suffix-routing rule while that rule existed (#451 +# deleted it). Keyed on the honorific alternation itself # (the maiden-marker rule's pattern): the trigger tokens are wholly # CJK characters, so the rule matches only a name whose TRAILING # token is a listed honorific -- a mostly-Latin name with one @@ -1064,13 +1047,16 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # GLUED diffs (田中さん, 김민준씨, and the 김씨 that segmentation # already reached) never match this rule -- the anchor is judged on # the name STRING, and a glued form has no space before its -# honorific. They fall to the fields-only suffix-routing rule by +# honorific. They fell to the fields-only suffix-routing rule by # their {first, last, suffix} shape (v1 held the whole token in -# first, so last moves too), which is field-honest even though that -# rule's prose describes the two-token Latin case. Verified by the +# first, so last moves too) -- field-honest, even though that rule's +# prose described the two-token Latin case. Verified by the # 2026-07-30 run, not assumed: #308's peel put a dozen glued names # in the corpus (Latin+glued like Andersonさん included) and every -# one classified there -- none carried a middle-field diff. +# one classified there -- none carried a middle-field diff. They have +# fix(cjk-glued-honorific-peel) above since #372, and the catch-all +# they were falling to is gone (#451), so the sentence is history: +# nothing absorbs a glued diff this rule declines any more. name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)\\.?$" fields = ["given", "middle", "family", "suffix"] @@ -1179,8 +1165,10 @@ issue = "fix(#369) a given-name title licenses the bound given-name join with on # reads given 'Abu Bakar' again with every field byte-identical to # this baseline, so no rule names its fields: restored, not moved. # -# Without this rule the name lands on the fields-only -# fix(suffix-routing) catch-all below, which moves no suffix here. +# Without this rule the name landed on the fields-only +# fix(suffix-routing) catch-all, which moved no suffix here. #451 +# deleted that catch-all, so this ledger has no fields-only rule left +# to absorb anything. name_regex = "(?i)^sheik\\s+abdul\\b" fields = ["given", "family"] @@ -1197,10 +1185,13 @@ issue = "fix(#401) the bound-given reserve counts the trailing numeral assign re # [[never]] shape, and its v1 reading carries the same family loss # that entry would then refuse to explain. # -# A rule with fewer fields than this diff silently hands the name to -# the fields-only fix(suffix-routing) catch-all below, which is how -# this fix was first recorded as "1.4.0 parity": the catch-all's -# printout is capped, and EXPLAINED NOTHING is not byte-identical. +# A rule with fewer fields than this diff silently handed the name to +# the fields-only fix(suffix-routing) catch-all, which is how this fix +# was first recorded as "1.4.0 parity": the catch-all's printout was +# capped, and EXPLAINED NOTHING is not byte-identical. #451 deleted +# that catch-all, so a narrowed rule here now costs the name its +# explanation outright instead of hiding it -- louder, and the reason +# the lesson is worth keeping written down. name_regex = "(?i)^abdul\\s+smith\\s+(jr\\s+)?v$" fields = ["given", "family", "suffix"] @@ -1270,9 +1261,10 @@ issue = "fix(#424) accepted: the chain keeps an acronym assign will not peel beh # and the acronym the fork counted with three pieces meets assign # with two; the chain asks the peel again and takes it. first 'von # Berg', last 'MA' -> family 'von Berg MA' (master's reading since -# 2.0). A rules.md example. Without this rule the name lands on the -# fields-only fix(suffix-routing) catch-all below -- the count is the -# tell. +# 2.0). A rules.md example. Without this rule the name landed on the +# fields-only fix(suffix-routing) catch-all -- the count was the tell. +# #451 deleted that catch-all, so this ledger has no fields-only rule +# left to absorb anything. name_regex = "(?i)^freiherr\\s+von\\s+berg\\s+ma$" fields = ["given", "family"] @@ -1316,8 +1308,10 @@ issue = "fix(#424) the particle chain stops before the trailing numeral" # as 'John Smith V' reads. Shipped since 1.x: the chain's stop asked # with the suffix-piece test, whose initial veto does not see a bare # V. A rules.md example; no differential corpus name has the shape. -# Without this rule the name lands on the fields-only -# fix(suffix-routing) catch-all below -- the count is the tell. +# Without this rule the name landed on the fields-only +# fix(suffix-routing) catch-all -- the count was the tell. #451 +# deleted that catch-all, so this ledger has no fields-only rule left +# to absorb anything. name_regex = "(?i)^john\\s+van\\s+der\\s+berg\\s+v$" fields = ["family", "suffix"] @@ -1414,8 +1408,10 @@ issue = "fix(#424) an unlisted abbreviation is as transparent as a listed title # H2's abbreviations; the scan asks assign's test now. Literal for the # reason the #367 rule gives: a regex cannot ask whether a word is a # title, and this one cannot ask whether it is unlisted. -# Without this rule the name lands on the fields-only -# fix(suffix-routing) catch-all below -- the count is the tell. +# Without this rule the name landed on the fields-only +# fix(suffix-routing) catch-all -- the count was the tell. #451 +# deleted that catch-all, so this ledger has no fields-only rule left +# to absorb anything. name_regex = "(?i)^esq\\.\\s+van\\s+gogh$" fields = ["given", "family"] @@ -1441,9 +1437,9 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # Three joined through #413's backtick harvest: 'Dr. Van Johnson', # 'Sir Van Johnson', 'Jr. Van Johnson'. Widened here as well as in the # 2.0 and 2.1 copies -- widening only those left these three landing on -# the fields-only fix(suffix-routing) below, which is the very -# absorption this comment describes two paragraphs up. None of them is -# a two-token name and none moves a suffix. +# the fields-only fix(suffix-routing) catch-all (deleted in #451), +# which is the very absorption this comment describes two paragraphs +# up. None of them is a two-token name and none moves a suffix. # # The title slot copies TITLES: all four spellings are in it. `jr` is # also suffix vocabulary and reads as a title here simply because it @@ -1479,12 +1475,14 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # rather than be absorbed here. # # This rule has to exist even though the run already exited 0 without -# it. The diff was being classified by fix(suffix-routing) below, a -# fields-only rule whose ["given", "family", "suffix"] is a superset of -# this diff's ["given", "family"] -- a rule about a trailing credential -# absorbing a leading-particle change, which is the mis-classification -# the README warns about. A name_regex rule outranks every fields-only -# one, so this claims it back. +# it. The diff was being classified by fix(suffix-routing), a +# fields-only rule whose ["given", "family", "suffix"] was a superset +# of this diff's ["given", "family"] -- a rule about a trailing +# credential absorbing a leading-particle change, which is the +# mis-classification the README warns about. A name_regex rule +# outranks every fields-only one, so this claimed it back; #451 has +# since deleted the catch-all, and no ledger has a fields-only rule +# for anything to be claimed back from. name_regex = "(?i)^(mr|dr|sir)\\.?\\s+van\\b" fields = ["given", "middle", "family"] @@ -1703,10 +1701,11 @@ issue = "fix(#410) a title and one name word name the family, whatever annotatio # every future given/family swap in this corpus. # # Its own rule because these have nothing to do with suffix routing: -# they were falling to the fields-only fix(suffix-routing) below, on a -# rule whose every other name moves a trailing token into `suffix`, -# while these move no suffix at all. Same surgery #372 did on the same -# catch-all, whose heading returned to 14 once this rule took them. +# they were falling to the fields-only fix(suffix-routing) catch-all +# (deleted in #451), on a rule whose every other name moves a trailing +# token into `suffix`, while these move no suffix at all. Same surgery +# #372 did on the same catch-all, whose heading returned to 14 once +# this rule took them. # # THREE names here, where the 2.0.0 and 2.1.0 ledgers classify four: # at this baseline 'Dr. Smith née Jones' also moves `middle` and @@ -1838,14 +1837,16 @@ issue = "fix(#335) a marker-led clause leaves the one name word its bare reading # the one shape in #445's class the gates do NOT report, which is # why nothing but reading the ledger could catch it. # -# What the narrowing buys differs by ledger, so do not read more into -# it here than it gives: at 2.0.0 and 2.1.0 a {given, family} diff on -# this name is now UNEXPLAINED, while at THIS baseline the fields-only -# fix(suffix-routing) rule below still matches every name with -# {given, family, suffix} and would take it. That catch-all is -# long-standing and out of #445's scope; the narrowing removes this -# rule from the path regardless, which is the part that was in -# scope. +# What the narrowing buys was said here to differ by ledger, and it +# did not. The sentence claimed the fields-only fix(suffix-routing) +# rule that then stood below "still matches every name with {given, +# family, suffix} and would take it"; that stopped being true when the +# same #451 bundle narrowed the catch-all to {family, suffix}, since a +# {given, family} diff is not a subset of those two. Measured against the +# ledger as it stood then, this name's {given, family} diff was +# already UNEXPLAINED at THIS baseline, exactly as at 2.0.0 and +# 2.1.0. #451 has since deleted the catch-all outright, and the +# measurement is unchanged. name_regex = "(?i)^Smith \\(n[ée]e Jones\\)$" fields = ["nickname", "maiden"] @@ -1894,13 +1895,12 @@ issue = "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in # '', suffix 'Jr.'. decisions.md#N3. # # Its own rule because it is three tokens -- a quoted nickname, one -# bare name word, one suffix word -- and #451 is retiring the -# fields-only fix(suffix-routing) catch-all above by giving its names -# narrower rules; this name was one of that catch-all's nine at the -# start of this task. No two-token suffix rule exists in this ledger -# yet to hold it either way: #451 is landing those in a later task, -# and this shape -- three tokens, a leading nickname -- would not fit -# one regardless. +# bare name word, one suffix word -- and #451 retired the fields-only +# fix(suffix-routing) catch-all by giving its names narrower +# rules; this name was one of that catch-all's nine when that work +# started. The two-token suffix rules #451 owed are now the four +# below, and this shape -- three tokens, a leading nickname -- fits +# none of them, which is why it kept a rule of its own. # # Note what is NOT settled here: decisions.md#H1 records that N3 and # H1 count a trailing suffix in opposite directions -- H1's @@ -1912,3 +1912,191 @@ issue = "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in # measured above; it does not endorse either reading. name_regex = "(?i)^['\"‘“][^'\"’”]+['\"’”]\\s+\\S+\\s+(jr|sr|ii|iii|iv|v)\\.?$" fields = ["family", "suffix"] + +# The four rules below are what is left of the fields-only +# fix(suffix-routing) catch-all, which stood between fix(#380) and +# fix(#342) above until #451 deleted it. It was the only rule in any +# ledger with no name_regex, so it sorted after every other rule and +# took whatever nothing narrower named -- 25 names when #372 first cut +# it back, six by the time this task began. Those six are the six the +# four rules below explain, one behaviour family each. The numeral and +# jr rules reach more corpus names than they explain -- the surplus is +# claimed by rules above, or does not diff at all, and each of those +# two comments names its own. _CORPUS_CLAIMS holds the reaches; do not +# restate them here, because nothing recomputes a number in a comment. +# +# They sit at the END of the file on purpose. The catch-all sorted last +# because it had no name_regex; with regexes these join the regex tier, +# where _sorted_rules is STABLE and FILE ORDER decides ties. Placing +# them last reproduces the catch-all's position exactly: every rule +# above keeps every contest it already won. The numeral rule's regex +# reaches 'Carod i' and '田中さん II', and _CROSS_RULE_WINNERS pins +# both -- but only the first is a real contest: measured by moving the +# numeral rule to the front of the file, 'Carod i' changes hands and +# '田中さん II' does not, because its diff moves {given, suffix} and +# these four declare {family, suffix}. File order decides one, the +# subset test the other, and the roster does not care which. + +[[change]] +issue = "fix(suffix-routing) a two-token name ending in a roman numeral keeps it in `suffix`" +# 'John V' and 'Mohamad X': v1 routed a lone trailing suffix to +# family/first (no comma is present in either), and 2.x reads the +# trailing numeral as the suffix. Whether 1.4 itself read that token as +# `family` is outside what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 release +# from a cached environment); the diff this rule explains at this +# baseline moves exactly {family, suffix}, which is consistent with +# that reading and is what the fields below claim. +# +# The numeral does not reach `suffix` through a wordlist. It reaches it +# through _vocab.is_trailing_numeral_suffix -- "assign's roman-numeral +# fork, shared with group's bound-given reserve (#401): a FINAL +# single-token piece that is a roman numeral reads as the suffix when +# the piece before it does not look like part of an initial run". That +# is a PATTERN test plus a layout test, and no vocabulary lookup at +# all: 'x' is in none of SUFFIX_WORDS, SUFFIX_ACRONYMS or their +# ambiguous subsets, and 'Mohamad X' moves anyway. Which is why this +# alternation is declared in _NOT_A_VOCABULARY_COPY rather than in +# _LATIN_ALTERNATION_SOURCES -- there is no vocabulary here to drift +# from. +# +# The alternation is _ROMAN's own body, r"^(X|IX|IV|V?I{0,3})$" in +# nameparser/_pipeline/_vocab.py, with the EMPTY alternative removed: +# I{0,3} becomes I{1,3}, and the bare V that change would lose is +# spelled out beside it. Measured over every string of length 1 to 4 +# drawn from IVXivx, the two patterns accept exactly the same strings +# with one exception, the empty one, which _ROMAN accepts and this does +# not. Dropping it is deliberate: behind "^\S+\s+" and before "$" an +# empty alternative would match a name ending in whitespace. No corpus +# name does, so this is a latent widening removed rather than a bug +# fixed -- and it is the reason the two patterns are NOT byte-identical +# copies of one another, which a reader comparing them should know +# before "fixing" the difference. +# +# 'v' is in SUFFIX_WORDS as well, and the jr rule below deliberately +# does not spell it: the numeral fork is what actually moves it, and +# two rules claiming one token would leave file order to decide which +# explains it. +# +# The regex also reaches two corpus names this rule does not explain. +# 'Carod i' is fix(#397)'s NOT-WANTED rule above -- a real contest, +# same fields and both regexes matching, decided by nothing but file +# order -- and '田中さん II' is fix(cjk-glued-honorific-peel)'s, where +# the diff moves {given, suffix} and this rule's `fields` cannot admit +# it whatever the order. _CROSS_RULE_WINNERS pins both. That reach is +# real, so neither is a _MUST_NOT_MATCH probe here: that roster tests +# the regex, not classify(). +name_regex = "(?i)^\\S+\\s+(X|IX|IV|V?I{1,3}|V)$" +fields = ["family", "suffix"] + +[[change]] +issue = "fix(suffix-routing) a two-token name ending in the suffix word jr keeps it in `suffix`" +# 'Smith Jr.': v1 routed a lone trailing suffix to family/first (no +# comma present) and 2.x keeps a recognized suffix WORD in `suffix`. +# Whether 1.4 itself read that token as `family` is outside what this +# ledger's worker can check directly (tools/differential/README.md's +# warning against reading a v1 release from a cached environment); the +# diff this rule explains at this baseline moves exactly {family, +# suffix}, which is consistent with that reading and is what the fields +# below claim. +# +# 'jr' is an entry in SUFFIX_WORDS (nameparser/config/suffixes.py), so +# unlike the numeral above this one really is a vocabulary claim. It is +# written as a single LITERAL rather than as a one-member alternation, +# which is what keeps it off _LATIN_ALTERNATION_SOURCES: that roster is +# discovered from alternation syntax, and a rule with no alternation +# owes it nothing. A second suffix word needing this shape should +# become an alternation and be enrolled against SUFFIX_WORDS then, not +# be smuggled in as another literal. +# +# The regex also reaches four corpus names this rule does not explain. +# 'Kim, Jr.' and 'Smith, Jr.' are fix(#296)'s lone post-comma +# credential and '김민준씨 Jr.' is fix(cjk-glued-honorific-peel)'s; all +# three rules sit above this one and win on file order, which is the +# whole reason these four rules were appended at the end. The fourth, +# 'Doe,, Jr.', does not diff against this baseline at all, so it has no +# winner to lose. That reach is real either way, so none of the four is +# a _MUST_NOT_MATCH probe here: that roster tests the regex, not +# classify(). +name_regex = "(?i)^\\S+\\s+jr\\.?$" +fields = ["family", "suffix"] + +[[change]] +issue = "fix(suffix-routing) a two-token name ending in a credential acronym keeps it in `suffix`" +# 'Donald mc' and 'QC MP': v1 read the trailing token as a name part +# and 2.x reads it as the postnominal it is -- 'QC MP' is two +# credentials and no name at all. Whether 1.4 itself read that token as +# `family` is outside what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 release +# from a cached environment); the diff this rule explains at this +# baseline moves exactly {family, suffix}, which is consistent with +# that reading and is what the fields below claim. +# +# Both members are entries in SUFFIX_ACRONYMS, which is a DIFFERENT +# vocabulary from the jr rule's SUFFIX_WORDS above, and that is why +# these are two rules rather than one. _LatinCopy carries exactly one +# `vocabulary`, and +# test_latin_alternations_mean_something_the_vocabulary_ships asserts +# exactly one roster key per alternation, so an alternation spanning +# both sources could not be pinned against either. The split is forced +# by the guard, not chosen for taste. +# +# 'mc' deserves its own paragraph, because it is in SUFFIX_ACRONYMS and +# in PARTICLES both -- the collision decisions.md#vocabulary-collisions +# C-ii settled for 'vd' ("the Dutch van der reading, as the more +# common"), and which its own C-ii note records that 'do' and 'mc' +# inherit "without being weighed". That decision reaches a written name +# only through rules.md#P6's attachment, and P6 is scoped: "the COMMA +# form only, deliberately. [...] Without the comma the written shape is +# not settled [...] Those keep their positional reading and are not +# tracked as deviations." 'Donald mc' has no comma, so it is outside +# P6, and S2's credential reading is what stands -- which is what this +# rule classifies, and it is the positional reading P6's scope note +# leaves in place rather than a decision this rule makes. +# +# The two neighbouring shapes both already have rules above, and the +# contrast is the point: 'Mc Donald' has the particle LEADING and folds +# into the family (fix(#360), #360), and 'Berg, Jan vd' has it trailing +# WITH a comma, so P6 attaches it to the family (fix(#380), #380). +# Trailing and comma-less is the third corner, and it is this one. +name_regex = "(?i)^\\S+\\s+(mc|mp)$" +fields = ["family", "suffix"] + +[[change]] +issue = "fix(suffix-routing) the dotted M.A. spelling reads as a credential (ma-do)" +# 'Jack M.A.': v1 read the trailing token as a name part and 2.x reads +# it as a credential. Whether 1.4 itself read that token as `family` is +# outside what this ledger's worker can check directly +# (tools/differential/README.md's warning against reading a v1 release +# from a cached environment); the diff this rule explains at this +# baseline moves exactly {family, suffix}, which is consistent with +# that reading and is what the fields below claim. +# +# This is a DECIDED reading, unlike the fix(#342) and fix(#397) rules +# above, which classify a diff because its cause is known and not +# because the reading is wanted. decisions.md#ma-do decides it: "ma and +# do joined the ambiguous acronym set because both are common surnames; +# the two-word 'Jack Ma' is kept intact by S2's words-to-spare guard, +# while the periods gate governs the dotted spellings ('M.A.' counts +# unambiguously)". The bare and dotted spellings of one name therefore +# read differently ON PURPOSE, which is why both spellings of the bare +# one are _MUST_NOT_MATCH probes here. +# +# Literal-anchored, and it could not be anything else. Measured: the +# member `m\.?a\.?` matches the fragment 'M.A.' in the corpus, and +# _normalize leaves that as 'm.a', which is not a SUFFIX_ACRONYMS entry +# -- so test_latin_alternations_mean_something_the_vocabulary_ships +# rejects it as an alternation member ("a fragment a member matches has +# to BE a vocabulary entry"). The ambiguous-surname-acronym rule above +# records the same finding and answers it the same way: keep the +# periods out of the members. There is no alternation here at all, so +# nothing is owed to _LATIN_ALTERNATION_SOURCES. +# +# Anchored on 'jack' as well as on the spelling: 'John Smith M.A.' is +# also a corpus name (it arrived with corpus_rules.jsonl, #414), it is +# three tokens rather than two, and it does not diff against this +# baseline at all -- so a rule keyed on the dotted spelling alone would +# stand ready to explain a future regression on it. It is a +# _MUST_NOT_MATCH probe for that reason. +name_regex = "(?i)^jack\\s+m\\.a\\.$" +fields = ["family", "suffix"] From e0ac1175518032c621d26f6c4003d325a593d942 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 23:05:27 -0700 Subject: [PATCH 10/15] fix(differential): reject a rule with fields and no name_regex (#451) Tasks 1-9 gave all fourteen names the #451 catch-all had absorbed their own rules and deleted it. This task closes the gap that let it happen: validate_rules now rejects any rule that has `fields` but no `name_regex`, so the shape cannot come back. A rule with no name narrowing claims every name whose diff fits its `fields` -- a very common shape -- and no guard could see it grow: _CORPUS_CLAIMS records a regexless rule's reach as the WHOLE CORPUS, so it sits at its maximum from the moment it is written and arrivals never move it. The one rule ever shaped this way grew from 4 explained names to 14, across six unrelated behaviour families, with every guard green and the gate reporting 0 unexplained throughout. This is a rejection of the SHAPE, not a specificity floor on role count -- that was proposed and declined in #372/#373 as vacuous, since the one fields-only rule named 3 of 7 roles and no floor rejecting it matches anything else. The check costs no migration: no rule in any shipped ledger (1.4.0, 2.0.0, 2.1.0) has this shape. Several existing tests constructed a fields-only rule as a shorthand to exercise something else -- the deeper `fields` validation, the `dormant` key, `_ambiguities` as a legal field name, or main()'s sort/dormancy reporting through a synthetic ledger. Each is updated to carry a `name_regex` so it keeps testing what it was written to test. One of them, test_main_sorts_a_name_regex_rule_ahead_of_a_fields_only_ one, pinned a scenario -- a broad fields-only rule shadowed by a specific name_regex rule -- that this check makes impossible to construct as a valid ledger; it is repurposed to prove main() rejects that shape instead, since the underlying sort mechanism stays covered directly by _sorted_rules' own tests. Verified: the two new tests (rejects the shape, accepts both neighbouring shapes), all three shipped ledgers still load and gate green (1.4.0 229/0, 2.0.0, 2.1.0), and the full tests/v2/ suite passes. Closes #451 Co-Authored-By: Claude Opus 5 --- tests/v2/test_differential.py | 110 +++++++++++++++++++++++----------- tools/differential/compare.py | 42 +++++++++++++ 2 files changed, 118 insertions(+), 34 deletions(-) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index 501ecb27..058608e1 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -212,6 +212,34 @@ def test_malformed_rule_error_names_the_ledger_it_came_from() -> None: compare.validate_rules([{}], "expected_since_1.4.0.toml") +def test_a_rule_with_fields_and_no_regex_is_rejected() -> None: + """The shape #451 retired, closed so it cannot return. + + A rule with no `name_regex` claims every name whose diff fits its + `fields`, and _CORPUS_CLAIMS records its reach as the whole corpus + -- already at maximum, so arrivals never move it. The one rule with + this shape grew from 4 explained names to 14 across six behavior + families with every guard green. + """ + with pytest.raises(SystemExit, match="no 'name_regex'"): + compare.validate_rules( + [{"issue": "fix(x) a rule with no name narrowing", + "fields": ["given", "family", "suffix"]}], + "test_ledger.toml") + + +def test_a_rule_with_a_regex_and_no_fields_or_both_stays_legal() -> None: + """The neighbouring shapes #451 did NOT retire. `name_regex` alone + still narrows by name; `name_regex` plus `fields` narrows by both. + Only the fields-only shape -- no name narrowing at all -- is new + to reject.""" + compare.validate_rules( + [{"issue": "x", "name_regex": "Smith"}], "test_ledger.toml") + compare.validate_rules( + [{"issue": "x", "name_regex": "Smith", "fields": ["given"]}], + "test_ledger.toml") + + def test_classify_declines_a_diff_touching_a_field_the_rule_omits() -> None: """The subset check is the tightness mechanism of every `fields` rule -- a rule claims a diff only when EVERY changed field is one it @@ -266,19 +294,24 @@ def test_v2_fields_matches_the_Role_enum() -> None: ({"issue": "x", "name_regex": r"\b"}, "matches every one of"), ({"issue": "x", "name_regex": r"[\s\S]"}, "matches every one of"), # seven roles without _ambiguities: below baseline 2.0 that IS the - # whole vocabulary, so it claims every diff - ({"issue": "x", "fields": ["title", "given", "middle", "family", - "suffix", "nickname", "maiden"]}, + # whole vocabulary, so it claims every diff. name_regex is along + # for the ride so this pins the roles check, not the #451 one. + ({"issue": "x", "name_regex": "Smith", + "fields": ["title", "given", "middle", "family", + "suffix", "nickname", "maiden"]}, "all seven roles"), # uncompilable: without this it raises mid-run, after the worker ({"issue": "x", "name_regex": "Smith("}, "invalid 'name_regex'"), - ({"issue": "x", "fields": []}, "empty 'fields'"), - ({"issue": "x", "fields": ["famly"]}, "not roles"), + ({"issue": "x", "name_regex": "Smith", "fields": []}, "empty 'fields'"), + ({"issue": "x", "name_regex": "Smith", "fields": ["famly"]}, + "not roles"), # facade vocabulary is not role vocabulary; it would never match - ({"issue": "x", "fields": ["first"]}, "not roles"), - ({"issue": "x", "fields": ["title", "given", "middle", "family", - "suffix", "nickname", "maiden", - "_ambiguities"]}, "all seven roles"), + ({"issue": "x", "name_regex": "Smith", "fields": ["first"]}, + "not roles"), + ({"issue": "x", "name_regex": "Smith", + "fields": ["title", "given", "middle", "family", + "suffix", "nickname", "maiden", + "_ambiguities"]}, "all seven roles"), ({"issue": "x", "fields": ["given"], "dormant": ""}, "not a non-empty"), ({"issue": "x", "fields": ["given"], "dormant": True}, "not a non-empty"), # widening _RULE_KEYS is exactly the edit that could let a near-miss @@ -353,7 +386,8 @@ def test_ambiguities_is_a_legal_field_name() -> None: this pseudo-field is the only name that can classify it -- and the 2.0 ledger's first rule depends on it.""" compare.validate_rules( - [{"issue": "x", "fields": ["_ambiguities"]}], "ledger.toml") + [{"issue": "x", "name_regex": "Smith", "fields": ["_ambiguities"]}], + "ledger.toml") #: What _run_worker was asked for, so a test can prove main forwarded @@ -454,7 +488,8 @@ def test_main_exits_0_when_every_diff_is_claimed( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: code, out = _run_main( tmp_path, monkeypatch, - '[[change]]\nissue = "claimed"\nfields = ["family"]\n', _DIFFERS) + '[[change]]\nissue = "claimed"\nname_regex = "Smith"\n' + 'fields = ["family"]\n', _DIFFERS) assert code == 0 assert "UNEXPLAINED" not in out assert "## claimed (1)" in out @@ -470,26 +505,28 @@ def test_main_validates_the_ledger_before_running_anything( '[[change]]\nissue = "wide"\nname_regex = ""\n', _DIFFERS) -def test_main_sorts_a_name_regex_rule_ahead_of_a_fields_only_one( +def test_main_rejects_a_broad_fields_only_rule_before_running_anything( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A broad fields-only rule written FIRST must not claim a diff the - specific name_regex rule below it owns. Deleting main's - _sorted_rules call leaves _sorted_rules' own test passing. - - 'broad' is declared dormant because in THIS fixture -- one corpus - name, always won by 'specific' -- it is permanently shadowed by - construction, which is exactly the case main()'s dormancy report - (#372) now calls out. Without the declaration this test would be - pinning main's sort order and main's dormancy report at once, and a - failure could not tell which one broke. + """This used to pin main's _sorted_rules call: a broad fields-only + rule written FIRST could not claim a diff a specific name_regex + rule below it owned, because _sorted_rules puts every name_regex + rule ahead of every fields-only one. That scenario is retired by + #451 -- a fields-only rule is rejected at validate_rules, so main() + never reaches _sorted_rules with one. What is left to pin here is + that main() still validates before running, for this shape too, + the same composition fact test_main_validates_the_ledger_before_ + running_anything covers for a different malformed shape. The + tier-sort mechanism itself stays pinned directly by + test_name_regex_rules_sort_ahead_of_fields_only_rules and + test_rule_sort_is_stable_within_a_tier, which call _sorted_rules + without going through validate_rules. """ - _, out = _run_main( - tmp_path, monkeypatch, - '[[change]]\nissue = "broad"\nfields = ["family"]\n' - 'dormant = "always shadowed by \'specific\' below, by construction ' - 'of this fixture"\n' - '[[change]]\nissue = "specific"\nname_regex = "Smith"\n', _DIFFERS) - assert "## specific (1)" in out and "broad" not in out + with pytest.raises(SystemExit, match="no 'name_regex'"): + _run_main( + tmp_path, monkeypatch, + '[[change]]\nissue = "broad"\nfields = ["family"]\n' + '[[change]]\nissue = "specific"\nname_regex = "Smith"\n', + _DIFFERS) def test_main_exits_1_and_names_a_rule_that_explained_nothing( @@ -503,7 +540,8 @@ def test_main_exits_1_and_names_a_rule_that_explained_nothing( """ code, out = _run_main( tmp_path, monkeypatch, - '[[change]]\nissue = "explains-it"\nfields = ["family"]\n' + '[[change]]\nissue = "explains-it"\nname_regex = "Smith"\n' + 'fields = ["family"]\n' '[[change]]\nissue = "idle"\nname_regex = "ZZNOSUCHNAME"\n' 'fields = ["family"]\n', _DIFFERS) assert code == 1 @@ -529,7 +567,8 @@ def test_main_only_feeds_diffing_names_to_the_dormancy_check( """ code, out = _run_main( tmp_path, monkeypatch, - '[[change]]\nissue = "explains-it"\nfields = ["family"]\n' + '[[change]]\nissue = "explains-it"\nname_regex = "Smith"\n' + 'fields = ["family"]\n' '[[change]]\nissue = "idle"\nname_regex = "Jones"\n' 'fields = ["family"]\n', _DIFFERS, extra=[("Alice Jones", @@ -551,7 +590,8 @@ def test_main_exits_1_when_a_declared_dormant_rule_wakes_up( a false statement in the ledger, so it fails the run too.""" code, out = _run_main( tmp_path, monkeypatch, - '[[change]]\nissue = "awake"\nfields = ["family"]\n' + '[[change]]\nissue = "awake"\nname_regex = "Smith"\n' + 'fields = ["family"]\n' 'dormant = "claims to be idle, but explains the only diff here"\n', _DIFFERS) assert code == 1 @@ -735,7 +775,8 @@ def test_main_claims_an_ambiguity_only_diff_when_a_rule_names_it( v2 = {**_SAME_V2, "_ambiguities": ["SEGMENTATION"]} code, out = _run_main( tmp_path, monkeypatch, - '[[change]]\nissue = "seg"\nfields = ["_ambiguities"]\n', + '[[change]]\nissue = "seg"\nname_regex = "Smith"\n' + 'fields = ["_ambiguities"]\n', _SAME_FACADE, baseline="2.0.0", baseline_v2=v2) assert code == 0 and "## seg (1)" in out @@ -951,7 +992,8 @@ def test_validate_rules_accepts_a_declared_dormant_rule() -> None: """`dormant` is a legal key, so a rule that declares one is not rejected as a misspelling.""" compare.validate_rules( - [{"issue": "x", "fields": ["given"], "dormant": "no corpus name"}], + [{"issue": "x", "name_regex": "ZZNOSUCHNAME", "fields": ["given"], + "dormant": "no corpus name"}], "expected_since_1.4.0.toml") diff --git a/tools/differential/compare.py b/tools/differential/compare.py index f065906b..741ff179 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -112,6 +112,21 @@ def _sorted_rules(rules: list[dict[str, object]]) -> list[dict[str, object]]: one tier and the order they are written in settles every tie among them -- three names match both honorific rules and are labelled by whichever comes first. + + SINCE #451 THAT IS EVERY LEDGER, and this function is the identity + on all of them. validate_rules now rejects a rule carrying `fields` + and no `name_regex`, which is the only shape that could occupy the + second tier -- so no ledger that loads can reach it, and file order + settles every tie there is. Verified against all three shipped + ledgers at the time of writing. + + Kept rather than deleted, deliberately. It is the defence that + makes the ban SAFE to state: a ledger read by anything that does + not call validate_rules first -- a future tool, a REPL session, a + test fixture -- still gets the tier ordering rather than silently + letting a regexless rule shadow the file. Deleting it would move + the guarantee from the code into a convention, which is the trade + #451 was filed to undo. """ return sorted(rules, key=lambda r: not isinstance(r.get("name_regex"), str)) @@ -375,6 +390,21 @@ def validate_rules(rules: list[dict[str, object]], ledger: str) -> None: buy a precise message rather than safety; they are here because the family is easier to reason about whole than split by direction. + The `fields`-without-`name_regex` check belongs to the dangerous + direction, and it is the family's sharpest example: no other + malformed shape can widen invisibly. #451 is the rule that lived + it -- no name narrowing, so it claimed every name whose diff fit + its `fields`, and _CORPUS_CLAIMS (the guard tracking each rule's + reach) recorded that reach as the whole corpus from the start, so + growth could never trip it. It grew from 4 explained names to 14 + across six unrelated behavior families before anyone noticed, with + every guard green throughout. A specificity FLOOR on role count was + proposed and declined as vacuous (#372/#373): that one rule named + 3 of 7 roles, and no floor rejecting it matches anything else. This + check rejects the SHAPE instead -- no name narrowing at all -- which + is a different proposal and costs no migration, since no rule in + any shipped ledger has it. + The TYPE and VALUE checks are the quiet ones, not the presence check: `classify` skips a `name_regex` that is not a str and a `fields` that is not a list, so a mistyped or misspelled key does @@ -435,6 +465,18 @@ def validate_rules(rules: list[dict[str, object]], ledger: str) -> None: raise SystemExit( f"{where} has neither 'name_regex' nor 'fields' -- it " f"would match every diff and shadow every later rule") + if has_fields and not has_regex: + raise SystemExit( + f"{where} has 'fields' but no 'name_regex' (#451). With " + f"no name narrowing, the rule claims every name whose " + f"diff fits its 'fields' -- and no guard can see that " + f"grow: _CORPUS_CLAIMS records a regexless rule's reach " + f"as the WHOLE CORPUS, so it starts at its maximum and " + f"arrivals never move it. The one rule ever shaped this " + f"way grew from 4 explained names to 14, across six " + f"unrelated behavior families, with every guard green " + f"the whole time. Narrow by name instead, or split this " + f"into the rules the diffs actually need") if has_regex: pattern = rule["name_regex"] if not isinstance(pattern, str): From 0f0985d322594bddcbdc62a4953cb37021319065 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 23:07:58 -0700 Subject: [PATCH 11/15] docs(decisions): record the fields-only ledger arc The third ledger arc section, in the two existing ones' idiom. What landed: the validate_rules ban; the over-declared-fields defect and its fix(#424) twin, which together are why #452 was filed; the cause-not-consent standing for a rule that classifies an unwanted reading; one-vocabulary-per-alternation as a constraint the guards impose rather than a preference; a rule's character class as a pinned copy of the range the implementation acts on; and past-tense counts staying where live ones go. Three things were FOUND rather than decided and get their own heading, because each is a fact about the shape rather than a choice about it: a fields-only rule silently falsifies other comments (measured -- zero of seven probes arrived unexplained while it lived); copy-paste is how a ledger comment goes false without being edited; and _sorted_rules is now the identity on every ledger that loads, kept anyway as the defence that makes the ban safe to state. Declined and why: a [[never]] for #342/#397 (reddens the gate for a whole milestone), the single-rule replacement with a mixed-vocabulary tail (measured -- the M.A. member reaches a non-entry, x is in no set), the literal-alternation fallback, and extending the bundle to the trailing-numeral ambiguity or to 'Donald mc', both of which are documented consequences of decided scopes rather than defects. Refs #451, #452 Co-Authored-By: Claude Opus 5 --- docs/design/decisions.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index ccc115f1..1d672ebd 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -620,6 +620,33 @@ Declined: - Specificity reordering of the rule sort (#328) — measured across all 751 names: width-then-regex-length moves five rule populations and sends 17 names into the generic fields-only rule, draining the CJK-specific ones. No reading of the sort produces the "exactly one label changes" originally claimed; that figure was corrected on the PR. - Unanchoring the honorific-suffix rule to reach glued forms (#376) — it would make a future suffix regression on 김지양 classify as a recognized honorific. A confidently wrong label is worse than a catch-all's honest breadth, and the gate reads the same either way. +### differential-ledger, the fields-only arc (2026-08-27, #451) + +The third ledger arc, and the one that ends a shape rather than adding vocabulary. Measurements are this session's, at the counts each commit records. + +Decisions that landed: + +- 2026-08-27 #451 — `validate_rules` REJECTS a rule carrying `fields` and no `name_regex`. Such a rule claims every name whose diff fits those fields, and `_CORPUS_CLAIMS` records a regexless rule's reach as the WHOLE corpus — already at its maximum, so arrivals never move it and `test_every_rule_claims_the_recorded_share_of_the_corpus` passes whether it explains 4 names or 40. The one rule with the shape drifted from the four names its prose described to SEVENTEEN at its high-water mark — decisions.md#H1 records the 14 -> 17 step and #410 taking three back — and stood at fourteen across six unrelated behaviour families when this bundle opened, every guard green throughout, absorbing two open v2.2 bugs (#342, #397) as intentional 2.0 changes while AGENTS.md names the 1.4 summary as the release log's source for Behavior Changes. Free to state only because the fourteen were given rules first; a role-count floor was the earlier proposal and stays declined (#372/#373, vacuous). +- 2026-08-27 #451 — a rule whose declared `fields` EXCEED every diff it explains is a defect in its own right, not untidiness. Measured on the catch-all: it declared `{given, family, suffix}`, and by the time this bundle's first three commits had given its `{given, family}` names rules of their own, nothing it still explained moved `given`. Note WHEN the excess appeared, because it is the whole mechanism — the declaration was earned when the rule was written and became excess as its explained set SHRANK under it, which is exactly `fix(#424)` at decisions.md#H1, where #410 narrowed a diff from three roles to two and the rule kept matching by superset with no run naming it. A rule does not have to widen to become over-declared; the diffs beneath it only have to leave. The two together are why #452 was filed — nothing counts what a rule actually explains, only whether it explains anything. +- 2026-08-27 #451 — a ledger rule may classify a reading NOBODY WANTS, provided its `issue` says so. `fix(#342)` and `fix(#397)` carry NOT WANTED in the issue text, the cause-not-consent sentence decisions.md#P5 established for the #369 regression, and a delete-when-fixed instruction the dormancy check enforces once the fix lands. +- 2026-08-27 #451 — one vocabulary per alternation, FORCED rather than chosen. `_LatinCopy` carries one `vocabulary` and `test_latin_alternations_mean_something_the_vocabulary_ships` asserts exactly one roster key per alternation, so `jr` (SUFFIX_WORDS) and `mc`/`mp` (SUFFIX_ACRONYMS) cannot share one. That constraint forces ONE of the three splits. The other two are forced separately and are recorded below under the declined single-rule tail: `m\.?a\.?` cannot be an alternation member at all, and the numeral alternation copies no wordlist. Four rules, three independent reasons, none of them taste. +- 2026-08-27 #451 — a ledger rule's character class must be a copy of the range the IMPLEMENTATION acts on, and is pinned to it. The emoji rule's first draft spanned `U+1F300-U+1FAFF` while `_tokenize.py`'s `_EMOJI_RANGES` divides on two narrower blocks: it claimed 1072 codepoints where the parser leaves the token whole, so its own prose ("an emoji inside a token divides it") was false for every one of them and a `{given, family}` diff with another cause would have classified as intended. `test_the_emoji_boundary_rule_copies_the_dividing_ranges` holds it, and fails on the draft it replaced. +- 2026-08-27 #451 — `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads. Verified against all three: the ban removes the only shape that could occupy its second tier, so file order settles every tie there is and the function reorders nothing. Kept because it is what makes the ban safe to state for any reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture. Deleting it would move the guarantee from the code into a convention, which is the trade #451 was filed to undo. Recorded as a decision rather than as a consequence precisely so a later dead-code sweep finds the reasoning before the `return`. +- 2026-08-27 #451 — past-tense counts stay in a ledger comment; live ones do not. "It claimed 25 until #372" is a fixed fact. "Four corpus names, and the prose fits all four" was written when the rule explained four and still read that way at fourteen, because nothing recomputes it. Where a count is owned by another comment, point at the owner rather than restating it — two copies means one gets updated and the other does not. + +Found rather than decided, and worth as much: + +- **A fields-only rule silently falsifies OTHER comments.** The 2.x ledgers' idiom "such a name should arrive UNEXPLAINED and be read once" is true there because they have no fields-only rule. In the 1.4.0 ledger it was false for anything of the catch-all's shape, and nothing said so. Measured while the rule still lived, by driving `compare.classify()` directly over the seven `_MUST_NOT_MATCH` probes of the two rules #451 ported, at diff shape `{given, family}`: ZERO arrived unexplained. Four went to the catch-all ('Mcintyre Smith Jr.', 'Los Santos', 'John Sir de Mesnil', 'Sir Smith'), two to `fix(comma-precomma-family)` ('McDonald, Ronald', 'Smith, Sir de Vaux'), and one ('Ste Marie') to the neighbouring `fix(#360) ste` rule that exists to claim it. Driven through `classify()` rather than read off a gate run, deliberately: six of the seven are in no corpus, so a run would never reach them — which is also why the promise they carried could go false unnoticed. The blast radius of the shape was never just its own heading. +- **Copy-paste is how a ledger comment goes false without being edited.** All three ledgers carried the same `fix(#379)` comment; four of its sentences were true only in the 1.4.0 file they were written for, including one citing `fix(suffix-routing)`, which exists in no 2.x ledger, and one saying a name "falls to the fields-only catch-all" twenty lines after the same file says no fields-only rule exists there. Corrected in `8237b54`. A cross-ledger claim needs re-deriving per file, not copying. + +Declined: + +- A `[[never]]` for #342 and #397 (2026-08-27) — honest, and it reddens the gate: an excluded name reports UNEXPLAINED and exits non-zero, so every release would block until both bugs are fixed. Both are on v2.2, so the window is real. The cause-not-consent rule says the same thing in the ledger and keeps the gate green. +- One replacement rule with a `(jr|v|mp|x|m\.?a\.?|mc)` tail (2026-08-27) — measured, not argued: `m\.?a\.?` matches the corpus fragment `'M.A.'`, which `_normalize` leaves as `'m.a'` and is not a `SUFFIX_ACRONYMS` entry, so the member guard rejects it — the same finding the `ambiguous-surname-acronym` rule already records. `x` is in no vocabulary set at all, reaching `suffix` through the trailing-roman-numeral fork, so it could not be a member either. +- A literal alternation of all 14 absorbed names (2026-08-27), on #413's literal-anchored precedent — kept as the fallback if a reading turned out contested, and not needed: every family had an existing rule in another ledger, a decision to cite, or an open issue. +- Extending #451 to the trailing-numeral ambiguity (2026-08-27) — `'Mohamad X'` loses its family to `_ROMAN`, and by C-i's "under uncertainty, default to AMBIGUOUS" a trailing `X` IS borne as an ordinary name in that position (Malcolm X). Numerals reach `suffix` through a pattern rather than a vocabulary set, so there is no ambiguous SUBSET for `x` to join — but the fork does not go unreported: measured, `parse('Mohamad X')` and `parse('John V')` both carry `AmbiguityKind.SUFFIX_OR_NAME`, which `_assign` raises from the fork itself because "V/X/I are ordinary middle initials, so taking it as a suffix is a call, not a fact". The contrast worth keeping is with the acronym half: `parse('Aishwarya Rai')` loses its family and reports NOTHING, because `rai` is unambiguous vocabulary. So the numeral fork already flags its own uncertainty and the vocabulary path does not — which is an argument about `SUFFIX_ACRONYMS_AMBIGUOUS`, not about `_ROMAN`, and it belongs to #342 rather than here. `'Malcolm X'` is in no corpus (`'Mohamad X'` is, and is what this bundle classified); no open issue on the numeral half. +- Extending it to `'Donald mc'` (2026-08-27) — `mc` is `SUFFIX_ACRONYMS` AND `PARTICLES`, the collision C-ii decided for `vd`, and C-ii reaches this shape only through P6's attachment, which is scoped to the comma form. The authority is rules.md#S2's Accepted — "an unambiguous suffix is consumed even when that leaves no family name at all" — and NOT P6's scope note, which was this entry's first citation and does not support it. P6 promises the comma-less shapes "keep their positional reading", and for the words that are both particle and suffix vocabulary it does not hold: measured, `'Jong de'` and `'Smith do'` keep theirs (family 'de', family 'do') while `'Donald mc'` and `'Smith vd'` lose the family entirely to `suffix`. So `'Donald mc'` is out of #451's scope on S2's statement, and P6's own Accepted clause is falsified by two of the three words it sweeps in — its only example, `"Jong Anke de"`, is `de`-shaped, which is why no test catches it. Recorded here rather than fixed; it is a rules.md defect, not a ledger one. + ### comma-suffix-arc — #291/#296/#316 (2026-07-26 → 2026-08-01) #291 was filed 2026-07-26 out of the 2.0 vocabulary cleanup, with From ad78f044ca242e70a3243e02553a27ad8e9213c3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 27 Aug 2026 23:29:22 -0700 Subject: [PATCH 12/15] docs(mechanisms,decisions,differential): the ban falsified six standing claims Every statement corrected here was TRUE before this bundle and is false after it, which is the class the arc entry itself names: "copy-paste is how a ledger comment goes false without being edited". Nothing here changes code or a regex. Measured first, on all three ledgers: 1.4.0 70 rules / 0 fields-only, 2.0.0 58 / 0, 2.1.0 51 / 0, and _sorted_rules is the identity on each. So there is one tier, the sort is stable, and FILE ORDER decides every tie -- the opposite of what the three ledger headers promised. mechanisms.md#LEDGER-RULE-SEPARATION was the worst of them, being the citable contract: headed "file order doesn't" separate rules and stating that "a fields-only rule sorts last unconditionally", it would have had a rule author write a rule compare.py now refuses at startup. Rewritten to state the ban, that file order is the only separator left, and that the tier sort is KEPT as the defence for a reader who does not call validate_rules first (its docstring in compare.py says why). The three ledger headers said "File order is not load-bearing" and "optional name_regex"; in the 1.4.0 file that sat twenty lines from prose the same bundle added saying FILE ORDER decides ties. All three now say name_regex is required and file order decides. tools/differential/README.md said a rule "may narrow its match with name_regex ... and/or fields" and that "a name_regex rule outranks a fields-only one". Required, and unreachable, respectively. Two decisions.md entries. The dormancy arc's declined bullet counted "exactly one fields-only rule exists in any ledger"; the count is zero and the DECLINE still stands -- it is anchored to when it was measured rather than restated live, per this bundle's own rule about counts. And P5's #369 entry said its 1.4.0 rule keeps 'Sheik abdul salam' off the catch-all "it otherwise lands on". There is no catch-all to land on: measured through classify() over the ledger minus that rule, the name and the 'Sheik abdul salam Jr' example both return None. The reason to keep the rule is stronger than the entry claimed -- a red gate, not a mislabel. Two the sweep found beyond the review's list. The fix(#360) ste comment carried the same "it lands on the fields-only catch-all" in the present tense (measured: None without it). And the header's "(spec S5)" was a citation to the gitignored spec that the armed legacy-citation guard bans -- it passed only because the line happened to wrap between the two words, so reflowing the header surfaced it. Repointed at the README, which is where that claim actually lives. Refs #451 Co-Authored-By: Claude Opus 5 --- docs/design/decisions.md | 6 ++-- docs/design/mechanisms.md | 6 ++-- tools/differential/README.md | 33 +++++++++++--------- tools/differential/expected_since_1.4.0.toml | 23 +++++++++----- tools/differential/expected_since_2.0.0.toml | 7 +++-- tools/differential/expected_since_2.1.0.toml | 7 +++-- 6 files changed, 51 insertions(+), 31 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 1d672ebd..e617cf9c 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -395,7 +395,7 @@ Excluded (SUFFIX_ACRONYMS / SUFFIX_WORDS — the esq dual membership, deliberate - 2026-08-22 #369 — a given-name title licenses the join with one word to spare. #367 regressed `Sheik Abu Bakar` from given 'Abu Bakar' to given 'Abu', family 'Bakar', and its release note recorded why the old reading had been an accident: `abu` is a particle as well as a bound word, and the title used to displace it out of the leading position so the particle CHAIN took 'Bakar'. `Sheik abdul salam` showed that a bound word alone never joined behind a title — the reserve wanted three name words and a title is not one. The question the issue put was whether `Sheik abdul salam` is a two-word given name with no surname, or given 'abdul' plus family 'salam'. Decided on the signal the title already carries: `sheik` is a GIVEN-NAME title, and H1 reads that membership as the assertion that the one word after it is the given name, family empty. P5 now reads it the same way — behind a given-name title the family comma's LENIENT reserve applies, 'abdul salam' joins, and `family=''` is the Accepted outcome H1 records for "Sir John". Not "after any title": `Dr. abdul salam` keeps given 'abdul', family 'salam', because `dr` addresses by family and the reserve exists for exactly that word. Keyed on the whole title run through `_title_key`, exactly as post_rules keys H1, so the two rules cannot read one run two ways: 'mr sir' is not a given-name title to either. Only STRICT relaxes — the family comma's DISABLED segment has no given name to join, and its post-comma segment is LENIENT already. - Measured at all three baselines: `Sheik abdul salam` moves (given, family), and with it the `Sheik abdul salam Jr` rules example — the same move with the suffix standing — and no other corpus name's fields do; `Sheik Abu Bakar` returns to the fields it had at 1.4.0, 2.0.0 and 2.1.0 — a restoration, which the ledgers record as a non-event — with one v2-surface change: the PARTICLE_OR_GIVEN report it carried through 2.1 is gone, because the join that takes 'Bakar' is P5's rather than the chain's, and a bound word read as the bound word is not a fork. The Arabic-script spelling comes along: `الشيخ عبد الله` read given 'عبد', family 'الله' and now reads given 'عبد الله', which is the name. The previous #369 ledger rules at all three baselines classified the regression "because the cause is known and recorded, NOT because the reading is wanted"; they are rewritten for the fix, and the 1.4.0 one is what keeps `Sheik abdul salam` off the fields-only suffix-routing catch-all it otherwise lands on. + Measured at all three baselines: `Sheik abdul salam` moves (given, family), and with it the `Sheik abdul salam Jr` rules example — the same move with the suffix standing — and no other corpus name's fields do; `Sheik Abu Bakar` returns to the fields it had at 1.4.0, 2.0.0 and 2.1.0 — a restoration, which the ledgers record as a non-event — with one v2-surface change: the PARTICLE_OR_GIVEN report it carried through 2.1 is gone, because the join that takes 'Bakar' is P5's rather than the chain's, and a bound word read as the bound word is not a fork. The Arabic-script spelling comes along: `الشيخ عبد الله` read given 'عبد', family 'الله' and now reads given 'عبد الله', which is the name. The previous #369 ledger rules at all three baselines classified the regression "because the cause is known and recorded, NOT because the reading is wanted"; they are rewritten for the fix, and the 1.4.0 one is what keeps `Sheik abdul salam` explained at all: it landed on the fields-only suffix-routing catch-all before #451 deleted that rule, and with no fields-only rule left in any ledger the name now arrives UNEXPLAINED without it — measured by driving `classify()` over the 1.4.0 ledger minus this rule, which returns None for the name and for the `Sheik abdul salam Jr` example. A stronger reason to keep the rule than the one this entry first gave: the alternative is a red gate, not a mislabel. Two things review added before merge. The licence lifts the reserve for two WORDS, so the piece it lets the join take must be a single word: a particle chain is one piece, but it is the family name P2 built, and without that clause `Sir abdul van der Berg` read given 'abdul van der Berg', family '' — where the untitled name keeps family 'van der Berg', and so does `Sir John van der Berg`. H1's own scope is "exactly one name word", and the licence now matches it. `Sheik abdul al Rahman` therefore stays given 'abdul', family 'al Rahman', the same limit the `abd` entry above records for the three-token spelling. `Sir abdul van der Berg` is a rules.md example and so sits in the rules corpus, where it reads byte-identical to every baseline — the gate witnesses the clause; `Sheik abdul al Rahman` is in no corpus, and the group test is its only witness. And P5 now states the precedence `Sheik Abu Bakar` only exercised: a bound word that is also a particle is read as the bound word, the join outranking P4's leading-position reading, with no fork reported. That is the reading `Abu Bakar Salim` has always had — P5's join and P4's "the words stay separate" were never reconciled in writing — and the two rules now point at each other. @@ -616,8 +616,8 @@ Decisions that landed: Declined: - Ambiguity reporting on multi-matching rules (#372/#373) — measured: 28% of claimed name×role pairs already have ≥2 matching rules (732 of 2619; 432 are one pair of rules alone). A report firing on 28% of what it inspects is wallpaper. The actionable slice shipped as the "shadowed by " diagnosis, which speaks only on FULL shadowing. -- A specificity floor for fields-only rules (#372/#373) — exactly one fields-only rule exists in any ledger, naming 3 of 7 roles; a six-of-seven floor matches nothing, and nothing would reveal it vacuous. -- Specificity reordering of the rule sort (#328) — measured across all 751 names: width-then-regex-length moves five rule populations and sends 17 names into the generic fields-only rule, draining the CJK-specific ones. No reading of the sort produces the "exactly one label changes" originally claimed; that figure was corrected on the PR. +- A specificity floor for fields-only rules (#372/#373) — when this was proposed, exactly one fields-only rule existed across the ledgers, naming 3 of 7 roles; a six-of-seven floor matched nothing, and nothing would reveal it vacuous. Still declined: #451 banned the SHAPE instead (the fields-only arc below), which is a different proposal and leaves no ledger with a rule for a floor to measure. +- Specificity reordering of the rule sort (#328) — measured across all 751 names, on the ledger as it then stood: width-then-regex-length moved five rule populations and sent 17 names into the generic fields-only rule, draining the CJK-specific ones. No reading of the sort produces the "exactly one label changes" originally claimed; that figure was corrected on the PR. - Unanchoring the honorific-suffix rule to reach glued forms (#376) — it would make a future suffix regression on 김지양 classify as a recognized honorific. A confidently wrong label is worse than a catch-all's honest breadth, and the gate reads the same either way. ### differential-ledger, the fields-only arc (2026-08-27, #451) diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 12aa08a4..1bffb19e 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -71,11 +71,11 @@ than inventing one. Problem shape. A guard needs to know what the answer WAS, so it can detect the answer changing. Contract statement. Store the measured answer as literal data (a roster) and compare against it; never re-derive the expectation from the same inputs the check reads, because a derivation from the same data always agrees with itself. Lives in. tests/v2/test_ledger_guards.py (_CORPUS_CLAIMS, _EXCLUSION_EFFECT, _CROSS_RULE_WINNERS, _SPAN_BEARING_RULES, _HONORIFIC_SOURCES, _LATIN_ALTERNATION_SOURCES), tools/differential/compare.py (_CORPUS_FLOORS), tests/v2/test_facade_cases.py (_CORE_ONLY_IDS). Reach for it when. Writing a check whose expected value is computed by the code under test, or a comment that enumerates ids/counts — make it data the suite asserts. -## LEDGER-RULE-SEPARATION — fields separate rules, file order doesn't +## LEDGER-RULE-SEPARATION — file order decides, fields narrow by subset -Problem shape. Two differential-ledger rules claim overlapping names. Contract statement. Fields subsets and matching predicates separate ledger rules BETWEEN tiers; within a tier, file order decides and the narrower rule must be written first. A fields-only rule sorts last unconditionally and takes what nothing narrower named. How it works. Detail is owned by tools/differential/README.md. The within-tier clause is measured, not theoretical: in the 1.4 ledger the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair should be separated by a predicate instead of by order is +Problem shape. Two differential-ledger rules claim overlapping names. Contract statement. Every ledger rule must carry a `name_regex` — since #451 `validate_rules` REJECTS a rule with `fields` and no `name_regex`, and one with neither was already rejected — so every rule sits in ONE tier, the sort is stable, and FILE ORDER decides every contest: the narrower rule must be written first. `fields` narrows a rule by subset; it does not separate rules by sorting. The two-tier sort in `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads (measured on all three): it is the defence for a reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture — and its docstring in tools/differential/compare.py says so. How it works. Detail is owned by tools/differential/README.md. The file-order clause is measured, not theoretical: in the 1.4 ledger the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair should be separated by a predicate instead of by order is [#382](https://github.com/derek73/python-nameparser/issues/382). The old #271/#272 -slug taboo is RETIRED (#333): the canonical-rule selector that keyed on those substrings is deliberately deleted — rule authors are free to use them in compound slugs — and the surviving rosters select on their own explicit keys (_HONORIFIC_SOURCES and _LATIN_ALTERNATION_SOURCES by named issue strings, _SPAN_BEARING_RULES by exact leading fix(...) tag). Lives in. tools/differential/compare.py, the expected_since_*.toml ledgers. Reach for it when. A ledger rule's behavior seems to depend on where it sits in the file — within a tier it does, and the reorder mutation is the test (run twice in #375; it fails _CROSS_RULE_WINNERS). History: #372 (closed) measured the fields-only rule owning 1639 of 5257 name×field pairs as filed (2026-08-10); #375/#376 then cut its classifier-of-record share sharply, and the residual pair ownership is the last-resort tier working as designed, not a defect. #372's two proposed mechanical checks were DECLINED with measurements (see decisions.md#differential-ledger), not left open. +slug taboo is RETIRED (#333): the canonical-rule selector that keyed on those substrings is deliberately deleted — rule authors are free to use them in compound slugs — and the surviving rosters select on their own explicit keys (_HONORIFIC_SOURCES and _LATIN_ALTERNATION_SOURCES by named issue strings, _SPAN_BEARING_RULES by exact leading fix(...) tag). Lives in. tools/differential/compare.py, the expected_since_*.toml ledgers. Reach for it when. A ledger rule's behavior seems to depend on where it sits in the file — it does, and the reorder mutation is the test (run twice in #375; it fails _CROSS_RULE_WINNERS). History: #372 (closed) measured the then-existing fields-only rule owning 1639 of 5257 name×field pairs as filed (2026-08-10); #375/#376 then cut its classifier-of-record share sharply, and the residual pair ownership was read as the last-resort tier working as designed rather than a defect — until #451 retired the shape outright (decisions.md#differential-ledger). #372's two proposed mechanical checks were DECLINED with measurements (see decisions.md#differential-ledger), not left open. ## CANONICAL-VOCABULARY-AT-THE-BOUNDARY — one vocabulary at the comparison diff --git a/tools/differential/README.md b/tools/differential/README.md index 56677a19..c0bca72b 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -299,20 +299,25 @@ history rather than being edited into the next one's. Each `[[change]]` entry needs `issue` (a short label, ideally an issue number or `fix()` matching a `tests/v2/cases.py` -classification) and may narrow its match with `name_regex` (searched -against the raw input string) and/or `fields` (the diffing rule -matches only if the observed diff fields are a subset of this list). -Keep both as tight as the actual diff allows -- a loose rule can mask -a real regression. - -Rules are sorted most-specific-first before matching: a `name_regex` -rule outranks a `fields`-only one (which is broad by construction) -wherever both match. **Within a tier, file order decides.** That is -not a detail -- every rule in `expected_since_2.0.0.toml` carries a -`name_regex`, so they all sit in one tier and the order they are -written in settles every tie between them. Append a rule to the bottom -of a file only after checking that nothing above it already claims the -diff you meant it for. +classification) and `name_regex` (searched against the raw input +string). It may narrow further with `fields` (the diffing rule matches +only if the observed diff fields are a subset of this list). Keep both +as tight as the actual diff allows -- a loose rule can mask a real +regression. `name_regex` is REQUIRED since #451: `validate_rules` +rejects a rule carrying `fields` and no `name_regex`, as it already +rejected one carrying neither. + +**File order decides.** That is not a detail -- every rule in every +ledger carries a `name_regex`, so they all sit in one tier, the sort +is stable, and the order they are written in settles every tie between +them. Append a rule to the bottom of a file only after checking that +nothing above it already claims the diff you meant it for. + +`_sorted_rules` still sorts `name_regex` rules ahead of `fields`-only +ones, and is now the identity on every ledger that loads. It is kept +as a defence for a reader that does not call `validate_rules` first -- +a future tool, a REPL, a test fixture -- rather than as a second tier +any ledger can reach; its docstring in `compare.py` says why. Some entries in `expected_since_1.4.0.toml` are for behavior families that a corpus happens to contain no example of (e.g. custom suffix-delimiter diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index a8595a92..720c8ea2 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1,10 +1,17 @@ -# Every rule needs `issue`; optional `name_regex` and `fields` narrow -# it. An unexplained diff is a release blocker until classified (spec -# S5). Rules were seeded from the 2.0 rewrite's working notes +# Every rule needs `issue` and `name_regex`; `fields` narrows further +# and is optional. Since #451 validate_rules REJECTS a rule carrying +# `fields` and no `name_regex`, as it already rejected one with +# neither. An unexplained diff is a release blocker until classified +# (tools/differential/README.md). The citation here used to name a +# gitignored spec requirement, and passed the armed legacy-citation +# guard only because the line wrapped between the two words. +# Rules were seeded from the 2.0 rewrite's working notes # and the `classification="fix(...)"` rows in tests/v2/cases.py; keep # each entry's `name_regex`/`fields` as tight as the diff allows. -# File order is not load-bearing: compare.py sorts `name_regex` rules -# ahead of `fields`-only ones before matching. +# FILE ORDER IS LOAD-BEARING: every rule carries a `name_regex`, so +# they all sit in one tier, _sorted_rules is stable, and file order +# decides every tie. Write the narrower rule first, and see the note +# above the four fix(suffix-routing) rules at the end of this file. [[change]] issue = "fix(#271/#272/#298) native-script CJK: family-first order, hangul segmentation, the kana license and the dots" @@ -1344,8 +1351,10 @@ issue = "fix(#360) ste moved into the never-given particles with mc" # 'Ste Marie': the second word of the vocabulary move, which the # fix(#360) comment in the 2.x ledgers names as a name that should # arrive UNEXPLAINED and be read once. It has. Without this rule it -# lands on the fields-only catch-all, whose prose describes a trailing -# credential rather than a leading particle folding into the family. +# landed on the fields-only catch-all, whose prose described a trailing +# credential rather than a leading particle folding into the family; +# #451 deleted that catch-all, so removing this rule now costs the +# name its explanation outright (measured through classify()). name_regex = "(?i)^ste\\s+\\S+$" fields = ["given", "family"] diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 43cc22c2..3e2e5b64 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -1,7 +1,10 @@ # Ledger for baseline 2.0.0 -- what changes for a user upgrading from # the previous minor. Same rule grammar as expected_since_1.4.0.toml: -# every rule needs `issue`; optional `name_regex` and `fields` narrow -# it, and compare.py sorts name_regex rules ahead of fields-only ones. +# every rule needs `issue` and `name_regex`; `fields` narrows further +# and is optional. Since #451 validate_rules REJECTS a rule carrying +# `fields` and no `name_regex`, as it already rejected one with +# neither. File order is therefore load-bearing; the note below says +# how it falls out here. # # `fields` names roles the way Role does, for BOTH compared surfaces, # plus the pseudo-field `_ambiguities`. The roster is deliberately not diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 78686367..f18976da 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -1,7 +1,10 @@ # Ledger for baseline 2.1.0 -- what changes for a user upgrading from # the previous minor. Same rule grammar as the other ledgers: every rule -# needs `issue`; optional `name_regex` and `fields` narrow it, and -# compare.py sorts name_regex rules ahead of fields-only ones. +# needs `issue` and `name_regex`; `fields` narrows further and is +# optional. Since #451 validate_rules REJECTS a rule carrying `fields` +# and no `name_regex`, as it already rejected one with neither. Every +# rule therefore sits in one tier, _sorted_rules is stable, and FILE +# ORDER decides every tie: write the narrower rule first. # # This file was opened empty the day 2.1.0 shipped (AGENTS.md release # step 8) and stayed that way through #354, #358 and #361, none of From 0b588236f23d00774cf66950012242b76069ba6e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 28 Aug 2026 22:31:17 -0700 Subject: [PATCH 13/15] fix(differential,guards,decisions): the five defects the PR review confirmed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four reviewers on #453. None of these is visible to the suite or the gate, which is why the review found them and CI could not. Three were claims corrected in one file and left standing in another -- the failure this bundle's own decisions.md entry names, committed while documenting it: - Both 2.x ledgers still said the 1.4 catch-all exists ("where there is one"). 8237b54's message predicted that would go stale when the rule was deleted and assigned the fix to ffd242a, which handled the 1.4 copy and not the two pointers back at it. - The mc|mp rule cited P6's scope note as its authority and attributed a decisions.md quote to rules.md#P6. decisions.md:648, added two commits later, records that citation as wrong: P6 promises the comma-less shapes keep their POSITIONAL reading, and measured, 'Donald mc' and 'Smith vd' lose the family entirely where 'Jong de' and 'Smith do' keep theirs. The authority is rules.md#S2's Accepted, and the rule now says so. - The N3 rule had H1 backwards twice. H1 does not decline beside a suffix -- rules.md#H1 reads "whatever suffix, nickname or maiden name stands beside it", and 'Dr. Smith Jr.' measures family 'Smith'; the declining term came out in #410, which is what that entry records. And with the suffix set aside N3 gives FAMILY 'Jones', not given: 'Jones' is the given precisely BECAUSE N3 counts the suffix and declines. The emoji guard could not see the widening it promised to refuse. It scanned 0x1F000-0x20000, so a bare '-' or a ⬀-⯿ span appended to the class left it green while the rule stood ready to explain a {given, family} regression on 22 hyphenated corpus names. Three reviewers reached it by three routes. It scans all of Unicode now (measured: same 976 codepoints on the shipped class, 0.3s, and it fires on both widenings), and the rule's comment no longer frames _SPAN's blindness to the 8-hex \U spelling as a safeguard -- it is a consequence, and the guard is what actually bounds the class. Last, the arc entry contradicted itself on the catch-all's maximum: "SEVENTEEN at its high-water mark" six lines above "It claimed 25 until #372". 17 is the post-#372 figure, and the entry says so now. Gates unmoved (1.4.0 229/0, 2.0.0 194/0, 2.1.0 102/0); 6062 passed. Refs #451 --- docs/design/decisions.md | 2 +- tests/v2/test_ledger_guards.py | 17 +++++-- tools/differential/expected_since_1.4.0.toml | 51 ++++++++++++++------ tools/differential/expected_since_2.0.0.toml | 3 +- tools/differential/expected_since_2.1.0.toml | 3 +- 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index e617cf9c..3d05deb0 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -626,7 +626,7 @@ The third ledger arc, and the one that ends a shape rather than adding vocabular Decisions that landed: -- 2026-08-27 #451 — `validate_rules` REJECTS a rule carrying `fields` and no `name_regex`. Such a rule claims every name whose diff fits those fields, and `_CORPUS_CLAIMS` records a regexless rule's reach as the WHOLE corpus — already at its maximum, so arrivals never move it and `test_every_rule_claims_the_recorded_share_of_the_corpus` passes whether it explains 4 names or 40. The one rule with the shape drifted from the four names its prose described to SEVENTEEN at its high-water mark — decisions.md#H1 records the 14 -> 17 step and #410 taking three back — and stood at fourteen across six unrelated behaviour families when this bundle opened, every guard green throughout, absorbing two open v2.2 bugs (#342, #397) as intentional 2.0 changes while AGENTS.md names the 1.4 summary as the release log's source for Behavior Changes. Free to state only because the fourteen were given rules first; a role-count floor was the earlier proposal and stays declined (#372/#373, vacuous). +- 2026-08-27 #451 — `validate_rules` REJECTS a rule carrying `fields` and no `name_regex`. Such a rule claims every name whose diff fits those fields, and `_CORPUS_CLAIMS` records a regexless rule's reach as the WHOLE corpus — already at its maximum, so arrivals never move it and `test_every_rule_claims_the_recorded_share_of_the_corpus` passes whether it explains 4 names or 40. The one rule with the shape drifted from the four names its prose described to 25 before #372 cut it back, and then to SEVENTEEN after — decisions.md#H1 records that 14 -> 17 step and #410 taking three back — standing at fourteen across six unrelated behaviour families when this bundle opened, every guard green throughout, absorbing two open v2.2 bugs (#342, #397) as intentional 2.0 changes while AGENTS.md names the 1.4 summary as the release log's source for Behavior Changes. Free to state only because the fourteen were given rules first; a role-count floor was the earlier proposal and stays declined (#372/#373, vacuous). - 2026-08-27 #451 — a rule whose declared `fields` EXCEED every diff it explains is a defect in its own right, not untidiness. Measured on the catch-all: it declared `{given, family, suffix}`, and by the time this bundle's first three commits had given its `{given, family}` names rules of their own, nothing it still explained moved `given`. Note WHEN the excess appeared, because it is the whole mechanism — the declaration was earned when the rule was written and became excess as its explained set SHRANK under it, which is exactly `fix(#424)` at decisions.md#H1, where #410 narrowed a diff from three roles to two and the rule kept matching by superset with no run naming it. A rule does not have to widen to become over-declared; the diffs beneath it only have to leave. The two together are why #452 was filed — nothing counts what a rule actually explains, only whether it explains anything. - 2026-08-27 #451 — a ledger rule may classify a reading NOBODY WANTS, provided its `issue` says so. `fix(#342)` and `fix(#397)` carry NOT WANTED in the issue text, the cause-not-consent sentence decisions.md#P5 established for the #369 regression, and a delete-when-fixed instruction the dormancy check enforces once the fix lands. - 2026-08-27 #451 — one vocabulary per alternation, FORCED rather than chosen. `_LatinCopy` carries one `vocabulary` and `test_latin_alternations_mean_something_the_vocabulary_ships` asserts exactly one roster key per alternation, so `jr` (SUFFIX_WORDS) and `mc`/`mp` (SUFFIX_ACRONYMS) cannot share one. That constraint forces ONE of the three splits. The other two are forced separately and are recorded below under the declined single-rule tail: `m\.?a\.?` cannot be an alternation member at all, and the numeral alternation copies no wordlist. Four rules, three independent reasons, none of them taste. diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 634c6ad4..600b5d00 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -562,7 +562,16 @@ def test_the_emoji_boundary_rule_copies_the_dividing_ranges() -> None: continue found += 1 pattern = rule["name_regex"] - claimed = {c for c in range(0x1F000, 0x20000) + # EVERY codepoint, not just the astral plane. Scanned + # 0x1F000-0x20000 until the #453 review measured what that + # missed: a bare '-' or a \u2B00-\u2BFF span appended to + # the class left this guard GREEN while the rule stood + # ready to explain a {given, family} regression on 22 + # hyphenated corpus names. The docstring promises to + # refuse a class reaching a codepoint the tokenizer does + # not divide on; a scan narrower than that promise is the + # #451 shape one level down. Costs 0.3s. + claimed = {c for c in range(0x20, 0x110000) if re.search(pattern, f"a{chr(c)}b")} stray = sorted(claimed - divides) assert not stray, ( @@ -574,9 +583,9 @@ def test_the_emoji_boundary_rule_copies_the_dividing_ranges() -> None: f"would classify here as intended. Copy from " f"_EMOJI_RANGES rather than widening the span.") assert claimed, ( - f"{ledger.name}: {rule['issue']!r} claims no astral " - f"codepoint at all; the class or the anchor is broken " - f"and the rule can explain nothing") + f"{ledger.name}: {rule['issue']!r} claims no codepoint " + f"at all; the class or the anchor is broken and the " + f"rule can explain nothing") assert found, ( "no emoji-boundary rule in any ledger; this pin is passing " "vacuously") diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 720c8ea2..ac088558 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -576,9 +576,13 @@ issue = "fix(emoji-boundary) an emoji inside a token divides it" # rather than copied: no corpus name reaches it once the token anchor # is on, and a rule should be no wider than the diffs it must explain. # A name that needs it will arrive unclassified and be read once. That -# omission also keeps the pattern free of \uXXXX-\uXXXX span text, so -# _SPAN_BEARING_RULES has nothing to declare about a rule that is not -# script-scoped. +# omission also leaves the pattern free of \uXXXX-\uXXXX span text. +# That is a CONSEQUENCE, not a safeguard, and the #453 review was +# right to say so: _SPAN reads the 4-hex spelling only, so the class +# below rides past _unrecognized_class_content whatever it contains. +# What actually bounds it is the guard named above, which probes every +# codepoint in Unicode against _EMOJI_RANGES, plus _CORPUS_CLAIMS' +# reach and digest. name_regex = "\\S[\\U0001F300-\\U0001F64F\\U0001F680-\\U0001F6FF]\\S" fields = ["given", "family"] @@ -1894,9 +1898,12 @@ fields = ["given", "middle", "family", "maiden"] [[change]] issue = "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`" # "'Smitty' Jones Jr.": the diff against 1.4.0 moves exactly {family, -# suffix} -- `given` is 'Jones' at both baselines, which is what the -# N3 lone-word nickname rule already gives it once the nickname and -# suffix are set aside. Whether 1.4 itself read the trailing word as +# suffix} -- `given` is 'Jones' at both baselines, and it is `given` +# BECAUSE N3 declines: N3's one-piece count does not set a trailing +# suffix aside, so the suffix disqualifies the rule and the positional +# read applies. Set the suffix aside and N3 does fire -- +# "'Smitty' Jones" reads family 'Jones' (rules.md#N3's own example). +# Whether 1.4 itself read the trailing word as # `family` is outside what this ledger's worker can check directly # (tools/differential/README.md's warning against reading a v1 # release from a cached environment); the diff shape is consistent @@ -1912,9 +1919,12 @@ issue = "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in # none of them, which is why it kept a rule of its own. # # Note what is NOT settled here: decisions.md#H1 records that N3 and -# H1 count a trailing suffix in opposite directions -- H1's -# title-plus-one-word rule declines the moment a suffix stands beside -# it, while "N3 counts a suffix the other way (`'Smitty' Jones Jr.` +# H1 count a trailing suffix in opposite directions. H1 IGNORES one -- +# rules.md#H1 makes the lone word the family "whatever suffix, +# nickname or maiden name stands beside it", so 'Dr. Smith Jr.' reads +# family 'Smith' (the declining term was there until #410 took it +# out, which is what that entry is recording). N3 counts one, and so +# declines: "N3 counts a suffix the other way (`'Smitty' Jones Jr.` # reads given 'Jones', family '')" -- and H1 states plainly that # "which of the two readings is right for a nickname-led name is not # decided here". This rule classifies only the 1.4-to-2.x movement @@ -2055,13 +2065,22 @@ issue = "fix(suffix-routing) a two-token name ending in a credential acronym kee # C-ii settled for 'vd' ("the Dutch van der reading, as the more # common"), and which its own C-ii note records that 'do' and 'mc' # inherit "without being weighed". That decision reaches a written name -# only through rules.md#P6's attachment, and P6 is scoped: "the COMMA -# form only, deliberately. [...] Without the comma the written shape is -# not settled [...] Those keep their positional reading and are not -# tracked as deviations." 'Donald mc' has no comma, so it is outside -# P6, and S2's credential reading is what stands -- which is what this -# rule classifies, and it is the positional reading P6's scope note -# leaves in place rather than a decision this rule makes. +# only through P6's attachment, which is scoped to the comma form -- +# decisions.md#P6: "the COMMA form only, deliberately. [...] Without +# the comma the written shape is not settled [...] Those keep their +# positional reading and are not tracked as deviations." 'Donald mc' +# has no comma, so P6 never reaches it. +# +# What stands instead is rules.md#S2: "an unambiguous suffix is +# consumed even when that leaves no family name at all". That is the +# authority here, and NOT P6's scope note, which does not support it: +# P6 promises the comma-less shapes keep their POSITIONAL reading, and +# for the words that are both particle and suffix vocabulary it does +# not hold. Measured, 'Jong de' and 'Smith do' keep theirs (family +# 'de', family 'do') while 'Donald mc' and 'Smith vd' lose the family +# entirely to `suffix`. decisions.md#P6's own arc entry records that as +# a rules.md defect rather than a ledger one; this rule classifies the +# movement on S2's statement and settles nothing about P6. # # The two neighbouring shapes both already have rules above, and the # contrast is the point: 'Mc Donald' has the particle LEADING and folds diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 3e2e5b64..c67f0366 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -820,7 +820,8 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # UNEXPLAINED and block the gate until someone read it, which is the # right outcome: this ledger has no fields-only rule to absorb it. # The claim that it "falls to the fields-only catch-all" was copied -# from the 1.4 ledger, where there is one -- #451.) The class is the +# from the 1.4 ledger, where one existed until #451 deleted it.) +# The class is the # neighbours' and compare.py's own: U+0000-U+024F, which stops at # Latin Extended-B, so a diacritic-bearing 'Nguyen, Van de la' in NFD # or Latin Extended Additional is outside it. diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index f18976da..b9ee3023 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -723,7 +723,8 @@ issue = "fix(#379) a tussenvoegsel after a family comma attaches to the family" # UNEXPLAINED and block the gate until someone read it, which is the # right outcome: this ledger has no fields-only rule to absorb it. # The claim that it "falls to the fields-only catch-all" was copied -# from the 1.4 ledger, where there is one -- #451.) The class is the +# from the 1.4 ledger, where one existed until #451 deleted it.) +# The class is the # neighbours' and compare.py's own: U+0000-U+024F, which stops at # Latin Extended-B, so a diacritic-bearing 'Nguyen, Van de la' in NFD # or Latin Extended Additional is outside it. From 841490a8272453965170e5455328e67e333e7002 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 28 Aug 2026 22:37:43 -0700 Subject: [PATCH 14/15] fix(differential,guards,docs): the four Important review findings, and three wrong sentences Follows 0b58823, which took the Criticals. Each of these is a defect this branch introduced or enlarged, or false prose it left standing. validate_rules' #451 check MOVES to the end of the family. Written directly under the neither-key check it pre-empted the three that follow, each of which buys a precise message for a MALFORMED `fields`: an empty one reported #451 instead of "empty 'fields'". That is why e0ac117 had to grow a name_regex onto five parametrized rows, and its message claiming "each is updated to keep testing what it was written to test" was true of five and not of the two `dormant` rows, which passed on check order alone. Measured after the move: empty/not-roles report their own messages again, and a well-formed fields-only rule still reports #451 with or without `dormant`. _EXCLUSION_EFFECT stops collapsing winners to `fix(tag)`. This ledger has three rules beginning fix(comma-family) and, since #451, four beginning fix(suffix-routing), so a tag-keyed tuple sits at its maximum for that tag and a sibling widening onto a protected reading would not move it. Full issue strings now, re-measured, not adjusted. Three statements in test_ledger_guards.py still called the banned shape "the most unbounded shape validate_rules permits". ad78f04 swept five files for these and missed this one. Each now says the shape is rejected AND why the code handling it stays: _rules() does not validate, so the fallbacks are reachable from this module even though no loadable ledger has one. The README and mechanisms.md read as though the absorption mechanism were gone. It is not: the only width check is the sentinel probe, and measured, `[a-z]` validates and reaches 941 of 1090 corpus names, `" "` 1027. What #451 bought is that such a rule now has a _CORPUS_CLAIMS reach and digest, so its breadth is visible once to a reviewer instead of never. Both files say that, and point at #452 for the ceiling that would bound it. Three sentences that were simply wrong: quotation marks around text that exists in no file (the nearest source is _reaches_non_vocabulary's docstring, now quoted and attributed); "all three rules" describing three names under two rules; and the numeral equivalence proved over strings of length 1 to 4, a population that excludes the empty string that IS the difference. Re-measured over length 0 to 4: the empty string is the only one. Gates unmoved (229/0, 194/0, 102/0); 6062 passed. Refs #451, #452 --- docs/design/mechanisms.md | 2 +- tests/v2/test_ledger_guards.py | 42 ++++++++++++++------ tools/differential/README.md | 13 ++++++ tools/differential/compare.py | 33 +++++++++------ tools/differential/expected_since_1.4.0.toml | 21 ++++++---- 5 files changed, 77 insertions(+), 34 deletions(-) diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 1bffb19e..cde269af 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -73,7 +73,7 @@ Problem shape. A guard needs to know what the answer WAS, so it can detect the a ## LEDGER-RULE-SEPARATION — file order decides, fields narrow by subset -Problem shape. Two differential-ledger rules claim overlapping names. Contract statement. Every ledger rule must carry a `name_regex` — since #451 `validate_rules` REJECTS a rule with `fields` and no `name_regex`, and one with neither was already rejected — so every rule sits in ONE tier, the sort is stable, and FILE ORDER decides every contest: the narrower rule must be written first. `fields` narrows a rule by subset; it does not separate rules by sorting. The two-tier sort in `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads (measured on all three): it is the defence for a reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture — and its docstring in tools/differential/compare.py says so. How it works. Detail is owned by tools/differential/README.md. The file-order clause is measured, not theoretical: in the 1.4 ledger the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair should be separated by a predicate instead of by order is +Problem shape. Two differential-ledger rules claim overlapping names. Contract statement. Every ledger rule must carry a `name_regex` — since #451 `validate_rules` REJECTS a rule with `fields` and no `name_regex`, and one with neither was already rejected — so every rule sits in ONE tier, the sort is stable, and FILE ORDER decides every contest: the narrower rule must be written first. `fields` narrows a rule by subset; it does not separate rules by sorting. The ban ends the SHAPE and not the property it enabled: a required `name_regex` bounds nothing by itself, since the only width check is the sentinel probe — measured, `[a-z]` validates and reaches 941 of 1090 corpus names. What changed is that such a rule now carries a `_CORPUS_CLAIMS` reach and digest, so its breadth is visible once at recording time rather than never (#452). The two-tier sort in `_sorted_rules` is KEPT although the ban makes it the identity on every ledger that loads (measured on all three): it is the defence for a reader that does not call `validate_rules` first — a future tool, a REPL, a test fixture — and its docstring in tools/differential/compare.py says so. How it works. Detail is owned by tools/differential/README.md. The file-order clause is measured, not theoretical: in the 1.4 ledger the comma-honorific-peel rule's fields are a strict subset of the comma-compound rule's, both carry a name_regex, and a pure reorder reattributes seven names — caught by _CROSS_RULE_WINNERS and by nothing else in the suite (#375's mutation). Whether that pair should be separated by a predicate instead of by order is [#382](https://github.com/derek73/python-nameparser/issues/382). The old #271/#272 slug taboo is RETIRED (#333): the canonical-rule selector that keyed on those substrings is deliberately deleted — rule authors are free to use them in compound slugs — and the surviving rosters select on their own explicit keys (_HONORIFIC_SOURCES and _LATIN_ALTERNATION_SOURCES by named issue strings, _SPAN_BEARING_RULES by exact leading fix(...) tag). Lives in. tools/differential/compare.py, the expected_since_*.toml ledgers. Reach for it when. A ledger rule's behavior seems to depend on where it sits in the file — it does, and the reorder mutation is the test (run twice in #375; it fails _CROSS_RULE_WINNERS). History: #372 (closed) measured the then-existing fields-only rule owning 1639 of 5257 name×field pairs as filed (2026-08-10); #375/#376 then cut its classifier-of-record share sharply, and the residual pair ownership was read as the last-resort tier working as designed rather than a defect — until #451 retired the shape outright (decisions.md#differential-ledger). #372's two proposed mechanical checks were DECLINED with measurements (see decisions.md#differential-ledger), not left open. diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 600b5d00..b701a211 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1420,9 +1420,10 @@ class _Claim(NamedTuple): comma rule kept its `,` regex and its 236 names while going from explaining 6 of the corpus to 242. """ - #: corpus names the name_regex reaches; the whole corpus when a - #: rule has none, which is the most unbounded shape validate_rules - #: permits and the one most worth writing down + #: corpus names the name_regex reaches. The whole corpus when a + #: rule has none -- a shape validate_rules REJECTED in #451, so no + #: ledger can carry one now; the fallback stays because this + #: module reads ledgers without validating them first names: int #: the roles it narrows by, sorted; () when it narrows by regex alone roles: tuple[str, ...] @@ -1899,9 +1900,10 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: f"{sorted({i for i in issues if issues.count(i) > 1})}. Every " f"roster here keys on it, so one of them would go unmeasured") # A rule with no name_regex narrows by `fields` alone and so - # reaches EVERY name -- the most unbounded shape validate_rules - # permits, and the one most worth recording. Counting it as the - # whole corpus is not a placeholder; it is what it claims. + # reaches EVERY name. #451 made that shape a startup error, so + # no ledger reaching this line has one -- but _rules() does not + # validate, so counting it as the whole corpus stays correct + # rather than becoming dead. It is what such a rule claims. actual = {rule["issue"]: _claim(rule) for rule in _rules(ledger)} moved = {issue: (recorded.get(issue), count) @@ -2134,7 +2136,18 @@ class _Excluded(NamedTuple): # SHRANK is the safe direction -- one fewer rule stands # ready to claim a protected reading -- which is why this # roster's message warns only about growth. - ("fix(comma-family)", "fix(comma-precomma-family)")), + # FULL issue strings, not `fix(tag)` prefixes. The + # #453 review measured what the truncation cost: this + # ledger carries three rules beginning "fix(comma-family)" + # and, since #451, four beginning "fix(suffix-routing)", + # so a tag-keyed tuple sits at its maximum for that tag + # -- any of the siblings could widen onto a protected + # reading and this roster would stay green. The same + # identity-free weakness _SPAN_BEARING_RULES records. + ("fix(comma-family) lone post-comma piece routes to " + "suffix/title, not first", + "fix(comma-precomma-family) pre-comma run reads as " + "family, not given")), '(^|[\\w.]\\s+)[("\'][^)"\']+[)"\'](\\s+\\w|\\s*$)': # 51 -> 54 as rules.md gained the bracketed Polish examples # (#434): 'Maria Kowalska (z domu Nowak)', 'Maria Kowalska @@ -2210,7 +2223,7 @@ def test_every_exclusion_silences_what_is_recorded() -> None: continue claimed = compare.classify(example, diff, rules) if claimed: - absorbed.add(claimed.split(")")[0] + ")") + absorbed.add(claimed) actual[entry["name_regex"]] = _Excluded( len(captured), hashlib.sha256( @@ -2318,9 +2331,12 @@ def test_a_rule_reaching_no_corpus_name_says_why_it_is_kept() -> None: for rule in _rules(ledger): if "dormant" in rule: continue - # fields-only rules reach every name by construction, so only - # a name_regex can be statically silent -- see _claim(), which - # counts them as the whole corpus for the same reason + # Every rule has a name_regex since #451, so this skip is + # unreachable for a ledger that loads; kept because _rules() + # does not validate. A fields-only rule would reach every + # name by construction and so could never be statically + # silent -- see _claim(), which counts one as the whole + # corpus for the same reason regex = rule.get("name_regex") if not isinstance(regex, str): continue @@ -2333,5 +2349,5 @@ def test_a_rule_reaching_no_corpus_name_says_why_it_is_kept() -> None: f"with the reason the rule is worth keeping, or delete it.") assert checked, ( "no rule was examined, so this guard is passing vacuously -- " - "every rule either declares `dormant` or narrows by `fields` " - "alone") + "every rule declares `dormant`, or (impossible since #451) " + "narrows by `fields` alone") diff --git a/tools/differential/README.md b/tools/differential/README.md index c0bca72b..6f77579d 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -307,6 +307,19 @@ regression. `name_regex` is REQUIRED since #451: `validate_rules` rejects a rule carrying `fields` and no `name_regex`, as it already rejected one carrying neither. +**That closes the SHAPE, not the property.** A required `name_regex` +is not a bound on how much a rule reaches: the only width check is the +sentinel probe, which rejects a pattern matching all four of +`_SENTINELS` and nothing narrower. Measured over the 1090-name corpus, +`name_regex = "[a-z]"` passes validation and reaches 941 names, `" "` +reaches 1027 -- the old catch-all with a fig leaf. What #451 changed is +that such a rule now has a `_CORPUS_CLAIMS` reach and digest to record, +so its breadth is visible ONCE, to whoever reviews that number, instead +of being invisible forever. That roster is by its own docstring "inert +for a brand-new rule", so the review is the check. A reach ceiling is +the mechanism that would bound this; it is proposed on +[#452](https://github.com/derek73/python-nameparser/issues/452). + **File order decides.** That is not a detail -- every rule in every ledger carries a `name_regex`, so they all sit in one tier, the sort is stable, and the order they are written in settles every tie between diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 741ff179..939f2b18 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -465,18 +465,6 @@ def validate_rules(rules: list[dict[str, object]], ledger: str) -> None: raise SystemExit( f"{where} has neither 'name_regex' nor 'fields' -- it " f"would match every diff and shadow every later rule") - if has_fields and not has_regex: - raise SystemExit( - f"{where} has 'fields' but no 'name_regex' (#451). With " - f"no name narrowing, the rule claims every name whose " - f"diff fits its 'fields' -- and no guard can see that " - f"grow: _CORPUS_CLAIMS records a regexless rule's reach " - f"as the WHOLE CORPUS, so it starts at its maximum and " - f"arrivals never move it. The one rule ever shaped this " - f"way grew from 4 explained names to 14, across six " - f"unrelated behavior families, with every guard green " - f"the whole time. Narrow by name instead, or split this " - f"into the rules the diffs actually need") if has_regex: pattern = rule["name_regex"] if not isinstance(pattern, str): @@ -527,6 +515,27 @@ def validate_rules(rules: list[dict[str, object]], ledger: str) -> None: f"below baseline 2.0: there the seven ARE the whole " f"vocabulary, and a rule listing them would have " f"claimed every diff in the 1.4 ledger") + # LAST of the family, deliberately. This rejects a + # WELL-FORMED `fields` that simply has no name beside it, so it + # must not pre-empt the three checks above, each of which buys a + # precise message for a `fields` that is malformed -- empty, not + # a list, not roles. Written directly under the neither-key + # check (where the shapes are cousins) it did exactly that: an + # empty `fields` reported #451 instead of "empty 'fields'", and + # the parametrized cases pinning those messages had to grow a + # name_regex to keep reaching them (#453 review). + if has_fields and not has_regex: + raise SystemExit( + f"{where} has 'fields' but no 'name_regex' (#451). With " + f"no name narrowing, the rule claims every name whose " + f"diff fits its 'fields' -- and no guard can see that " + f"grow: _CORPUS_CLAIMS records a regexless rule's reach " + f"as the WHOLE CORPUS, so it starts at its maximum and " + f"arrivals never move it. The one rule ever shaped this " + f"way grew from 4 explained names to 14, across six " + f"unrelated behavior families, with every guard green " + f"the whole time. Narrow by name instead, or split this " + f"into the rules the diffs actually need") def validate_exclusions(entries: list[dict[str, object]], diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index ac088558..0a25d369 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1982,10 +1982,13 @@ issue = "fix(suffix-routing) a two-token name ending in a roman numeral keeps it # The alternation is _ROMAN's own body, r"^(X|IX|IV|V?I{0,3})$" in # nameparser/_pipeline/_vocab.py, with the EMPTY alternative removed: # I{0,3} becomes I{1,3}, and the bare V that change would lose is -# spelled out beside it. Measured over every string of length 1 to 4 -# drawn from IVXivx, the two patterns accept exactly the same strings -# with one exception, the empty one, which _ROMAN accepts and this does -# not. Dropping it is deliberate: behind "^\S+\s+" and before "$" an +# spelled out beside it. Measured over every string of length 0 to 4 +# drawn from IVXivx -- length 0 included on purpose, since the empty +# string is the whole difference -- the two accept exactly the same +# strings but that one, which _ROMAN takes and this does not. Dropping +# it is a latent widening removed: with the `^\S+\s+` prefix the empty +# alternative would match a name ending in whitespace, which no corpus +# name is today. Dropping it is deliberate: behind "^\S+\s+" and before "$" an # empty alternative would match a name ending in whitespace. No corpus # name does, so this is a latent widening removed rather than a bug # fixed -- and it is the reason the two patterns are NOT byte-identical @@ -2030,8 +2033,9 @@ issue = "fix(suffix-routing) a two-token name ending in the suffix word jr keeps # # The regex also reaches four corpus names this rule does not explain. # 'Kim, Jr.' and 'Smith, Jr.' are fix(#296)'s lone post-comma -# credential and '김민준씨 Jr.' is fix(cjk-glued-honorific-peel)'s; all -# three rules sit above this one and win on file order, which is the +# credential and '김민준씨 Jr.' is fix(cjk-glued-honorific-peel)'s. +# Three names, two rules, and both sit above this one and win on file +# order, which is the # whole reason these four rules were appended at the end. The fourth, # 'Doe,, Jr.', does not diff against this baseline at all, so it has no # winner to lose. That reach is real either way, so none of the four is @@ -2114,8 +2118,9 @@ issue = "fix(suffix-routing) the dotted M.A. spelling reads as a credential (ma- # member `m\.?a\.?` matches the fragment 'M.A.' in the corpus, and # _normalize leaves that as 'm.a', which is not a SUFFIX_ACRONYMS entry # -- so test_latin_alternations_mean_something_the_vocabulary_ships -# rejects it as an alternation member ("a fragment a member matches has -# to BE a vocabulary entry"). The ambiguous-surname-acronym rule above +# rejects it as an alternation member -- its helper +# _reaches_non_vocabulary puts the rule as "every fragment a member +# matches must BE an entry". The ambiguous-surname-acronym rule above # records the same finding and answers it the same way: keep the # periods out of the members. There is no alternation here at all, so # nothing is owed to _LATIN_ALTERNATION_SOURCES. From 6c5b5736cfc11a90e3b3b1e055d8904afaadc526 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 28 Aug 2026 22:45:16 -0700 Subject: [PATCH 15/15] test(guards,differential): pin the jr rule's surplus and the dormant/ban interaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two hardening gaps the #453 review named, plus a third the measuring found. _CROSS_RULE_WINNERS gains three rows for names the jr rule's regex reaches and does not explain: 'Kim, Jr.' and 'Smith, Jr.' are fix(#296)'s, '김민준씨 Jr.' is fix(cjk-glued-honorific-peel)'s. Shapes measured against the 1.4.0 wheel. Nothing else here would report a handover: reach is per-rule, and the jr rule's own _CORPUS_CLAIMS count of 5 does not move when a name inside it changes hands. Mutation-tested -- widening the rule's `fields` AND moving it to the front hands all three over, and all three rows fire. 'Doe,, Jr.' is the fourth name the regex reaches and gets no row: measured, it does not diff at this baseline, so there is no winner to pin. Measuring corrected the rule's own comment, which this branch had already corrected once. It said the three "sit above this one and win on file order". They do sit above, but order is not what protects them: their diffs are {family, given, suffix, title} and {family, given, suffix}, neither a subset of the jr rule's two roles, so that rule is INELIGIBLE wherever it sits. Order is the second line, and handing one over takes two edits rather than one. The comment says that now. test_dormant_does_not_buy_an_exemption_from_the_ban pins that a valid `dormant` does not short-circuit the #451 check -- the plausible future edit being "it is declared idle, so its reach cannot matter". It can: `dormant` is a claim about today's corpus, and the retired catch-all explained four names when it was written. The test also pins the other direction, that a MALFORMED `dormant` still reports its own message, since check order is what makes both hold. Verified the ban is the only thing rejecting that rule: add a name_regex and it validates. 6063 passed; gates unmoved (229/0, 194/0, 102/0). Refs #451 --- tests/v2/test_differential.py | 34 ++++++++++++++++++++ tests/v2/test_ledger_guards.py | 18 +++++++++++ tools/differential/expected_since_1.4.0.toml | 15 ++++++--- 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index 058608e1..f211193a 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -228,6 +228,40 @@ def test_a_rule_with_fields_and_no_regex_is_rejected() -> None: "test_ledger.toml") +def test_dormant_does_not_buy_an_exemption_from_the_ban() -> None: + """`dormant` says a rule explains nothing; it does not say the rule + may be unbounded. + + The two are independent, and conflating them is the plausible + future edit: "it is declared idle, so its reach cannot matter." + It can. `dormant` is a claim about TODAY's corpus, and #372's + lesson is that a rule's reach is what it will claim tomorrow -- + the retired catch-all explained four names when it was written. + A regexless rule declared dormant would sit at the whole corpus + the moment one diffing name arrived, and its `dormant` reason + would then be false as well as its bound missing. + + Pinned because check ORDER is what makes this hold: the dormancy + check runs before the #451 one, so a malformed `dormant` still + reports its own message, and a well-formed one falls through to + the ban. Nothing else asserts that a valid `dormant` does not + short-circuit it (#453 review). + """ + with pytest.raises(SystemExit, match="no 'name_regex'"): + compare.validate_rules( + [{"issue": "fix(x) idle and unbounded", + "fields": ["given", "family"], + "dormant": "a reason nobody could fault"}], + "test_ledger.toml") + # the other order still holds: a malformed `dormant` reports its + # own defect rather than the ban's, because it is checked first + with pytest.raises(SystemExit, match="'dormant' that is not a"): + compare.validate_rules( + [{"issue": "fix(x) idle and unbounded", + "fields": ["given", "family"], "dormant": ""}], + "test_ledger.toml") + + def test_a_rule_with_a_regex_and_no_fields_or_both_stays_legal() -> None: """The neighbouring shapes #451 did NOT retire. `name_regex` alone still narrows by name; `name_regex` plus `fields` narrows by both. diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index b701a211..860772b1 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -2031,6 +2031,24 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: # they are in the same tier and both reach the name -- and a # later edit that moves either one silently hands it back. (".,", ("given",)): "fix(A2) content-free input names nobody, so every role empties", + # The jr rule's surplus, added by the #453 review. Its regex + # reaches these three and does not explain them; `fields` is + # what makes it ineligible -- none of the shapes below is a + # subset of its {family, suffix} -- and file order is only the + # second line. Recorded because handing one over takes TWO + # edits (widen `fields`, move the rule up) and nothing else + # here would report the pair: reach is per-rule, and the jr + # rule's own _CORPUS_CLAIMS count of 5 does not move when a + # name inside it changes hands. Shapes measured against the + # 1.4.0 wheel, not guessed. 'Doe,, Jr.' is the fourth name the + # regex reaches and has no row: it does not diff at this + # baseline, so there is no winner to pin. + ("Kim, Jr.", ("family", "given", "suffix", "title")): + "fix(#296) a lone post-comma credential is a suffix", + ("Smith, Jr.", ("family", "given", "suffix", "title")): + "fix(#296) a lone post-comma credential is a suffix", + ("김민준씨 Jr.", ("family", "given", "suffix")): + "fix(cjk-glued-honorific-peel) glued honorific peels into suffix", }, } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 0a25d369..85c6ca27 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -2034,11 +2034,16 @@ issue = "fix(suffix-routing) a two-token name ending in the suffix word jr keeps # The regex also reaches four corpus names this rule does not explain. # 'Kim, Jr.' and 'Smith, Jr.' are fix(#296)'s lone post-comma # credential and '김민준씨 Jr.' is fix(cjk-glued-honorific-peel)'s. -# Three names, two rules, and both sit above this one and win on file -# order, which is the -# whole reason these four rules were appended at the end. The fourth, -# 'Doe,, Jr.', does not diff against this baseline at all, so it has no -# winner to lose. That reach is real either way, so none of the four is +# Three names, two rules, and what keeps them there is `fields`, not +# file order: measured, they diff {family, given, suffix, title} and +# {family, given, suffix}, and neither is a subset of the two roles +# below, so this rule is INELIGIBLE for all three wherever it sits. +# Order is the second line of defence, not the first -- a widening of +# `fields` alone would still lose to the rules above, and it would +# take a widening AND a move to hand any of them over. +# _CROSS_RULE_WINNERS pins all three against that pair of edits. The +# fourth, 'Doe,, Jr.', does not diff against this baseline at all, so +# it has no winner to lose and no row. That reach is real either way, so none of the four is # a _MUST_NOT_MATCH probe here: that roster tests the regex, not # classify(). name_regex = "(?i)^\\S+\\s+jr\\.?$"