fix: preserve source text and bound JSON escape expansion

BACKLOG.md

@@ -55,6 +55,5 @@ records, generated release manifests, or the owning docs named above.
 | DEFERRED | VALUE-01 | Admit exact value-evidence comparisons only after a real producer and downstream consumer establish the public record boundary; detailed candidate contract is retained in [issue #65](https://github.com/research-engineering/agentic-proofkit/issues/65). | A real execution-receipt projection, baseline producer, and downstream consumer prove an exact producer-output-to-admission round trip plus compact/full-graph inclusion or an intentional omission non-claim; otherwise no public command is added. |
 | BLOCKED | RELOCATION-01 | Add provenance-bounded witness relocation candidates without introducing a second binding path or trusting a caller-authored prior digest; detailed candidate contract is retained in [issue #66](https://github.com/research-engineering/agentic-proofkit/issues/66). | An owner-admitted content-addressed baseline binds witness id, prior path and digest, source revision, evidence class, authentication non-claims, and freshness non-claims; the scanner then proves the zero/one/many match partition while remaining non-current until fresh execution evidence exists. |
 | BLOCKED | RELEASE-01 | Prove signed protected-tag release policy as provider-side release governance, not source-only intent. | Repository tag protection/ruleset and release workflow variables require signed annotated release tags; the next public release records provider-side evidence or the row is explicitly retired as an accepted non-claim. |
-| BLOCKED | RELEASE-02 | Retire the inaccurate PyPI `0.1.159` wheel compatibility and license projection without mutating immutable release history. | After a public replacement release proves that each advertised macOS wheel minimum is no lower than its embedded Mach-O minimum, embedded MIT license identity, npm/PyPI/GitHub byte closure, and installed-package smoke, yank PyPI `0.1.159` with an exact compatibility-and-license reason and retain provider evidence of the yank. |
 | DEFERRED | INSTALLED-CONSUMER-01 | Evaluate one carrier-neutral installed-contract and route-verification protocol without merging npm process transport with Python module transport. | Exact npm and wheel decision tables plus a shared mutant corpus first prove behavioral equivalence for contract admission, route/help identity, and byte-bound carrier checks; extract only the proven common protocol while retaining carrier-specific installation and execution owners, or retire the row if the common layer does not reduce semantic duplication. |
 | DEFERRED | CLI-ARGS-01 | Evaluate one immutable typed parse result between descriptor admission and command execution instead of independently interpreting already-admitted command operands. | A reproducible descriptor-versus-handler drift falsifier establishes the defect class; a bounded prototype proves exact flag, multiplicity, value, help, input, and presentation parity across every affected command with no new ambient authority or generic option bag; otherwise retain the current bounded parsers and retire the row. |

internal/kernel/requirementsourcecodec/limit_manifest_test.go

@@ -3,6 +3,7 @@ package requirementsourcecodec
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"os"
 	"reflect"
 	"sort"
@@ -69,21 +70,32 @@ func TestLimitArithmeticRejectsOverflow(t *testing.T) {
 }
 
 func TestCanonicalByteBoundCoversWorstAdmittedEscapeExpansion(t *testing.T) {
+	for _, scalar := range []string{"\x00", "\u0085"} {
+		t.Run(fmt.Sprintf("U+%04X", []rune(scalar)[0]), func(t *testing.T) {
+			assertMaximalEscapedTextRoundTrip(t, scalar)
+		})
+	}
+}
+
+func assertMaximalEscapedTextRoundTrip(t *testing.T, scalar string) {
+	t.Helper()
 	limits := compactTestModelLimits()
 	low := 0
-	high := limits.MaxTotalTextBytes/2 + 1
+	high := limits.MaxTotalTextBytes/len(scalar) + 1
 	for low+1 < high {
 		middle := low + (high-low)/2
 		draft := testDraft()
-		draft.NonClaimDefinitions[0].Statement = "X" + strings.Repeat("\u0085", middle) + "Y"
+		draft.NonClaimDefinitions[0].Statement = "X" + strings.Repeat(scalar, middle) + "Y"
 		if _, err := requirementsourcemodel.NormalizeWithLimits(draft, limits); err == nil {
 			low = middle
-		} else {
+		} else if requirementsourcemodel.ErrorCode(err) == "text_budget_exceeded" {
 			high = middle
+		} else {
+			t.Fatalf("boundary search encountered an unrelated rejection: %v", err)
 		}
 	}
 	draft := testDraft()
-	draft.NonClaimDefinitions[0].Statement = "X" + strings.Repeat("\u0085", low) + "Y"
+	draft.NonClaimDefinitions[0].Statement = "X" + strings.Repeat(scalar, low) + "Y"
 	model, err := requirementsourcemodel.NormalizeWithLimits(draft, limits)
 	if err != nil {
 		t.Fatalf("maximum admitted escape fixture error = %v", err)
@@ -96,11 +108,16 @@ func TestCanonicalByteBoundCoversWorstAdmittedEscapeExpansion(t *testing.T) {
 	if int64(len(payload)) > codecLimits.MaxOutputBytes {
 		t.Fatalf("canonical bytes = %d, bound = %d", len(payload), codecLimits.MaxOutputBytes)
 	}
-	if !bytes.Contains(payload, []byte(`\u0085`)) {
-		t.Fatal("worst-case admitted control scalar was not escaped")
+	escaped := []byte(fmt.Sprintf(`\u%04x`, []rune(scalar)[0]))
+	if low == 0 || bytes.Count(payload, escaped) != low {
+		t.Fatal("maximal control text was not escaped exactly")
+	}
+	parsed, err := ParseWithLimits(payload, codecLimits, limits)
+	if err != nil || !projectionsEqual(model, parsed.Model) {
+		t.Fatal("maximal admitted text did not round trip under paired limits")
 	}
 	over := testDraft()
-	over.NonClaimDefinitions[0].Statement = "X" + strings.Repeat("\u0085", high) + "Y"
+	over.NonClaimDefinitions[0].Statement = "X" + strings.Repeat(scalar, high) + "Y"
 	if _, err := requirementsourcemodel.NormalizeWithLimits(over, limits); requirementsourcemodel.ErrorCode(err) != "text_budget_exceeded" {
 		t.Fatalf("limit-plus-one model error = %v", err)
 	}

internal/kernel/requirementsourcecodec/limits.go

@@ -86,7 +86,8 @@ func MaxLexicalTokens(limits requirementsourcemodel.Limits) (int, error) {
 
 func canonicalByteCoefficients(limits requirementsourcemodel.Limits) []limitCoefficient {
 	return []limitCoefficient{
-		{ID: "total_text_bytes", Count: limits.MaxTotalTextBytes, Coefficient: 3},
+		// A one-byte control scalar expands to six bytes in a JSON Unicode escape.
+		{ID: "total_text_bytes", Count: limits.MaxTotalTextBytes, Coefficient: 6},
 		{ID: "collection_items", Count: limits.MaxCollectionItems, Coefficient: 32},
 		{ID: "definitions", Count: limits.MaxDefinitions, Coefficient: 96},
 		{ID: "terms", Count: limits.MaxTerms, Coefficient: 160},

internal/kernel/requirementsourcecodec/testdata/codec-limit-coefficients.v1.json

@@ -15,7 +15,7 @@
     {"id": "profiles", "coefficient": 448},
     {"id": "scenarios", "coefficient": 896},
     {"id": "terms", "coefficient": 160},
-    {"id": "total_text_bytes", "coefficient": 3}
+    {"id": "total_text_bytes", "coefficient": 6}
   ],
   "lexicalTokenCoefficients": [
     {"id": "collection_items", "coefficient": 32},

internal/kernel/requirementsourcecodec/text_identity_test.go

@@ -0,0 +1,105 @@
+package requirementsourcecodec
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/research-engineering/agentic-proofkit/internal/kernel/requirementsourcemodel"
+)
+
+func TestTextRolesAndStableScenarioIdentityRoundTrip(t *testing.T) {
+	const completion = "accept\nrequests\twith \"quotes\", \\ paths,\r\nUnicode \U0001f680 and \x00 data."
+	const boundary = "TODO denotes caller terminology,\nnot missing behavior."
+	const review = "Review TBD wording\nafter owner confirmation."
+	const scenarioID = "proofkit.package-boundary.root-export-and-deep-import-denial"
+	draft := testDraft()
+	draft.Groups[0].Members[0].StatementCompletion = completion
+	draft.NonClaimDefinitions[0].Statement = boundary
+	draft.Groups[1].Members[0].Fields.Deferral.Value.ReviewCondition = review
+	draft.Scenarios[0].ScenarioID = scenarioID
+	model, err := requirementsourcemodel.Normalize(draft)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wire, err := Format(model)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !json.Valid(wire) || bytes.ContainsRune(wire, '\x00') {
+		t.Fatal("formatter did not safely escape text")
+	}
+	parsed, err := Parse(wire)
+	if err != nil {
+		t.Fatal(err)
+	}
+	atomic := parsed.Model.Atomic()
+	if atomic.Requirements[0].Invariant != "The service must "+completion ||
+		atomic.NonClaimDefinitions[0].Statement != boundary ||
+		atomic.Requirements[2].Deferral.ReviewCondition != review ||
+		atomic.Scenarios[0].ScenarioID != scenarioID {
+		t.Fatal("reader and writer agreed but lost an independently expected value")
+	}
+	if !projectionsEqual(parsed.Model, model) {
+		t.Fatal("codec lost an atomic, layout or reference projection")
+	}
+	location, ok := parsed.SourceMap.Location("/groups/1/members/0/statementCompletion")
+	if !ok {
+		t.Fatal("multiline text lacks a lexical source location")
+	}
+	var replay string
+	if err := json.Unmarshal(wire[location.ValueSpan.Start:location.ValueSpan.End], &replay); err != nil || replay != completion {
+		t.Fatal("source-map span cannot replay the exact authored text")
+	}
+	second, err := Format(parsed.Model)
+	if err != nil || !bytes.Equal(second, wire) {
+		t.Fatal("canonical multiline representation is not idempotent")
+	}
+}
+
+func TestWireTextPoliciesRejectWithoutDisclosure(t *testing.T) {
+	roles := []struct {
+		name string
+		set  func(map[string]any, string)
+	}{
+		{"invariant", func(root map[string]any, s string) {
+			groups := root["groups"].([]any)
+			group := groups[1].(map[string]any)
+			group["members"].([]any)[0].(map[string]any)["statementCompletion"] = s
+		}},
+		{"nonclaim", func(root map[string]any, s string) {
+			root["nonClaimDefinitions"].([]any)[0].(map[string]any)["statement"] = s
+		}},
+		{"review", func(root map[string]any, s string) {
+			group := root["groups"].([]any)[0].(map[string]any)
+			fields := group["members"].([]any)[0].(map[string]any)["fields"].(map[string]any)
+			fields["deferral"].(map[string]any)["reviewCondition"] = s
+		}},
+	}
+	const secret = "ghp_0123456789abcdefghijklmnopqrstuvwxyz"
+	for _, role := range roles {
+		t.Run(role.name, func(t *testing.T) {
+			wire := mutateRoot(t, mustPayload(t), func(root map[string]any) {
+				role.set(root, "token="+secret)
+			})
+			_, err := Parse(wire)
+			if ErrorCode(err) != "invalid_text" || strings.Contains(err.Error(), secret) {
+				t.Fatal("wire admission did not reject secret text without disclosure")
+			}
+		})
+	}
+}
+
+func TestWireStableScenarioIdentityRejectsDuplicates(t *testing.T) {
+	wire := mutateRoot(t, mustPayload(t), func(root map[string]any) {
+		scenarios := root["scenarios"].([]any)
+		scenario := scenarios[0].(map[string]any)
+		scenario["scenarioId"] = "proofkit.package-boundary.root-export-and-deep-import-denial"
+		root["scenarios"] = append(scenarios, scenario)
+	})
+	_, err := Parse(wire)
+	if ErrorCode(err) != "duplicate_id" {
+		t.Fatalf("duplicate scenario wire identity was not rejected: %v", err)
+	}
+}

internal/kernel/requirementsourcemodel/metadata.go

@@ -108,7 +108,7 @@ func normalizeDeferral(value *Deferral, path string) (*Deferral, error) {
 	if err != nil {
 		return nil, err
 	}
-	reviewCondition, err := canonicalText(value.ReviewCondition, path+".reviewCondition", false, true)
+	reviewCondition, err := canonicalText(value.ReviewCondition, path+".reviewCondition", false, false)
 	if err != nil {
 		return nil, err
 	}

internal/kernel/requirementsourcemodel/normalize.go

@@ -107,7 +107,7 @@ func normalizeDefinitions(values []NonClaimDefinition) ([]NonClaimDefinition, ma
 		if _, exists := ids[id]; exists {
 			return nil, nil, invalid("duplicate_id", "nonClaimDefinitions")
 		}
-		statement, err := canonicalText(value.Statement, path+"statement", false, true)
+		statement, err := canonicalText(value.Statement, path+"statement", false, false)
 		if err != nil {
 			return nil, nil, err
 		}

internal/kernel/requirementsourcemodel/package_boundary_test.go

@@ -60,6 +60,7 @@ func TestRepresentationNeutralPackageBoundaryIsExact(t *testing.T) {
 		"regexp",
 		"sort",
 		"strings",
+		"unicode/utf8",
 	}
 	if !reflect.DeepEqual(actualImports, expectedImports) {
 		t.Fatalf("production direct imports = %v, representation-neutral allowlist = %v", actualImports, expectedImports)

internal/kernel/requirementsourcemodel/scenario_normalization.go

@@ -10,7 +10,7 @@ func normalizeScenarios(values []Scenario, requirements map[string]AtomicRequire
 	ids := make(map[string]struct{}, len(values))
 	for index, value := range values {
 		path := indexed("scenarios", index, "")
-		scenarioID, err := canonicalID(value.ScenarioID, "SCN-", path+"scenarioId")
+		scenarioID, err := canonicalExternalID(value.ScenarioID, path+"scenarioId")
 		if err != nil {
 			return nil, err
 		}

internal/kernel/requirementsourcemodel/text_identity_test.go

@@ -0,0 +1,105 @@
+package requirementsourcemodel
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNormalizePreservesTextBySemanticRole(t *testing.T) {
+	const completion = "accept\nrequests\twith \"quoted\" \\ values,\r\nUnicode \U0001f680 and \x00 data."
+	const boundary = "TODO is a caller label,\nnot an implementation claim."
+	const review = "Review the TBD policy\nafter the owner meeting."
+	draft := validDraft()
+	draft.Groups[0].Members[0].StatementCompletion = completion
+	draft.NonClaimDefinitions[0].Statement = boundary
+	draft.Groups[1].Members[0].Fields.Deferral.Value.ReviewCondition = review
+	model, err := Normalize(draft)
+	if err != nil {
+		t.Fatal(err)
+	}
+	atomic := model.Atomic()
+	if atomic.Requirements[0].Invariant != "The service must "+completion {
+		t.Fatal("normalization changed invariant bytes")
+	}
+	if atomic.NonClaimDefinitions[0].Statement != boundary || atomic.Requirements[2].Deferral.ReviewCondition != review {
+		t.Fatal("operational text was rejected or rewritten as an invariant")
+	}
+	for _, group := range model.Layout().Groups {
+		for _, member := range group.Members {
+			if member.RequirementID == "REQ-MODEL-001" && member.StatementCompletion != completion {
+				t.Fatal("layout lost original completion bytes")
+			}
+		}
+	}
+}
+
+func TestNormalizeTextSafetyRemainsRoleIndependent(t *testing.T) {
+	roles := []struct {
+		name string
+		set  func(*Draft, string)
+	}{
+		{"invariant", func(d *Draft, s string) { d.Groups[0].Members[0].StatementCompletion = s }},
+		{"nonclaim", func(d *Draft, s string) { d.NonClaimDefinitions[0].Statement = s }},
+		{"review", func(d *Draft, s string) { d.Groups[1].Members[0].Fields.Deferral.Value.ReviewCondition = s }},
+	}
+	for _, role := range roles {
+		for _, value := range []struct{ name, text string }{
+			{"secret", "token=ghp_0123456789abcdefghijklmnopqrstuvwxyz"},
+			{"invalid_utf8", "invalid-\xff-text"},
+			{"outer_space", " untrimmed caller text "},
+		} {
+			t.Run(role.name+"/"+value.name, func(t *testing.T) {
+				draft := validDraft()
+				role.set(&draft, value.text)
+				_, err := Normalize(draft)
+				if ErrorCode(err) != "invalid_text" {
+					t.Fatalf("error code = %q, want invalid_text", ErrorCode(err))
+				}
+				if strings.Contains(err.Error(), value.text) {
+					t.Fatal("diagnostic disclosed caller text")
+				}
+			})
+		}
+	}
+	for _, placeholder := range []string{"TODO", "fixme", "TBD"} {
+		draft := validDraft()
+		draft.Groups[0].Members[0].StatementCompletion = "accept " + placeholder + " requests."
+		if _, err := Normalize(draft); ErrorCode(err) != "placeholder_text" {
+			t.Fatalf("unfinished invariant accepted: code=%q", ErrorCode(err))
+		}
+	}
+}
+
+func TestNormalizeScenarioIdentityUsesStableRuleDomain(t *testing.T) {
+	const id = "proofkit.package-boundary.root-export-and-deep-import-denial"
+	draft := validDraft()
+	draft.Scenarios[0].ScenarioID = id
+	model, err := Normalize(draft)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := model.Atomic().Scenarios[0].ScenarioID; got != id {
+		t.Fatalf("scenario ID changed to %q", got)
+	}
+	edges := 0
+	for _, edge := range model.References().Edges {
+		if edge.From.Kind == EntityScenario {
+			if edge.From.ID != id {
+				t.Fatal("scenario reference origin was renamed")
+			}
+			edges++
+		}
+	}
+	if edges != 3 {
+		t.Fatalf("scenario reference edges = %d, want 3", edges)
+	}
+	draft.Scenarios = append(draft.Scenarios, draft.Scenarios[0])
+	if _, err := Normalize(draft); ErrorCode(err) != "duplicate_id" {
+		t.Fatalf("duplicate scenario code = %q", ErrorCode(err))
+	}
+	draft = validDraft()
+	draft.Scenarios[0].ScenarioID = "invalid scenario identity"
+	if _, err := Normalize(draft); ErrorCode(err) != "invalid_id" {
+		t.Fatalf("invalid scenario code = %q", ErrorCode(err))
+	}
+}

internal/kernel/requirementsourcemodel/validation.go

@@ -5,6 +5,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/research-engineering/agentic-proofkit/internal/kernel/admit"
 )
@@ -61,14 +62,9 @@ func canonicalText(value string, path string, allowEmpty bool, rejectPlaceholder
 		return "", nil
 	}
 	admitted, err := admit.NonEmptyText(value, path)
-	if err != nil || admitted != value {
+	if err != nil || admitted != value || !utf8.ValidString(value) {
 		return "", invalid("invalid_text", path)
 	}
-	for _, character := range value {
-		if character < 0x20 || character == 0x7f {
-			return "", invalid("invalid_text", path)
-		}
-	}
 	if rejectPlaceholders && placeholderPattern.MatchString(value) {
 		return "", invalid("placeholder_text", path)
 	}

internal/tools/releasechange/record_test.go

@@ -197,8 +197,8 @@ func TestCurrentChangeRecordNamesReviewedSemanticChanges(t *testing.T) {
 var currentBreakingChanges = []Change{}
 
 var currentAdditions = []Change{
-	{ChangeID: "proofkit.adoption.capability-input-guide", Summary: "Expose two complete synthetic capability-map inputs and field relationships through existing command help. Code-observation adoption tasks link to this guide on demand without embedding examples or inventing missing witnesses. Accepted input sets, report shapes and candidate-only authority are unchanged; the adoption task text and native-source contract digests are updated."},
-	{ChangeID: "proofkit.package.capability-guide-execution", Summary: "Verify that installed npm and Python entrypoints expose the exact command-owned input guide and admit both examples through stdin. Whole-CLI negative tests preserve missing-witness, reference, malformed-input and nondisclosure boundaries. These checks do not execute consumer tests or establish product correctness."},
+	{ChangeID: "proofkit.source-model.lexical-parity", Summary: "Repair the private candidate source model to preserve canonical internal UTF-8 text, including line breaks and safely escaped controls, while rejecting invalid UTF-8 and secret-shaped values. Keep unfinished-invariant checks separate from non-claim and deferral text, and admit existing stable scenario identifiers without requiring a new prefix. Public source admission, CLI contracts and execution-binding ownership are unchanged; no source-v2 cutover is included."},
+	{ChangeID: "proofkit.source-codec.escape-budget", Summary: "Correct the private codec byte bound for the six-byte JSON escape of a one-byte control scalar. Native tests now exercise maximal admitted control text, the next rejected input unit, exact escaping and a complete model-format-parse round trip, alongside duplicate scenario and nondisclosure controls. This does not change published platform requirements or expose a new source format."},
 }
 
 var currentMigrationSteps = []string{}
@@ -221,7 +221,7 @@ func validateCurrentChangeRecord(record Record, notes string) error {
 
 func currentExpectedReleaseNotes() string {
 	lines := []string{
-		"# @research-engineering/agentic-proofkit 0.14.2",
+		"# @research-engineering/agentic-proofkit 0.14.3",
 		"",
 		"## Breaking Contract Changes",
 		"",
@@ -274,7 +274,7 @@ func currentExpectedReleaseNotes() string {
 		"Primary npm channel:",
 		"",
 		"```bash",
-		"npm install --save-dev --save-exact @research-engineering/agentic-proofkit@0.14.2",
+		"npm install --save-dev --save-exact @research-engineering/agentic-proofkit@0.14.3",
 		"```",
 		"",
 		"Pre-1.0 npm consumers must keep this dependency exact-pinned.",
@@ -286,7 +286,7 @@ func currentExpectedReleaseNotes() string {
 		"## Rollback",
 		"",
 		"- First follow the migration and persistent-state compatibility restrictions above; changing a package pin does not roll back repository state.",
-		"- Pin npm consumers to the previous admitted version 0.14.1 with `npm install --save-dev --save-exact @research-engineering/agentic-proofkit@0.14.1`.",
+		"- Pin npm consumers to the previous admitted version 0.14.2 with `npm install --save-dev --save-exact @research-engineering/agentic-proofkit@0.14.2`.",
 		"- Treat local package artifacts as candidates until registry identity is proven.",
 	)
 	return strings.Join(lines, "\n") + "\n"