From 5d3fd7f412600b3aad068f76f27490cc8188b373 Mon Sep 17 00:00:00 2001 From: Jeff Musgrave <1578618+JeffMusgrave@users.noreply.github.com> Date: Sat, 17 Oct 2026 02:57:05 -0400 Subject: [PATCH 1/2] Preserve bounded Rust syntax evidence through release gates Prepare 0.29.0-beta.3 with governed tool discovery, bounded per-source syntax streaming, and the shared Python program transport. Keep gate plans, command logs, and failure records intact. Validation: targeted engine, Rust facts, quality, and architecture suites pass; format and corrected source policy check pass. Cloud Devbox diagnostic validates 207 of 211 Rust files; four pinned-grammar failures remain explicit. Final gate, native CI, publication, and Fable review remain pending; Fable returned a usage-credit limit. --- .code-polishy.json | 3 +- CHANGELOG.md | 11 ++ VERSION | 2 +- docs/design/cargo-facts.md | 16 +++ docs/design/execution-observability.md | 10 ++ docs/design/historical-review-policy.md | 3 + internal/architecture/rust_test.go | 6 + internal/engine/engine.go | 6 +- internal/engine/gate_command_runner.go | 53 ++++++-- internal/engine/gate_stream_test.go | 134 +++++++++++++++++++ internal/engine/governed_environment_test.go | 44 ++++++ internal/engine/merge_gate.go | 11 ++ internal/quality/cargo_test.go | 6 + internal/quality/rust_format_test.go | 19 ++- internal/rustfacts/source.go | 33 ++++- internal/rustfacts/source_batch_test.go | 46 ++++++- internal/rustfacts/source_parser.py | 30 +++-- internal/rustfacts/source_stream.go | 88 ++++++++++++ internal/rustfacts/source_stream_test.go | 97 ++++++++++++++ 19 files changed, 576 insertions(+), 42 deletions(-) create mode 100644 internal/engine/gate_stream_test.go create mode 100644 internal/rustfacts/source_stream.go create mode 100644 internal/rustfacts/source_stream_test.go diff --git a/.code-polishy.json b/.code-polishy.json index f5cf222c..931d2c83 100644 --- a/.code-polishy.json +++ b/.code-polishy.json @@ -332,6 +332,7 @@ "internal/engine/report_types.go", "internal/engine/format.go", "internal/engine/gate_command_runner.go", + "internal/engine/gate_stream_test.go", "internal/engine/gate_run.go", "internal/engine/test_evidence.go", "internal/gaterun/run.go", @@ -449,7 +450,7 @@ { "name": "rustfacts", "paths": ["internal/rustfacts/**"], - "dependsOn": ["policy", "repository", "runner", "release"] + "dependsOn": ["policy", "pythonfacts", "repository", "runner", "release"] }, { "name": "runner", diff --git a/CHANGELOG.md b/CHANGELOG.md index fe43d50b..25e030d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +## 0.29.0-beta.3 - 2026-10-17 + +- Resolve configured prerequisites through the installed tool inventory, including + the optional Cargo companion, without relying on ambient PATH entries. +- Stream Rust syntax facts as bounded source records so expanded results larger + than 8 MiB retain complete coverage. Oversized individual results remain + explicit coverage failures while valid sibling sources remain available. +- Route Rust syntax workers through the shared Python program transport and + preserve their planned command identities, bounded logs, and failed attempts + throughout gate execution. + ## 0.29.0-beta.2 - 2026-10-16 - Recognize pinned OSV-Scanner, ty and Python tools on Windows, where files diff --git a/VERSION b/VERSION index 443690c1..ca3b989f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.29.0-beta.2 +0.29.0-beta.3 diff --git a/docs/design/cargo-facts.md b/docs/design/cargo-facts.md index df8d8665..158b638c 100644 --- a/docs/design/cargo-facts.md +++ b/docs/design/cargo-facts.md @@ -108,6 +108,16 @@ oversized source or a failed batch produces coverage findings for the affected paths without discarding valid siblings. Cancellation prevents later batches from starting and leaves their files explicitly uncovered. +Responses stream one newline-terminated `rust-source/v2` record per requested +source, in request order, through the common runner's writer interface. Each +record is limited to 8388608 bytes including its terminator and validated before +the next is retained; diagnostics remain limited to 65536 bytes. Syntax facts +can expand beyond their input size without creating a repository-wide response +ceiling. An individual source whose facts exceed the record budget emits an +explicit incomplete result, preserving its valid siblings. A missing, repeated, +foreign, malformed, oversized or unterminated record, or a failed worker, +invalidates its batch. No partial stream establishes complete coverage. + Locations unpack Tree-sitter's point tuples and translate byte columns into original Unicode coordinates. The pinned Python binding's [point attribute getters](https://github.com/tree-sitter/py-tree-sitter/blob/v0.26.0/tree_sitter/binding/point.c) @@ -309,3 +319,9 @@ probe before atomic replacement. Corrupt downloads, invalid payloads, failed probes and replacement failures preserve a previous working executable. Reuse checks the installed release; it does not hash local state. This provisions a runner, not test evidence or doctest execution. + +The syntax worker uses the shared Python program transport: a bounded program +record precedes the request on standard input. Command arguments contain only +the fixed bootstrap, so multiline source remains valid in gate identities and +does not depend on platform command-line limits. Rust facts depend on the +existing Python transport owner rather than maintaining a second bootstrap. diff --git a/docs/design/execution-observability.md b/docs/design/execution-observability.md index c25f6ac5..687dce03 100644 --- a/docs/design/execution-observability.md +++ b/docs/design/execution-observability.md @@ -28,6 +28,16 @@ identifies both scope and cost. The observation wrapper forwards structured process results unchanged to suite execution. Recording an exit status while discarding it from the caller would make native test-artifact validation disagree with the command that produced it. +Configured prerequisites resolve installed release tools through the same +governed tool inventory as execution. Their registered absolute paths do not +depend on ambient PATH entries, and an ambient executable cannot conceal a +missing member of an installed tool bundle. Unregistered external commands +retain ordinary host discovery. +Gate execution forwards typed streams through the same planned-command, log, +and attempt-recording path. Consumers validate bounded records incrementally; +the diagnostic log retains its existing bounded head and tail. Streaming does +not introduce an aggregate capture limit or permit reuse without the facts. +Consumer failures remain failed command attempts. Bounded capture overflow is an operational failure even when the child exits with a normal test-failure code. A partial stream cannot establish native outcome evidence or a regression-proof red result. Cancellation and timeout retain their diff --git a/docs/design/historical-review-policy.md b/docs/design/historical-review-policy.md index dd375063..4aa9e89b 100644 --- a/docs/design/historical-review-policy.md +++ b/docs/design/historical-review-policy.md @@ -19,6 +19,9 @@ historical ownership relations explicitly for impact analysis, without inventing a focused suite or changing the candidate's ownership rules. Suite defaults preserve the command identity used by historical review evidence. Only the candidate configuration governs current checks and test execution. +The planned runner enforces the same command identity for captured and streamed +analyzers; stream forwarding retains the gate attempt and log boundary described +in [Execution observability](execution-observability.md). The engine reads the snapshot from the bounded regular Git blob at the exact base revision. An absent configuration selects first-adoption behavior; diff --git a/internal/architecture/rust_test.go b/internal/architecture/rust_test.go index 427e07a4..8471feb1 100644 --- a/internal/architecture/rust_test.go +++ b/internal/architecture/rust_test.go @@ -2,6 +2,7 @@ package architecture import ( "context" + "io" "path/filepath" "slices" "strings" @@ -298,3 +299,8 @@ func (boundary *rustGraphRecordingRunner) RunStructured(ctx context.Context, roo boundary.names = append(boundary.names, command.Name) return boundary.OSRunner.RunStructured(ctx, root, command) } + +func (boundary *rustGraphRecordingRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + boundary.names = append(boundary.names, command.Name) + return boundary.OSRunner.RunWithWriters(ctx, root, command, stdout, stderr) +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 3d6fe3a9..4dcc0004 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -1021,8 +1021,10 @@ func checkedInCommandFinding(repo repository.Repository, name, executable string } func configuredToolPath(repo repository.Repository, executable string) string { - if executable == "go" || executable == "gofmt" { - return repo.GoTool(executable) + for _, tool := range repo.CommandEnvironment().Tools { + if tool.Name == executable { + return tool.Path + } } return executable } diff --git a/internal/engine/gate_command_runner.go b/internal/engine/gate_command_runner.go index 5a202bde..4d1335aa 100644 --- a/internal/engine/gate_command_runner.go +++ b/internal/engine/gate_command_runner.go @@ -106,20 +106,49 @@ func (commandRunner *gateArtifactRunner) Run(ctx context.Context, root string, c } func (commandRunner *gateArtifactRunner) RunWithResult(ctx context.Context, root string, command policy.Command) (runner.Result, error) { - result, _, err := commandRunner.runNext(ctx, root, command, false, nil) + result, _, err := commandRunner.runNext(ctx, root, command, gateCommandOutput{}) return result, err } func (commandRunner *gateArtifactRunner) RunWithOutput(ctx context.Context, root string, command policy.Command) (runner.Result, runner.Output, error) { - return commandRunner.runNext(ctx, root, command, true, nil) + return commandRunner.runNext(ctx, root, command, gateCommandOutput{capture: true}) } func (commandRunner *gateArtifactRunner) RunWithReportOutput(ctx context.Context, root string, command policy.Command, accept runner.ReportOutcomeAcceptor) (runner.Result, runner.Output, error) { - return commandRunner.runNext(ctx, root, command, true, accept) + return commandRunner.runNext(ctx, root, command, gateCommandOutput{capture: true, accept: accept}) } func (commandRunner *gateArtifactRunner) RunStructured(ctx context.Context, root string, command policy.Command) (runner.Result, runner.Output, error) { - return commandRunner.runNext(ctx, root, command, true, nil) + return commandRunner.runNext(ctx, root, command, gateCommandOutput{capture: true}) +} + +func (commandRunner *gateArtifactRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + if stdout == nil { + stdout = io.Discard + } + if stderr == nil { + stderr = io.Discard + } + result, _, err := commandRunner.runNext(ctx, root, command, gateCommandOutput{stdout: stdout, stderr: stderr}) + return result, err +} + +type gateCommandOutput struct { + capture bool + accept runner.ReportOutcomeAcceptor + stdout io.Writer + stderr io.Writer +} + +func (output gateCommandOutput) run(ctx context.Context, delegate runner.Runner, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, runner.Output, error) { + if output.stdout != nil { + if _, ok := delegate.(runner.StreamRunner); !ok { + return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, runner.Output{}, fmt.Errorf("gate command %q requires streamed output", command.Name) + } + stdout = io.MultiWriter(stdout, output.stdout) + stderr = io.MultiWriter(stderr, output.stderr) + } + return runGateCommand(ctx, delegate, root, command, stdout, stderr, output.capture || testartifact.NeedsCommandOutput(command)) } func (commandRunner *gateArtifactRunner) ManagesTestArtifacts() bool { return true } @@ -188,7 +217,7 @@ func (commandRunner *gateArtifactRunner) TestRetryRunner() runner.Runner { return &gateRetryRunner{parent: commandRunner} } -func (commandRunner *gateArtifactRunner) runNext(ctx context.Context, root string, command policy.Command, captureOutput bool, accept runner.ReportOutcomeAcceptor) (runner.Result, runner.Output, error) { +func (commandRunner *gateArtifactRunner) runNext(ctx context.Context, root string, command policy.Command, output gateCommandOutput) (runner.Result, runner.Output, error) { if commandRunner.err != nil { return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, runner.Output{}, commandRunner.err } @@ -206,7 +235,7 @@ func (commandRunner *gateArtifactRunner) runNext(ctx context.Context, root strin fmt.Fprintf(commandRunner.progress, "REUSE %s\n", command.Name) return runner.Result{ExitStatus: 0}, runner.Output{}, nil } - return commandRunner.execute(ctx, root, index, command, captureOutput, false, accept) + return commandRunner.execute(ctx, root, index, command, output, false) } func (commandRunner *gateArtifactRunner) plannedIndex(root string, command policy.Command) (int, error) { @@ -233,7 +262,7 @@ func (commandRunner *gateArtifactRunner) plannedIndex(root string, command polic return index, nil } -func (commandRunner *gateArtifactRunner) execute(ctx context.Context, root string, index int, command policy.Command, captureOutput, diagnostic bool, accept runner.ReportOutcomeAcceptor) (runner.Result, runner.Output, error) { +func (commandRunner *gateArtifactRunner) execute(ctx context.Context, root string, index int, command policy.Command, options gateCommandOutput, diagnostic bool) (runner.Result, runner.Output, error) { fmt.Fprintf(commandRunner.progress, "RUN %s\n", command.Name) actual, err := commandRunner.prepareCommand(index, command) if err != nil { @@ -245,7 +274,7 @@ func (commandRunner *gateArtifactRunner) execute(ctx context.Context, root strin commandRunner.err = err return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, runner.Output{}, err } - result, output, runErr := runGateCommand(ctx, commandRunner.delegate, root, actual, commandLog.Stdout(), commandLog.Stderr(), captureOutput || testartifact.NeedsCommandOutput(actual)) + result, output, runErr := options.run(ctx, commandRunner.delegate, root, actual, commandLog.Stdout(), commandLog.Stderr()) runErr = captureGateTestOutput(ctx, actual, &result, output, runErr) artifacts, artifactErr := commandRunner.validateTestArtifacts(index, actual, &result) runErr = errors.Join(runErr, artifactErr) @@ -254,7 +283,7 @@ func (commandRunner *gateArtifactRunner) execute(ctx context.Context, root strin commandRunner.err = logErr return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, output, errors.Join(runErr, logErr) } - status, category, reportBearing := gateCommandAttempt(ctx, result, output, runErr, accept) + status, category, reportBearing := gateCommandAttempt(ctx, result, output, runErr, options.accept) gateCategory := gaterun.FailureCategory("") if status == gaterun.Failed { gateCategory = gaterun.FailureCategory(category) @@ -345,7 +374,7 @@ func (commandRunner *gateArtifactRunner) runDiagnostic(ctx context.Context, root commandRunner.err = errors.Join(commandRunner.err, err) return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, err } - result, _, err := commandRunner.execute(ctx, root, index, command, false, true, nil) + result, _, err := commandRunner.execute(ctx, root, index, command, gateCommandOutput{}, true) return result, err } @@ -357,7 +386,7 @@ func (commandRunner *gateArtifactRunner) runRetry(ctx context.Context, root stri commandRunner.err = errors.Join(commandRunner.err, err) return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, err } - result, _, err := commandRunner.execute(ctx, root, index, command, false, false, nil) + result, _, err := commandRunner.execute(ctx, root, index, command, gateCommandOutput{}, false) return result, err } @@ -488,7 +517,7 @@ func (commandRunner *gateArtifactRunner) runTestPrerequisite(ctx context.Context if _, failed := commandRunner.failedTests[suite]; failed { for index, expected := range commandRunner.expected { if expected.Category == gaterun.Check && samePolicyCommand(expected.Command, command) { - return commandRunner.execute(ctx, root, index, command, true, false, nil) + return commandRunner.execute(ctx, root, index, command, gateCommandOutput{capture: true}, false) } } } diff --git a/internal/engine/gate_stream_test.go b/internal/engine/gate_stream_test.go new file mode 100644 index 00000000..ee3a2db6 --- /dev/null +++ b/internal/engine/gate_stream_test.go @@ -0,0 +1,134 @@ +package engine + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/riteofstring/code-polishy/internal/gaterun" + "github.com/riteofstring/code-polishy/internal/repository" + "github.com/riteofstring/code-polishy/internal/runner" + "github.com/riteofstring/code-polishy/internal/rustfacts" +) + +func TestGateStreamsLargeRustFactsAndRecordsBoundedLogs(t *testing.T) { + t.Parallel() + repo := repository.Repository{Root: t.TempDir(), PolicyRoot: enginePolicyRoot(t)} + var source strings.Builder + for index := range 1200 { + fmt.Fprintf(&source, "pub fn function_%d() {}\n", index) + } + paths := make([]string, 20) + for index := range paths { + paths[index] = fmt.Sprintf("source_%02d.rs", index) + writeEngineFile(t, repo.Root, paths[index], source.String(), 0o600) + } + plan := rustfacts.PlanSources(repo, paths) + boundary := rustSourceGate(t, repo, plan) + planned := &mergeGatePlannedRunner{root: repo.Root, delegate: boundary, expected: boundary.expected} + result := rustfacts.ReadSources(t.Context(), repo, plan, planned) + if len(result.Findings) != 0 || len(result.Files) != len(paths) || planned.next != 1 { + t.Fatalf("gate lost streamed syntax evidence: files=%d findings=%+v", len(result.Files), result.Findings) + } + for index, file := range result.Files { + if file.Path != paths[index] || len(file.Functions) != 1200 || file.Functions[1199].Name != "function_1199" { + t.Fatalf("gate changed parsed function inventory for %s", file.Path) + } + } + outcome := finishRustSourceGate(t, boundary, gaterun.RunPassed) + data, err := os.ReadFile(filepath.Join(repo.Root, outcome.Attempts[0].LogPath)) + if err != nil { + t.Fatal(err) + } + var log struct { + StdoutBytes int64 `json:"stdout_bytes"` + StdoutTruncated bool `json:"stdout_truncated"` + StdoutTail []byte `json:"stdout_tail"` + } + if err := json.Unmarshal(data, &log); err != nil { + t.Fatal(err) + } + if log.StdoutBytes <= 8<<20 || !log.StdoutTruncated || !strings.Contains(string(log.StdoutTail), "function_1199") { + t.Fatalf("gate lost its bounded large-stream log: bytes=%d truncated=%v", log.StdoutBytes, log.StdoutTruncated) + } +} + +func TestGateRecordsStreamConsumerFailures(t *testing.T) { + t.Parallel() + repo := repository.Repository{Root: t.TempDir(), PolicyRoot: enginePolicyRoot(t)} + writeEngineFile(t, repo.Root, "source.rs", "pub fn example() {}\n", 0o600) + plan := rustfacts.PlanSources(repo, []string{"source.rs"}) + boundary := rustSourceGate(t, repo, plan) + planned := &mergeGatePlannedRunner{root: repo.Root, delegate: boundary, expected: boundary.expected} + result, err := planned.RunWithWriters(t.Context(), repo.Root, plan.Commands[0], rejectingGateStream{}, io.Discard) + if err == nil || result.FailureCategory != runner.FailureOperational { + t.Fatalf("consumer failure was accepted: result=%+v error=%v", result, err) + } + outcome := finishRustSourceGate(t, boundary, gaterun.RunFailed) + if outcome.Attempts[0].FailureCategory != gaterun.FailureCategory(runner.FailureOperational) { + t.Fatalf("gate lost the stream failure category: %+v", outcome) + } +} + +func TestGateRejectsUnplannedRustStream(t *testing.T) { + t.Parallel() + repo := repository.Repository{Root: t.TempDir(), PolicyRoot: enginePolicyRoot(t)} + writeEngineFile(t, repo.Root, "source.rs", "pub fn example() {}\n", 0o600) + plan := rustfacts.PlanSources(repo, []string{"source.rs"}) + boundary := rustSourceGate(t, repo, plan) + planned := &mergeGatePlannedRunner{root: repo.Root, delegate: boundary, expected: boundary.expected} + command := plan.Commands[0] + command.Stdin = append(append([]byte{}, command.Stdin...), '\n') + result, err := planned.RunWithWriters(t.Context(), repo.Root, command, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "request") || result.FailureCategory != runner.FailureOperational || boundary.next != 0 { + t.Fatalf("unplanned stream reached gate execution: result=%+v next=%d error=%v", result, boundary.next, err) + } +} + +type rejectingGateStream struct{} + +func (rejectingGateStream) Write([]byte) (int, error) { + return 0, errors.New("syntax consumer rejected the output") +} + +func rustSourceGate(t *testing.T, repo repository.Repository, plan rustfacts.SourcePlan) *gateArtifactRunner { + t.Helper() + if len(plan.Findings) != 0 || len(plan.Commands) != 1 { + t.Fatalf("unexpected parser plan: %+v", plan.Findings) + } + planned := MergeGateExecutionCommand{Root: repo.Root, Category: gaterun.Check, Scope: "repository", Cost: "quick", Command: plan.Commands[0]} + identity, err := gaterun.NewIdentity(gaterun.IdentityInput{ + Gate: gaterun.MergeGate, RequestedBase: "main", ExactBase: strings.Repeat("a", 40), Candidate: strings.Repeat("b", 40), PolicyLevel: "recommended", + Release: gaterun.ReleaseIdentity{Version: "test", Digest: strings.Repeat("a", 64)}, ConfigurationSHA256: strings.Repeat("b", 64), + Platform: gaterun.Platform{OS: "test", Arch: "test"}, Commands: []gaterun.CommandSpec{gateRunCommandSpec(planned, nil)}, + Environment: []gaterun.EnvironmentInput{}, AmbientEnvironment: []gaterun.EnvironmentInput{}, BehaviorReview: testGateRunBehaviorReview(), + }) + if err != nil { + t.Fatal(err) + } + run, err := gaterun.Start(gaterun.StartOptions{RepositoryRoot: repo.Root, Identity: identity}) + if err != nil { + t.Fatal(err) + } + return &gateArtifactRunner{delegate: runner.OSRunner{}, run: run, expected: []MergeGateExecutionCommand{planned}, progress: io.Discard} +} + +func finishRustSourceGate(t *testing.T, boundary *gateArtifactRunner, status gaterun.RunStatus) gaterun.CommandOutcome { + t.Helper() + report, err := boundary.run.Finalize(gaterun.FinalizeOptions{Status: status, BehaviorReview: testGateRunBehaviorReview()}) + if err != nil { + t.Fatal(err) + } + if _, err := gaterun.LoadReport(boundary.expected[0].Root, report.Identity); err != nil { + t.Fatal(err) + } + if len(report.Commands) != 1 || len(report.Commands[0].Attempts) != 1 || string(report.Commands[0].Status) != string(status) { + t.Fatalf("gate did not record its command attempt: %+v", report.Commands) + } + return report.Commands[0] +} diff --git a/internal/engine/governed_environment_test.go b/internal/engine/governed_environment_test.go index b41b36a9..2e7bad73 100644 --- a/internal/engine/governed_environment_test.go +++ b/internal/engine/governed_environment_test.go @@ -3,12 +3,56 @@ package engine import ( "os" "path/filepath" + "slices" "strings" "testing" + "github.com/riteofstring/code-polishy/internal/policy" "github.com/riteofstring/code-polishy/internal/repository" ) +func TestDoctorFindsInstalledRustToolsWithoutAmbientPath(t *testing.T) { + root := contentRepository(t, nil) + policyEngine, err := Open(root, root, "") + if err != nil { + t.Fatal(err) + } + writeEngineFile(t, root, "tools/rust-version.txt", "1.97.1\n", 0o600) + tool := policyEngine.Repository.RustTool("cargo") + if err := os.MkdirAll(filepath.Dir(tool), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tool, []byte("fixture executable"), 0o700); err != nil { + t.Fatal(err) + } + policyEngine.Repository.Config.Checks = []policy.Command{{Name: "native-build", Argv: []string{"cargo", "build"}}} + t.Setenv("PATH", t.TempDir()) + report, err := policyEngine.Doctor(t.Context()) + if err != nil { + t.Fatal(err) + } + if slices.ContainsFunc(report.Findings, func(finding policy.Finding) bool { + return finding.Check == "policy.tool" && finding.Subject == "cargo" + }) { + t.Fatalf("installed Cargo was reported missing: %+v", report.Findings) + } + if err := os.Remove(tool); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(os.Getenv("PATH"), filepath.Base(tool)), []byte("ambient executable"), 0o700); err != nil { + t.Fatal(err) + } + report, err = policyEngine.Doctor(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(report.Findings, func(finding policy.Finding) bool { + return finding.Check == "policy.tool" && finding.Subject == "cargo" + }) { + t.Fatal("ambient Cargo concealed an incomplete governed installation") + } +} + func TestGovernedEnvironmentKeepsRustCompanionOptionalAndComplete(t *testing.T) { root := contentRepository(t, nil) repo := repository.Repository{Root: root, PolicyRoot: root} diff --git a/internal/engine/merge_gate.go b/internal/engine/merge_gate.go index c4679c8d..407ddf1b 100644 --- a/internal/engine/merge_gate.go +++ b/internal/engine/merge_gate.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "reflect" "slices" "time" @@ -615,6 +616,16 @@ func (commandRunner *mergeGatePlannedRunner) RunStructured(ctx context.Context, return runner.Result{ExitStatus: -1}, runner.Output{}, fmt.Errorf("merge gate command runner cannot capture structured output for %q", command.Name) } +func (commandRunner *mergeGatePlannedRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + if err := commandRunner.start(root, command); err != nil { + return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, err + } + if observed, ok := commandRunner.delegate.(runner.StreamRunner); ok { + return observed.RunWithWriters(ctx, root, command, stdout, stderr) + } + return runner.Result{ExitStatus: -1, FailureCategory: runner.FailureOperational}, fmt.Errorf("merge gate command runner cannot stream output for %q", command.Name) +} + func (commandRunner *mergeGatePlannedRunner) start(root string, command policy.Command) error { if commandRunner.err != nil { return commandRunner.err diff --git a/internal/quality/cargo_test.go b/internal/quality/cargo_test.go index 01be5696..c1dfbb89 100644 --- a/internal/quality/cargo_test.go +++ b/internal/quality/cargo_test.go @@ -2,6 +2,7 @@ package quality import ( "context" + "io" "path/filepath" "reflect" "slices" @@ -106,3 +107,8 @@ func (boundary *cargoRecordingRunner) RunStructured(ctx context.Context, root st boundary.names = append(boundary.names, command.Name) return boundary.OSRunner.RunStructured(ctx, root, command) } + +func (boundary *cargoRecordingRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + boundary.names = append(boundary.names, command.Name) + return boundary.OSRunner.RunWithWriters(ctx, root, command, stdout, stderr) +} diff --git a/internal/quality/rust_format_test.go b/internal/quality/rust_format_test.go index 9053e80b..3007ef31 100644 --- a/internal/quality/rust_format_test.go +++ b/internal/quality/rust_format_test.go @@ -3,6 +3,7 @@ package quality import ( "context" "errors" + "io" "os" "path/filepath" "slices" @@ -193,15 +194,20 @@ type rustSyntaxFailureRunner struct { } func (boundary *rustSyntaxFailureRunner) RunStructured(ctx context.Context, root string, command policy.Command) (runner.Result, runner.Output, error) { - if command.Name == "policy-rust-source-facts" { - return runner.Result{}, runner.Output{Stdout: []byte("{}")}, boundary.failure - } if strings.HasPrefix(command.Name, "policy-rustfmt-") { boundary.formatted++ } return boundary.OSRunner.RunStructured(ctx, root, command) } +func (boundary *rustSyntaxFailureRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + if command.Name == "policy-rust-source-facts" { + _, err := io.WriteString(stdout, "{}\n") + return runner.Result{}, errors.Join(err, boundary.failure) + } + return boundary.OSRunner.RunWithWriters(ctx, root, command, stdout, stderr) +} + type rustEditingRunner struct { runner.OSRunner before string @@ -215,6 +221,13 @@ func (boundary rustEditingRunner) RunStructured(ctx context.Context, root string return boundary.OSRunner.RunStructured(ctx, root, command) } +func (boundary rustEditingRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + if strings.HasPrefix(command.Name, boundary.before) { + boundary.edit() + } + return boundary.OSRunner.RunWithWriters(ctx, root, command, stdout, stderr) +} + func rustFormatRepository(t *testing.T) repository.Repository { t.Helper() repo := cargoQualityRepository(t) diff --git a/internal/rustfacts/source.go b/internal/rustfacts/source.go index abc5dfae..81780fbb 100644 --- a/internal/rustfacts/source.go +++ b/internal/rustfacts/source.go @@ -7,10 +7,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "slices" "unicode/utf8" "github.com/riteofstring/code-polishy/internal/policy" + "github.com/riteofstring/code-polishy/internal/pythonfacts" "github.com/riteofstring/code-polishy/internal/repository" "github.com/riteofstring/code-polishy/internal/runner" ) @@ -101,14 +103,29 @@ func (plan *SourcePlan) appendBatch(repo repository.Repository, encoded [][]byte request = append(request, sourceRequestPrefix...) request = append(request, bytes.Join(encoded, []byte(","))...) request = append(request, ']', '}') + stdin, err := sourceCommandInput(request) + if err != nil { + start := len(plan.Inputs) - len(encoded) + plan.Findings = append(plan.Findings, sourceBatchFindings(plan.Inputs[start:], err.Error())...) + plan.Inputs = plan.Inputs[:start] + return + } name := "policy-rust-source-facts" if len(plan.Commands) > 0 { name += fmt.Sprintf("-%d", len(plan.Commands)+1) } - plan.Commands = append(plan.Commands, policy.Command{Name: name, Argv: repo.PythonCommand(sourceParser), Cwd: ".", Stdin: request, Managed: true, SealedEnvironment: true, TimeoutSeconds: 60}) + plan.Commands = append(plan.Commands, policy.Command{Name: name, Argv: repo.PythonCommand(pythonfacts.ProgramBootstrap), Cwd: ".", Stdin: stdin, Managed: true, SealedEnvironment: true, TimeoutSeconds: 60}) plan.batchEnds = append(plan.batchEnds, len(plan.Inputs)) } +func sourceCommandInput(request []byte) ([]byte, error) { + input, err := pythonfacts.ProgramInput(sourceParser, bytes.NewReader(request)) + if err != nil { + return nil, err + } + return io.ReadAll(input) +} + func sourceAnalysisSelected(repo repository.Repository, path string) bool { if repo.Language(path) != "rust" || repo.IsGenerated(path) || repo.IsData(path) { return false @@ -121,7 +138,7 @@ func ReadSources(ctx context.Context, repo repository.Repository, plan SourcePla if len(plan.Commands) == 0 { return result } - structured, ok := boundary.(runner.StructuredRunner) + streamed, ok := boundary.(runner.StreamRunner) if !ok { result.Findings = append(result.Findings, sourceBatchFindings(plan.Inputs, "the runner cannot capture bounded Rust syntax facts")...) return result @@ -129,7 +146,7 @@ func ReadSources(ctx context.Context, repo repository.Repository, plan SourcePla start := 0 for index, command := range plan.Commands { end := plan.batchEnds[index] - batch := readSourceBatch(ctx, repo, plan.Inputs[start:end], command, structured) + batch := readSourceBatch(ctx, repo, plan.Inputs[start:end], command, streamed) result.Files = append(result.Files, batch.Files...) result.Findings = append(result.Findings, batch.Findings...) start = end @@ -137,15 +154,17 @@ func ReadSources(ctx context.Context, repo repository.Repository, plan SourcePla return result } -func readSourceBatch(ctx context.Context, repo repository.Repository, inputs []SourceInput, command policy.Command, boundary runner.StructuredRunner) Sources { +func readSourceBatch(ctx context.Context, repo repository.Repository, inputs []SourceInput, command policy.Command, boundary runner.StreamRunner) Sources { if err := ctx.Err(); err != nil { return Sources{Findings: sourceBatchFindings(inputs, "Rust syntax analysis did not run: "+err.Error())} } - _, output, err := boundary.RunStructured(ctx, repo.Root, command) + output := sourceStream{repo: repo, inputs: inputs} + diagnostics := sourceDiagnostics{} + _, err := boundary.RunWithWriters(ctx, repo.Root, command, &output, &diagnostics) if err != nil { - return Sources{Findings: sourceBatchFindings(inputs, fmt.Sprintf("Rust syntax analysis failed: %v; %s", err, diagnosticOutput(output.Stderr)))} + return Sources{Findings: sourceBatchFindings(inputs, fmt.Sprintf("Rust syntax analysis failed: %v; %s", err, diagnosticOutput(diagnostics.Bytes())))} } - sources, err := ParseSources(repo, inputs, output.Stdout) + sources, err := output.finish() if err != nil { return Sources{Findings: sourceBatchFindings(inputs, "invalid Rust syntax evidence: "+err.Error())} } diff --git a/internal/rustfacts/source_batch_test.go b/internal/rustfacts/source_batch_test.go index 61ab19f7..00053482 100644 --- a/internal/rustfacts/source_batch_test.go +++ b/internal/rustfacts/source_batch_test.go @@ -1,10 +1,12 @@ package rustfacts import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "slices" "strings" "testing" @@ -13,6 +15,29 @@ import ( "github.com/riteofstring/code-polishy/internal/runner" ) +func TestRustSourceFactsKeepLargeResultInventories(t *testing.T) { + t.Parallel() + repo := rustRepository(t) + var source strings.Builder + for index := range 1200 { + fmt.Fprintf(&source, "pub fn function_%d() {}\n", index) + } + paths := make([]string, 20) + for index := range paths { + paths[index] = fmt.Sprintf("source_%02d.rs", index) + write(t, repo.Root, paths[index], source.String()) + } + result := ReadSources(t.Context(), repo, PlanSources(repo, paths), runner.OSRunner{}) + if len(result.Findings) != 0 || len(result.Files) != len(paths) { + t.Fatalf("expanded syntax facts lost source coverage: files=%d findings=%+v", len(result.Files), result.Findings) + } + for index, file := range result.Files { + if file.Path != paths[index] || len(file.Functions) != 1200 || file.Functions[1199].Name != "function_1199" { + t.Fatalf("large result lost its original function inventory: path=%s functions=%d", file.Path, len(file.Functions)) + } + } +} + func TestRustSourceFactsBatchLargeScopes(t *testing.T) { t.Parallel() for _, scenario := range []struct { @@ -41,11 +66,15 @@ func TestRustSourceFactsBatchLargeScopes(t *testing.T) { Protocol string `json:"protocol"` Sources []SourceInput `json:"sources"` } - if err := json.Unmarshal(command.Stdin, &request); err != nil { + program, payload, found := bytes.Cut(command.Stdin, []byte("\n")) + if !found || len(program) > 1<<20 { + t.Fatal("parser command omitted its bounded program record") + } + if err := json.Unmarshal(payload, &request); err != nil { t.Fatal(err) } - if len(command.Stdin) > 8<<20 || len(request.Sources) > 4096 || len(request.Sources) == 0 || request.Protocol != SourceProtocol { - t.Fatalf("unbounded or empty parser request: bytes=%d sources=%d protocol=%s", len(command.Stdin), len(request.Sources), request.Protocol) + if len(payload) > 8<<20 || len(request.Sources) > 4096 || len(request.Sources) == 0 || request.Protocol != SourceProtocol { + t.Fatalf("unbounded or empty parser request: bytes=%d sources=%d protocol=%s", len(payload), len(request.Sources), request.Protocol) } for _, source := range request.Sources { seen = append(seen, source.Path) @@ -147,11 +176,18 @@ type sourceBatchRunner struct { after func(int, runner.Output) (runner.Output, error) } -func (boundary *sourceBatchRunner) RunStructured(ctx context.Context, root string, command policy.Command) (runner.Result, runner.Output, error) { +func (boundary *sourceBatchRunner) RunWithWriters(ctx context.Context, root string, command policy.Command, stdout, stderr io.Writer) (runner.Result, error) { boundary.commands = append(boundary.commands, command) result, output, err := boundary.OSRunner.RunStructured(ctx, root, command) if err == nil { output, err = boundary.after(len(boundary.commands), output) } - return result, output, err + if err != nil { + return result, err + } + if _, err := stderr.Write(output.Stderr); err != nil { + return result, err + } + _, err = stdout.Write(output.Stdout) + return result, err } diff --git a/internal/rustfacts/source_parser.py b/internal/rustfacts/source_parser.py index 1c661205..dfbb25af 100644 --- a/internal/rustfacts/source_parser.py +++ b/internal/rustfacts/source_parser.py @@ -722,19 +722,23 @@ def visit(self, node, function, test_context, depth, scope): return children -def source_facts(parser, entry): - source = entry["source"].encode("utf-8") - tree = parser.parse(source) - result = { +def source_result(entry, error=""): + return { "path": entry["path"], "complete": False, - "error": "", + "error": error, "comments": [], "attributes": [], "functions": [], "macros": [], "graph": [], } + + +def source_facts(parser, entry): + source = entry["source"].encode("utf-8") + tree = parser.parse(source) + result = source_result(entry) if tree is None or tree.root_node.has_error: result["error"] = ( "Rust syntax is incomplete or unsupported by the pinned grammar" @@ -771,17 +775,21 @@ def read_request(): def main(): request = read_request() parser = Parser(Language(tree_sitter_rust.language())) - result = { + metadata = { "protocol": PROTOCOL, "pythonVersion": platform.python_version(), "parserVersion": importlib.metadata.version("tree-sitter"), "grammarVersion": importlib.metadata.version("tree-sitter-rust"), - "sources": [source_facts(parser, entry) for entry in request["sources"]], } - output = json.dumps(result) - if len(output.encode("utf-8")) > MAXIMUM_REQUEST: - raise ValueError("Rust syntax response exceeds 8388608 bytes") - sys.stdout.write(output + "\n") + for entry in request["sources"]: + result = source_facts(parser, entry) + output = json.dumps({**metadata, "sources": [result]}) + if len(output.encode("utf-8")) + 1 > MAXIMUM_REQUEST: + result = source_result( + entry, "Rust syntax source record exceeds 8388608 bytes" + ) + output = json.dumps({**metadata, "sources": [result]}) + sys.stdout.write(output + "\n") if __name__ == "__main__": diff --git a/internal/rustfacts/source_stream.go b/internal/rustfacts/source_stream.go new file mode 100644 index 00000000..abe11c49 --- /dev/null +++ b/internal/rustfacts/source_stream.go @@ -0,0 +1,88 @@ +package rustfacts + +import ( + "bytes" + "errors" + "fmt" + + "github.com/riteofstring/code-polishy/internal/repository" +) + +type sourceStream struct { + repo repository.Repository + inputs []SourceInput + sources []Source + pending []byte + err error +} + +func (stream *sourceStream) Write(data []byte) (int, error) { + if stream.err != nil { + return 0, stream.err + } + written := len(data) + for len(data) > 0 { + end := bytes.IndexByte(data, '\n') + terminated := end >= 0 + if !terminated { + end = len(data) + } + if len(stream.pending)+end >= maximumSourceRequest { + stream.err = errors.New("rust syntax source record exceeds 8388608 bytes") + return 0, stream.err + } + stream.pending = append(stream.pending, data[:end]...) + if !terminated { + break + } + if stream.err = stream.consume(); stream.err != nil { + return 0, stream.err + } + stream.pending = stream.pending[:0] + data = data[end+1:] + } + return written, nil +} + +func (stream *sourceStream) consume() error { + index := len(stream.sources) + if index >= len(stream.inputs) { + return errors.New("rust syntax stream contains an unrequested source record") + } + sources, err := ParseSources(stream.repo, stream.inputs[index:index+1], stream.pending) + if err != nil { + return err + } + stream.sources = append(stream.sources, sources[0]) + return nil +} + +func (stream *sourceStream) finish() ([]Source, error) { + if stream.err != nil { + return nil, stream.err + } + if len(stream.pending) != 0 || len(stream.sources) != len(stream.inputs) { + return nil, errors.New("rust syntax stream ended before its complete source inventory") + } + seen := map[string]bool{} + for _, source := range stream.sources { + if seen[source.Path] { + return nil, fmt.Errorf("rust syntax stream repeats source %s", source.Path) + } + seen[source.Path] = true + } + return stream.sources, nil +} + +type sourceDiagnostics struct { + bytes.Buffer +} + +func (diagnostics *sourceDiagnostics) Write(data []byte) (int, error) { + remaining := (64 << 10) - diagnostics.Len() + if len(data) > remaining { + _, _ = diagnostics.Buffer.Write(data[:remaining]) + return remaining, errors.New("rust syntax diagnostics exceed 65536 bytes") + } + return diagnostics.Buffer.Write(data) +} diff --git a/internal/rustfacts/source_stream_test.go b/internal/rustfacts/source_stream_test.go new file mode 100644 index 00000000..e419c645 --- /dev/null +++ b/internal/rustfacts/source_stream_test.go @@ -0,0 +1,97 @@ +package rustfacts + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + + "github.com/riteofstring/code-polishy/internal/policy" + "github.com/riteofstring/code-polishy/internal/runner" +) + +func TestRustSourceStreamRetainsSiblingsOfOversizedFacts(t *testing.T) { + t.Parallel() + repo := rustRepository(t) + var large strings.Builder + for index := range 30000 { + fmt.Fprintf(&large, "pub fn function_%d() {}\n", index) + } + write(t, repo.Root, "before.rs", "pub fn before() {}\n") + write(t, repo.Root, "large.rs", large.String()) + write(t, repo.Root, "after.rs", "pub fn after() {}\n") + result := ReadSources(t.Context(), repo, PlanSources(repo, []string{"before.rs", "large.rs", "after.rs"}), runner.OSRunner{}) + if len(result.Files) != 2 || result.Files[0].Path != "before.rs" || result.Files[1].Path != "after.rs" { + t.Fatalf("oversized facts discarded valid siblings: files=%d findings=%+v", len(result.Files), result.Findings) + } + if len(result.Findings) != 1 || result.Findings[0].Path != "large.rs" || !strings.Contains(result.Findings[0].Message, "source record exceeds 8388608") { + t.Fatalf("oversized source lacks an explicit coverage failure: %+v", result.Findings) + } +} + +func TestRustSourceStreamValidatesCompleteBoundedEvidence(t *testing.T) { + t.Parallel() + repo := rustRepository(t) + write(t, repo.Root, "first.rs", "pub fn café() {}\n") + write(t, repo.Root, "second.rs", "pub fn second() {}\n") + plan := PlanSources(repo, []string{"first.rs", "second.rs"}) + _, output, err := (runner.OSRunner{}).RunStructured(t.Context(), repo.Root, plan.Commands[0]) + if err != nil { + t.Fatal(err) + } + frames := bytes.SplitAfter(output.Stdout, []byte("\n")) + for _, scenario := range []struct { + name string + output []byte + stderr []byte + err error + valid bool + }{ + {name: "chunked records", output: output.Stdout, valid: true}, + {name: "empty stream"}, + {name: "missing record", output: frames[0]}, + {name: "repeated record", output: bytes.Repeat(frames[0], 2)}, + {name: "extra record", output: append(bytes.Clone(output.Stdout), frames[0]...)}, + {name: "unterminated record", output: bytes.TrimSuffix(output.Stdout, []byte("\n"))}, + {name: "malformed record", output: []byte("{\n")}, + {name: "oversized record", output: bytes.Repeat([]byte(" "), 8<<20)}, + {name: "oversized diagnostics", output: output.Stdout, stderr: bytes.Repeat([]byte("x"), (64<<10)+1)}, + {name: "late worker failure", output: output.Stdout, err: errors.New("worker failed after output")}, + } { + t.Run(scenario.name, func(t *testing.T) { + boundary := sourceStreamRunner{stdout: scenario.output, stderr: scenario.stderr, err: scenario.err} + result := ReadSources(t.Context(), repo, plan, boundary) + if scenario.valid { + if len(result.Findings) != 0 || len(result.Files) != 2 || result.Files[0].Functions[0].Name != "café" { + t.Fatalf("chunk boundaries changed valid syntax facts: %+v", result.Findings) + } + } else if len(result.Files) != 0 || len(result.Findings) != 2 { + t.Fatalf("invalid stream established source coverage: files=%d findings=%+v", len(result.Files), result.Findings) + } + }) + } +} + +type sourceStreamRunner struct { + runner.OSRunner + stdout []byte + stderr []byte + err error +} + +func (boundary sourceStreamRunner) RunWithWriters(_ context.Context, _ string, _ policy.Command, stdout, stderr io.Writer) (runner.Result, error) { + for remaining := boundary.stdout; len(remaining) > 0; { + count := min(31, len(remaining)) + if _, err := stdout.Write(remaining[:count]); err != nil { + return runner.Result{}, err + } + remaining = remaining[count:] + } + if _, err := stderr.Write(boundary.stderr); err != nil { + return runner.Result{}, err + } + return runner.Result{}, boundary.err +} From 3a6cac545640859988be36f0e50adc3619cca7da Mon Sep 17 00:00:00 2001 From: Jeff Musgrave <1578618+JeffMusgrave@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:12:50 -0400 Subject: [PATCH 2/2] Keep Rust fact framing portable across hosts Write bounded records as binary UTF-8 with one LF and verify Windows text translation cannot change the frame. Limit tool discovery documentation to Doctor's prerequisite checks. Focused Rust and gate regression tests and formatting pass; release gate and native CI remain pending. --- docs/design/cargo-facts.md | 3 ++- docs/design/execution-observability.md | 10 +++++----- internal/rustfacts/source_parser.py | 10 ++++++---- internal/rustfacts/source_stream_test.go | 21 +++++++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/docs/design/cargo-facts.md b/docs/design/cargo-facts.md index 158b638c..725483ca 100644 --- a/docs/design/cargo-facts.md +++ b/docs/design/cargo-facts.md @@ -109,7 +109,8 @@ paths without discarding valid siblings. Cancellation prevents later batches from starting and leaves their files explicitly uncovered. Responses stream one newline-terminated `rust-source/v2` record per requested -source, in request order, through the common runner's writer interface. Each +source, in request order, through the common runner's writer interface. Binary +UTF-8 output uses a single LF terminator on every host, including Windows. Each record is limited to 8388608 bytes including its terminator and validated before the next is retained; diagnostics remain limited to 65536 bytes. Syntax facts can expand beyond their input size without creating a repository-wide response diff --git a/docs/design/execution-observability.md b/docs/design/execution-observability.md index 687dce03..02669914 100644 --- a/docs/design/execution-observability.md +++ b/docs/design/execution-observability.md @@ -28,11 +28,11 @@ identifies both scope and cost. The observation wrapper forwards structured process results unchanged to suite execution. Recording an exit status while discarding it from the caller would make native test-artifact validation disagree with the command that produced it. -Configured prerequisites resolve installed release tools through the same -governed tool inventory as execution. Their registered absolute paths do not -depend on ambient PATH entries, and an ambient executable cannot conceal a -missing member of an installed tool bundle. Unregistered external commands -retain ordinary host discovery. +Configured prerequisite checks resolve registered executables through the +governed tool inventory. Doctor uses these absolute paths instead of ambient +PATH entries, so an ambient executable cannot conceal a missing member of an +installed tool bundle. Unregistered external commands retain ordinary host +discovery. Gate execution forwards typed streams through the same planned-command, log, and attempt-recording path. Consumers validate bounded records incrementally; the diagnostic log retains its existing bounded head and tail. Streaming does diff --git a/internal/rustfacts/source_parser.py b/internal/rustfacts/source_parser.py index dfbb25af..c9d0fa7e 100644 --- a/internal/rustfacts/source_parser.py +++ b/internal/rustfacts/source_parser.py @@ -783,13 +783,15 @@ def main(): } for entry in request["sources"]: result = source_facts(parser, entry) - output = json.dumps({**metadata, "sources": [result]}) - if len(output.encode("utf-8")) + 1 > MAXIMUM_REQUEST: + output = (json.dumps({**metadata, "sources": [result]}) + "\n").encode("utf-8") + if len(output) > MAXIMUM_REQUEST: result = source_result( entry, "Rust syntax source record exceeds 8388608 bytes" ) - output = json.dumps({**metadata, "sources": [result]}) - sys.stdout.write(output + "\n") + output = (json.dumps({**metadata, "sources": [result]}) + "\n").encode( + "utf-8" + ) + sys.stdout.buffer.write(output) if __name__ == "__main__": diff --git a/internal/rustfacts/source_stream_test.go b/internal/rustfacts/source_stream_test.go index e419c645..9e7be51e 100644 --- a/internal/rustfacts/source_stream_test.go +++ b/internal/rustfacts/source_stream_test.go @@ -13,6 +13,27 @@ import ( "github.com/riteofstring/code-polishy/internal/runner" ) +func TestRustSourceStreamUsesPortableRecordTerminators(t *testing.T) { + t.Parallel() + repo := rustRepository(t) + write(t, repo.Root, "source.rs", "pub fn café() {}\n") + plan := PlanSources(repo, []string{"source.rs"}) + command := plan.Commands[0] + command.Argv = append([]string{}, command.Argv...) + command.Argv[4] = "import sys;sys.stdout.reconfigure(newline='\\r\\n');" + command.Argv[4] + _, output, err := (runner.OSRunner{}).RunStructured(t.Context(), repo.Root, command) + if err != nil { + t.Fatal(err) + } + if bytes.ContainsRune(output.Stdout, '\r') || !bytes.HasSuffix(output.Stdout, []byte("\n")) { + t.Fatal("Windows text translation changed the bounded syntax record terminator") + } + sources, err := ParseSources(repo, plan.Inputs, output.Stdout) + if err != nil || len(sources) != 1 || len(sources[0].Functions) != 1 || sources[0].Functions[0].Name != "café" { + t.Fatalf("portable framing changed the parsed source: %v", err) + } +} + func TestRustSourceStreamRetainsSiblingsOfOversizedFacts(t *testing.T) { t.Parallel() repo := rustRepository(t)