-
Notifications
You must be signed in to change notification settings - Fork 0
fix(rollup): a reviewer that never ran must not report zero findings #566
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -243,10 +243,29 @@ func sumDurations(workstreams []WorkstreamUsage, field func(WorkstreamUsage) *in | |
| return &total | ||
|
piekstra marked this conversation as resolved.
|
||
| } | ||
|
|
||
| func writeReviewerTable(out *strings.Builder, reviewers []ReviewerSummary) { | ||
| // writeReviewerTable renders the headline per-reviewer counts. | ||
| // | ||
| // A reviewer that never produced a result must not be shown as "0". Zero | ||
| // findings and "did not run" are the same number and opposite meanings: the | ||
| // first says the code is clean, the second says nothing was examined. Rendering | ||
| // both as 0 let a run where four of five reviewers failed to start read as a | ||
| // clean review, with the failure visible only further down in the coverage | ||
| // section that a reader skimming the summary never reaches. | ||
| func writeReviewerTable(out *strings.Builder, reviewers []ReviewerSummary, coverage []ReviewerCoverageSummary) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. writeReviewerTable still builds its own local Reply inline to this comment. |
||
| produced := make(map[string]bool, len(coverage)) | ||
| for _, entry := range coverage { | ||
| produced[entry.AgentID] = coverageResultProduced(entry.Status) | ||
| } | ||
| out.WriteString("| Reviewer | Findings |\n") | ||
| out.WriteString("|----------|----------|\n") | ||
| for _, reviewer := range reviewers { | ||
| // Absent from coverage means nothing was reported either way; only an | ||
| // explicit non-producing status is called out, so this cannot mask a | ||
| // genuine zero. | ||
| if ran, known := produced[reviewer.Name]; known && !ran { | ||
| fmt.Fprintf(out, "| %s | ⚠️ did not run |\n", escapeCell(reviewer.Name)) | ||
| continue | ||
| } | ||
| fmt.Fprintf(out, "| %s | %d |\n", escapeCell(reviewer.Name), reviewer.Findings) | ||
| } | ||
| out.WriteString("\n") | ||
|
|
@@ -316,6 +335,18 @@ func writeReviewerCoverageDiagnostics(out *strings.Builder, coverage []ReviewerC | |
| out.WriteString("\n") | ||
| } | ||
|
|
||
| // ReviewersProducedResults maps each reviewer to whether it actually produced | ||
| // a result. Both the rendered rollup and the JSON view derive their | ||
| // "did not run" state from this, so the two cannot disagree -- which is what | ||
| // Summary's contract promises and what a markdown-only fix would have broken. | ||
| func ReviewersProducedResults(coverage []ReviewerCoverageSummary) map[string]bool { | ||
| produced := make(map[string]bool, len(coverage)) | ||
| for _, entry := range coverage { | ||
| produced[entry.AgentID] = coverageResultProduced(entry.Status) | ||
| } | ||
| return produced | ||
| } | ||
|
|
||
| func coverageResultProduced(status string) bool { | ||
| switch strings.TrimSpace(status) { | ||
| case "complete_broad", "complete_constrained", "incomplete_skipped": | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package reviewplan | ||
|
|
||
| import ( | ||
| "strings" | ||
| "testing" | ||
| ) | ||
|
|
||
| // A reviewer that never produced a result must not appear as "0". | ||
| // | ||
| // Zero findings and "did not run" are the same number and opposite meanings: | ||
| // one says the code is clean, the other says nothing was examined. Rendering | ||
| // both as 0 let a run where most reviewers failed to start read as a clean | ||
| // review, with the failure visible only in a coverage section further down. | ||
| func TestReviewerTableDistinguishesFailureFromZeroFindings(t *testing.T) { | ||
| reviewers := []ReviewerSummary{ | ||
| {Name: "security:code-auditor", Findings: 0}, // failed to start | ||
| {Name: "documentation:docs", Findings: 0}, // genuinely found nothing | ||
| } | ||
| coverage := []ReviewerCoverageSummary{ | ||
| {AgentID: "security:code-auditor", Status: "incomplete_failed"}, | ||
| {AgentID: "documentation:docs", Status: "complete_broad"}, | ||
| } | ||
|
|
||
| var out strings.Builder | ||
| writeReviewerTable(&out, reviewers, coverage) | ||
| got := out.String() | ||
|
|
||
| for _, line := range strings.Split(got, "\n") { | ||
| if !strings.Contains(line, "security:code-auditor") { | ||
| continue | ||
| } | ||
| if strings.Contains(line, "| 0 |") { | ||
| t.Fatalf("a reviewer that did not run is reported as zero findings: %q", line) | ||
| } | ||
| if !strings.Contains(line, "did not run") { | ||
| t.Fatalf("failed reviewer row does not say it did not run: %q", line) | ||
| } | ||
| } | ||
|
|
||
| // The reviewer that really did run must still show its honest zero. | ||
| if !strings.Contains(got, "| documentation:docs | 0 |") { | ||
| t.Fatalf("a completed reviewer lost its zero count:\n%s", got) | ||
| } | ||
| } | ||
|
|
||
| // A reviewer absent from coverage keeps its count: unknown status must not be | ||
| // reported as a failure, or genuine zeros start reading as breakage. | ||
| func TestReviewerTableKeepsCountWhenCoverageIsUnknown(t *testing.T) { | ||
| var out strings.Builder | ||
| writeReviewerTable(&out, | ||
| []ReviewerSummary{{Name: "policies:conventions", Findings: 0}}, | ||
| nil, | ||
| ) | ||
| if !strings.Contains(out.String(), "| policies:conventions | 0 |") { | ||
| t.Fatalf("unknown coverage should leave the count alone:\n%s", out.String()) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package view | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/open-cli-collective/codereview-cli/internal/reviewplan" | ||
| ) | ||
|
|
||
| // The JSON view must carry the same did-not-run distinction as the rendered | ||
| // rollup. Summary's contract is that both consumers agree; a markdown-only fix | ||
| // would leave a failed reviewer serializing as findings: 0, which is the exact | ||
| // ambiguity being removed. | ||
| func TestReviewSummaryJSONDistinguishesFailedReviewer(t *testing.T) { | ||
| summary := reviewplan.Summary{ | ||
| Reviewers: []reviewplan.ReviewerSummary{ | ||
| {Name: "security:code-auditor", Findings: 0}, | ||
| {Name: "documentation:docs", Findings: 0}, | ||
| {Name: "policies:conventions", Findings: 0}, | ||
| }, | ||
| Run: reviewplan.RunSummary{ | ||
| ReviewerCoverage: []reviewplan.ReviewerCoverageSummary{ | ||
| {AgentID: "security:code-auditor", Status: "incomplete_failed"}, | ||
| {AgentID: "documentation:docs", Status: "complete_broad"}, | ||
| // policies:conventions absent: status unknown. | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| raw, err := json.Marshal(newReviewSummary(summary)) | ||
| if err != nil { | ||
| t.Fatalf("marshal: %v", err) | ||
| } | ||
| got := string(raw) | ||
|
|
||
| if !strings.Contains(got, `"name":"security:code-auditor","findings":0,"ran":false`) { | ||
| t.Fatalf("failed reviewer must serialize ran:false, got:\n%s", got) | ||
| } | ||
| if !strings.Contains(got, `"name":"documentation:docs","findings":0,"ran":true`) { | ||
| t.Fatalf("completed reviewer must serialize ran:true, got:\n%s", got) | ||
| } | ||
| // Unknown coverage omits the field rather than guessing a failure. | ||
| if !strings.Contains(got, `"name":"policies:conventions","findings":0}`) { | ||
| t.Fatalf("unknown coverage must omit ran, got:\n%s", got) | ||
| } | ||
| } |
Uh oh!
There was an error while loading. Please reload this page.