diff --git a/src/SignsOfAI.Cli/Program.cs b/src/SignsOfAI.Cli/Program.cs index 5cade48..bf27ad8 100644 --- a/src/SignsOfAI.Cli/Program.cs +++ b/src/SignsOfAI.Cli/Program.cs @@ -364,7 +364,12 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor) string Col(string s, int code) => useColor ? $"[{code}m{s}" : s; string Bold(string s) => useColor ? $"{s}" : s; - int scoreColor = r.OverallScore switch { >= 70 => 31, >= 45 => 33, >= 20 => 33, _ => 32 }; + int scoreColor = VerdictBands.Emphasis(r.OverallScore) switch + { + VerdictEmphasis.High => 31, + VerdictEmphasis.Elevated or VerdictEmphasis.Present => 33, + _ => 32, + }; Console.WriteLine(); Console.WriteLine(Bold($" ✍ Signs of AI Writing — {Path.GetFileName(path)}")); Console.WriteLine($" {Col($"{r.OverallScore:0}/100", scoreColor)} {Bold(r.Verdict)} " + diff --git a/src/SignsOfAI.Core/Model/AnalysisResult.cs b/src/SignsOfAI.Core/Model/AnalysisResult.cs index 5a17937..e48572f 100644 --- a/src/SignsOfAI.Core/Model/AnalysisResult.cs +++ b/src/SignsOfAI.Core/Model/AnalysisResult.cs @@ -85,12 +85,19 @@ public sealed record AnalysisResult /// public CitationReport Citations { get; init; } = CitationReport.Empty; - /// Human-readable one-line verdict derived from . - public string Verdict => OverallScore switch + /// + /// Human-readable one-line verdict derived from , in English. + /// + /// English-only on purpose: this is what a machine consumer gets — the CLI's `--json`, the MCP + /// tool's payload — where a stable string is more use than a translated one. Anything shown to a + /// person goes through the interface's localiser or the report's own resources, both of which + /// take their boundary from exactly as this does. + /// + public string Verdict => VerdictBands.Emphasis(OverallScore) switch { - >= 70 => "Strong signs of AI writing", - >= 45 => "Moderate signs of AI writing", - >= 20 => "Light signs of AI writing", + VerdictEmphasis.High => "Strong signs of AI writing", + VerdictEmphasis.Elevated => "Moderate signs of AI writing", + VerdictEmphasis.Present => "Light signs of AI writing", _ => "Reads mostly human", }; } diff --git a/src/SignsOfAI.Core/Model/VerdictBands.cs b/src/SignsOfAI.Core/Model/VerdictBands.cs new file mode 100644 index 0000000..09fd5fa --- /dev/null +++ b/src/SignsOfAI.Core/Model/VerdictBands.cs @@ -0,0 +1,90 @@ +using SignsOfAI.Core.Calibration; + +namespace SignsOfAI.Core.Model; + +/// +/// Where the verdict changes, and the only place that decides it. +/// +/// It used to be decided in eight: the analysis result, the report, the interface's localiser, the +/// CLI's colour picker, two switches in the web page, the batch page and the live-rewrite panel — +/// each with the numbers written out again, kept in step by a comment reading "mirrors the bands in +/// AnalysisResult.Verdict". They did not stay in step, and the drift was not cosmetic: the report +/// withheld the verdict from every document ever analysed while the interface and the CLI printed +/// one for every document ever analysed. One engine gave three answers about the same text. +/// +public static class VerdictBands +{ + /// + /// The score at which this build is willing to say something about a text, taken from the + /// calibration it ships with rather than chosen. + /// + /// Null when no calibration is embedded — a fork that has never measured itself. That case must + /// stay silent rather than inherit a boundary somebody else measured on somebody else's corpus, + /// which is the same rule the report already follows for the error rate. + /// + public static double? Threshold => PublishedCalibration.Current?.RecommendedThreshold; + + /// + /// Whether a score has earned a verdict at all. Below the boundary the number stands alone: a + /// low score is not evidence that a person wrote something, and the wording must not imply it. + /// + public static bool Holds(double score) => Threshold is { } threshold && score >= threshold; + + /// + /// The same question for a document in a named language, which is stricter and has to be. + /// + /// There are three states here, not two, and collapsing them is how this went wrong before: + /// + /// + /// A language in the corpus — English, Spanish — has a measured false-positive bound + /// of its own, even when its sample is too small to set its own boundary. It borrows the pooled + /// boundary and the page prints its bound beside the verdict, so the reader weighs the right + /// number. + /// A language absent from the corpus has no bound to print. A verdict there would + /// imply a reliability nobody measured, and there would be nothing on the page to correct the + /// impression. It gets the score and the reason it gets nothing else. + /// No calibration at all — a fork that has never measured itself — speaks about nothing. + /// + /// + public static bool Holds(double score, string? language) => Holds(score) && Measured(language); + + /// Whether the corpus contains this language at all, however thinly. + public static bool Measured(string? language) => + PublishedCalibration.Current?.For(language) is not null; + + /// + /// How loudly to present a score that has earned a verdict. + /// + /// The two upper cuts are a **display convention and nothing more**. No text in the calibration + /// corpus came within twenty points of them — the highest scoring human text reached 23.4 — so + /// the corpus can locate and can say nothing whatever about 45 or 70. + /// Separating "moderate" from "strong" would need machine-written text, and Docs/CALIBRATION.md + /// argues at length against collecting any: it dates badly and flatters whoever assembles it. + /// + /// They survive here to colour a reading, never to make a claim. See issue #32. + /// + public static VerdictEmphasis Emphasis(double score) => score switch + { + _ when !Holds(score) => VerdictEmphasis.None, + >= 70 => VerdictEmphasis.High, + >= 45 => VerdictEmphasis.Elevated, + _ => VerdictEmphasis.Present, + }; +} + +/// +/// How prominently a verdict is shown. Not a measurement, and deliberately not a number: anything +/// that reads as a quantity here would be read as one that was measured, and only the boundary +/// between and the rest was. +/// +public enum VerdictEmphasis +{ + /// Below the boundary this build can support. The score stands without a verdict. + None, + + Present, + + Elevated, + + High, +} diff --git a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs index 53ee045..ed5421b 100644 --- a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs +++ b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs @@ -373,19 +373,25 @@ public static string FolderToHtml( } /// - /// Whether the score has earned the right to carry a verdict: only at or above the threshold - /// measured for the language actually analysed. Everywhere else the number stands alone. + /// Whether the score has earned the right to carry a verdict. + /// + /// This asked for the threshold measured for the language actually analysed, refusing to borrow + /// the aggregate. That reading was too strict by a wide margin, and the cost was not theoretical: + /// no language in the corpus supports its own threshold — English has 65 texts and Spanish 25, + /// against the ~75 the interval needs — so the condition was false for every document in every + /// language, and the exported report never carried a verdict at all. A tool that renders nothing + /// has not been careful, it has been switched off, and Spanish would have stayed switched off + /// for years while the page still promised a reading. + /// + /// The distinction that resolves it: borrowing the aggregate *error rate* would misstate how + /// often this build is wrong about Spanish (13.3% measured, against 5.6% for English and 4.1% + /// pooled), and that remains forbidden — the caveat below still quotes the language's own bound + /// and never the pooled one. Borrowing the aggregate *boundary* states nothing about reliability; + /// it decides when the tool speaks, and it is published, measured and printed on the page beside + /// the language's own figure. See issue #32. /// - private static bool VerdictHolds(AnalysisResult result) - { - var c = PublishedCalibration.Current; - if (c is null) return false; - - // Never borrow the aggregate. A language absent from the corpus has no supported verdict, - // even when the combined EN/ES sample happens to support one. - return c.For(result.Language)?.RecommendedThreshold is { } threshold - && result.OverallScore >= threshold; - } + private static bool VerdictHolds(AnalysisResult result) => + VerdictBands.Holds(result.OverallScore, result.Language); /// /// The sentence that has to appear on every report. Written from the embedded calibration so it @@ -493,9 +499,9 @@ private static void HowOftenWrong(StringBuilder sb, ReportText text, string? lan private static string Verdict(ReportText text, double score) => text.Get(score switch { - >= 70 => ReportMessages.VerdictStrong, - >= 45 => ReportMessages.VerdictModerate, - >= 20 => ReportMessages.VerdictLight, + _ when VerdictBands.Emphasis(score) is VerdictEmphasis.High => ReportMessages.VerdictStrong, + _ when VerdictBands.Emphasis(score) is VerdictEmphasis.Elevated => ReportMessages.VerdictModerate, + _ when VerdictBands.Emphasis(score) is VerdictEmphasis.Present => ReportMessages.VerdictLight, _ => ReportMessages.VerdictMinimal, }).Text; diff --git a/src/SignsOfAI.UI/Components/LiveRewritePanel.razor b/src/SignsOfAI.UI/Components/LiveRewritePanel.razor index 27c3c5b..ab85da6 100644 --- a/src/SignsOfAI.UI/Components/LiveRewritePanel.razor +++ b/src/SignsOfAI.UI/Components/LiveRewritePanel.razor @@ -269,11 +269,11 @@ if (cursor < Text.Length) yield return new Seg(Text[cursor..], null); } - private static string ScoreClass(double score) => score switch + private static string ScoreClass(double score) => VerdictBands.Emphasis(score) switch { - >= 70 => "danger", - >= 45 => "warn", - >= 20 => "notice", + VerdictEmphasis.High => "danger", + VerdictEmphasis.Elevated => "warn", + VerdictEmphasis.Present => "notice", _ => "good", }; } diff --git a/src/SignsOfAI.UI/Pages/Batch.razor b/src/SignsOfAI.UI/Pages/Batch.razor index 6e62119..f30b20e 100644 --- a/src/SignsOfAI.UI/Pages/Batch.razor +++ b/src/SignsOfAI.UI/Pages/Batch.razor @@ -214,10 +214,12 @@ else } } - private static string BandClass(int score) => score switch + // This said 40 where every other surface said 45 — the drift the single source exists to stop, + // and it had already happened: one document could be coloured two ways by the same build. + private static string BandClass(int score) => VerdictBands.Emphasis(score) switch { - >= 70 => "high", - >= 40 => "medium", + VerdictEmphasis.High => "high", + VerdictEmphasis.Elevated => "medium", _ => "low", }; } diff --git a/src/SignsOfAI.UI/Pages/Home.razor b/src/SignsOfAI.UI/Pages/Home.razor index 6370c93..17ce2c1 100644 --- a/src/SignsOfAI.UI/Pages/Home.razor +++ b/src/SignsOfAI.UI/Pages/Home.razor @@ -715,19 +715,19 @@ else $"signsofai-report-{stamp}.html", EvidenceReport.ToHtml(r, options)); } - private static string ScoreClass(double score) => score switch + private static string ScoreClass(double score) => VerdictBands.Emphasis(score) switch { - >= 70 => "danger", - >= 45 => "warn", - >= 20 => "notice", + VerdictEmphasis.High => "danger", + VerdictEmphasis.Elevated => "warn", + VerdictEmphasis.Present => "notice", _ => "good", }; - private static string ScoreHex(double score) => score switch + private static string ScoreHex(double score) => VerdictBands.Emphasis(score) switch { - >= 70 => "#dc2626", - >= 45 => "#ea580c", - >= 20 => "#ca8a04", + VerdictEmphasis.High => "#dc2626", + VerdictEmphasis.Elevated => "#ea580c", + VerdictEmphasis.Present => "#ca8a04", _ => "#16a34a", }; diff --git a/src/SignsOfAI.UI/Services/Loc.cs b/src/SignsOfAI.UI/Services/Loc.cs index 67a14eb..c0bcbaf 100644 --- a/src/SignsOfAI.UI/Services/Loc.cs +++ b/src/SignsOfAI.UI/Services/Loc.cs @@ -245,12 +245,15 @@ private async Task LogAsync(string message) public string Sev(Severity severity) => this["sev." + severity.ToString().ToLowerInvariant()]; - /// The one-line verdict for an overall score. Mirrors the bands in AnalysisResult.Verdict. - public string Verdict(double score) => score switch + /// + /// The one-line verdict for an overall score, in the interface's language. The boundary comes + /// from ; this only chooses the words for it. + /// + public string Verdict(double score) => VerdictBands.Emphasis(score) switch { - >= 70 => this["verdict.strong"], - >= 45 => this["verdict.moderate"], - >= 20 => this["verdict.light"], + VerdictEmphasis.High => this["verdict.strong"], + VerdictEmphasis.Elevated => this["verdict.moderate"], + VerdictEmphasis.Present => this["verdict.light"], _ => this["verdict.minimal"], }; diff --git a/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs b/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs new file mode 100644 index 0000000..6b2dc4a --- /dev/null +++ b/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs @@ -0,0 +1,88 @@ +using SignsOfAI.Core.Calibration; +using SignsOfAI.Core.Model; +using SignsOfAI.Core.Reporting; + +namespace SignsOfAI.Core.Tests; + +/// +/// One text, one engine, one answer. +/// +/// Nothing in this suite used to notice that the three surfaces disagreed, which is why they were +/// allowed to disagree for weeks. The exported report withheld the verdict from every document ever +/// analysed — its gate demanded a per-language threshold, and no language in the corpus has one — +/// while the CLI and the web interface printed a hand-picked band on every document ever analysed. +/// A text scoring 90/100 was called "Strong signs of AI writing" on screen and given no verdict at +/// all in the document a teacher would print and carry to a committee. +/// +/// These tests exist so that the next disagreement fails a build instead of shipping. +/// +public class VerdictAgreementTests +{ + /// Dense with the tells the packs describe: a score at the top of the scale. + private const string ObviouslyMachine = + "In today's rapidly evolving digital landscape, it is important to note that artificial " + + "intelligence has fundamentally transformed how we approach problem-solving. Moreover, this " + + "comprehensive framework facilitates a robust understanding of the underlying mechanisms. " + + "Furthermore, it is worth noting that such systems not only enhance productivity but also " + + "streamline operations. In order to leverage these capabilities, organizations must delve " + + "into the intricacies of implementation. Additionally, the multifaceted nature of these " + + "tools underscores their pivotal role."; + + [Fact] + public void The_report_speaks_about_a_text_the_product_calls_the_strongest_case_it_has() + { + var result = new AiWritingAnalyzer().Analyze(ObviouslyMachine, "en"); + + // Not a claim that the score is right — only that a document scoring this high is the case + // the tool exists for. If the report stays silent here it stays silent everywhere. + Assert.True(result.OverallScore >= 70, + $"The fixture stopped being an extreme case (scored {result.OverallScore:0}); " + + "pick a stronger one rather than lowering this bar."); + + var report = EvidenceReport.ToMarkdown(result); + + Assert.DoesNotContain("no verdict is given", report); + } + + [Fact] + public void No_language_is_silenced_merely_because_its_own_corpus_is_small() + { + // The per-language threshold needs roughly seventy-five texts. Spanish has twenty-five and + // will not reach seventy-five for a long time. A gate that waits for it does not protect a + // Spanish writer; it withholds the tool from them while serving everyone else — and on this + // corpus it withholds it from everyone, because English has sixty-five. + var calibration = PublishedCalibration.Current; + Assert.NotNull(calibration); + + foreach (var language in calibration!.Languages) + Assert.True(language.RecommendedThreshold is null, + $"'{language.Language}' now supports its own threshold. That is good news and it " + + "makes this test's premise obsolete — keep the fallback, but re-read it."); + + var spanish = new AiWritingAnalyzer().Analyze( + "En el panorama actual, es importante destacar que la inteligencia artificial ha " + + "transformado fundamentalmente nuestro enfoque. Además, este marco integral facilita " + + "una comprensión robusta de los mecanismos subyacentes. Asimismo, cabe señalar que " + + "dichos sistemas no solo mejoran la productividad sino que también optimizan las " + + "operaciones.", "es"); + + var report = EvidenceReport.ToMarkdown(spanish); + + // Whatever it says about Spanish, it must not be silence justified by a missing number that + // the aggregate already supplies. + Assert.Contains("13.3%", report); + } + + [Fact] + public void The_boundary_the_product_draws_is_the_boundary_the_project_publishes() + { + // The product used to say "light signs of AI writing" from 20, five points below the only + // threshold this project publishes, and below the highest-scoring human text in the corpus + // (23.4). Publishing a measured figure while shipping an unmeasured boundary is the + // inconsistency this project exists to complain about in other tools. + var threshold = PublishedCalibration.Current?.RecommendedThreshold; + Assert.NotNull(threshold); + + Assert.Equal(threshold!.Value, VerdictBands.Threshold); + } +}