diff --git a/Docs/CALIBRATION.md b/Docs/CALIBRATION.md index c9828e1..2d76569 100644 --- a/Docs/CALIBRATION.md +++ b/Docs/CALIBRATION.md @@ -73,6 +73,14 @@ Across these groups the median score runs from 7.2 (**es-wikipedia**) down to 4. | 95 | 0 / 90 | 0% | 0% – 4.1% | | 100 | 0 / 90 | 0% | 0% – 4.1% | +## What the product does with this number + +The tool speaks at **25/100** and nowhere else, taking the boundary from the table above rather than from anybody's judgement. Below it a document gets its score and the reason it gets nothing more: a low score is not evidence that a person wrote something, since a detector that detects nothing also returns a low score, and this project has deliberately never measured how much machine writing it catches. The boundary moves when this page moves — including upward if a larger corpus turns out to be less flattering. + +Above it there is **one** verdict, not a scale of them. This corpus can place a boundary and can say nothing whatever about how much further past it a score has travelled: no text known to be human came close to the upper reaches, and grading "moderate" against "strong" would need machine-written text, which the opening of this page argues against collecting. Interfaces do shade a high score more urgently than a low one, and those shades are a display convention — they are not on this page because nothing measured them. + +A language absent from the corpus entirely gets no verdict at all, whatever it scores. A language present but too thin to bound its own rate borrows this boundary and carries its own figure beside it, so the reader weighs the rate that was measured for the writing in front of them rather than the pooled one. + ## Which rules misfire Every rule below fired on text no machine wrote, so each hit is a false positive by construction — there is no judgement call to make. This is the most immediately useful thing the exercise produces: it turns "our rules probably have false positives somewhere" into a ranked list. diff --git a/src/SignsOfAI.Core/Model/AnalysisResult.cs b/src/SignsOfAI.Core/Model/AnalysisResult.cs index e48572f..ccad11e 100644 --- a/src/SignsOfAI.Core/Model/AnalysisResult.cs +++ b/src/SignsOfAI.Core/Model/AnalysisResult.cs @@ -93,11 +93,7 @@ public sealed record AnalysisResult /// person goes through the interface's localiser or the report's own resources, both of which /// take their boundary from exactly as this does. /// - public string Verdict => VerdictBands.Emphasis(OverallScore) switch - { - VerdictEmphasis.High => "Strong signs of AI writing", - VerdictEmphasis.Elevated => "Moderate signs of AI writing", - VerdictEmphasis.Present => "Light signs of AI writing", - _ => "Reads mostly human", - }; + public string Verdict => VerdictBands.Holds(OverallScore) + ? "Signs of AI writing" + : "No signs above the measured boundary"; } diff --git a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs index ed5421b..94c7d38 100644 --- a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs +++ b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs @@ -499,10 +499,8 @@ private static void HowOftenWrong(StringBuilder sb, ReportText text, string? lan private static string Verdict(ReportText text, double score) => text.Get(score switch { - _ when VerdictBands.Emphasis(score) is VerdictEmphasis.High => ReportMessages.VerdictStrong, - _ when VerdictBands.Emphasis(score) is VerdictEmphasis.Elevated => ReportMessages.VerdictModerate, - _ when VerdictBands.Emphasis(score) is VerdictEmphasis.Present => ReportMessages.VerdictLight, - _ => ReportMessages.VerdictMinimal, + _ when VerdictBands.Holds(score) => ReportMessages.VerdictSigns, + _ => ReportMessages.VerdictNone, }).Text; private static string LanguageName(ReportText text, string language) => text.Get( diff --git a/src/SignsOfAI.Core/Reporting/ReportMessages.cs b/src/SignsOfAI.Core/Reporting/ReportMessages.cs index 0a11f65..792920c 100644 --- a/src/SignsOfAI.Core/Reporting/ReportMessages.cs +++ b/src/SignsOfAI.Core/Reporting/ReportMessages.cs @@ -29,10 +29,27 @@ public static class ReportMessages public const string SectionObservations = "section.observations"; public const string SectionErrorRate = "section.error-rate"; public const string SectionUnreadable = "section.unreadable"; - public const string VerdictStrong = "verdict.strong"; - public const string VerdictModerate = "verdict.moderate"; - public const string VerdictLight = "verdict.light"; - public const string VerdictMinimal = "verdict.minimal"; + /// + /// Above the measured boundary. One wording, not three. + /// + /// "Strong", "Moderate" and "Light" read as three measured degrees, and they never were: the + /// corpus locates the boundary and says nothing whatever about 45 or 70, since no text known to + /// be human came within twenty points of either. Separating degrees would need machine-written + /// text, and Docs/CALIBRATION.md argues against ever collecting it. Printing the three words + /// with a footnote admitting they are unmeasured does not fix that — the footnote is read once + /// and the heading is read every time. + /// + public const string VerdictSigns = "verdict.signs"; + + /// + /// Below it. A statement about this tool, never about the writer. + /// + /// The report used to say "Reads mostly human" and the interface "Minimal signs of AI writing" — + /// the same state, two claims, and the first one is not ours to make. A detector that detects + /// nothing also returns a low score, and this project has deliberately never measured how much + /// machine writing it catches, so a low score is evidence about the boundary and nothing else. + /// + public const string VerdictNone = "verdict.none"; public const string AnalysisScoreWithVerdict = "analysis.score.with-verdict"; public const string AnalysisScoreWithoutVerdict = "analysis.score.without-verdict"; public const string AnalysisNoVerdict = "analysis.no-verdict"; @@ -107,10 +124,8 @@ public static class ReportMessages [SectionObservations] = 0, [SectionErrorRate] = 0, [SectionUnreadable] = 0, - [VerdictStrong] = 0, - [VerdictModerate] = 0, - [VerdictLight] = 0, - [VerdictMinimal] = 0, + [VerdictSigns] = 0, + [VerdictNone] = 0, [AnalysisScoreWithVerdict] = 2, [AnalysisScoreWithoutVerdict] = 1, [AnalysisNoVerdict] = 0, @@ -195,10 +210,8 @@ public static class ReportMessages [SectionObservations] = "Found, but at a rate people write at", [SectionErrorRate] = "How often this is wrong", [SectionUnreadable] = "Could not be read", - [VerdictStrong] = "Strong signs of AI writing", - [VerdictModerate] = "Moderate signs of AI writing", - [VerdictLight] = "Light signs of AI writing", - [VerdictMinimal] = "Reads mostly human", + [VerdictSigns] = "Signs of AI writing", + [VerdictNone] = "No signs above the measured boundary", [AnalysisScoreWithVerdict] = "**{0}/100 — {1}**", [AnalysisScoreWithoutVerdict] = "**{0}/100**", [AnalysisNoVerdict] = "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*", @@ -264,7 +277,7 @@ public static class ReportMessages FallbackMarker, FallbackSummary, FallbackLanguage, DefaultTitle, SectionAnalysis, SectionCheckable, SectionCharacters, SectionCitations, SectionSignals, SectionObservations, SectionErrorRate, SectionUnreadable, - VerdictStrong, VerdictModerate, VerdictLight, VerdictMinimal, + VerdictSigns, VerdictNone, AnalysisNoVerdict, LanguageEnglish, LanguageSpanish, LanguageOther, CaveatUncalibrated, CaveatAggregateNoThreshold, CaveatLanguageUnmeasured, diff --git a/src/SignsOfAI.Core/Reporting/report.en.json b/src/SignsOfAI.Core/Reporting/report.en.json index e3763c3..6887e14 100644 --- a/src/SignsOfAI.Core/Reporting/report.en.json +++ b/src/SignsOfAI.Core/Reporting/report.en.json @@ -18,10 +18,8 @@ "section.observations": { "text": "Found, but at a rate people write at" }, "section.error-rate": { "text": "How often this is wrong" }, "section.unreadable": { "text": "Could not be read" }, - "verdict.strong": { "text": "Strong signs of AI writing" }, - "verdict.moderate": { "text": "Moderate signs of AI writing" }, - "verdict.light": { "text": "Light signs of AI writing" }, - "verdict.minimal": { "text": "Reads mostly human" }, + "verdict.signs": { "text": "Signs of AI writing" }, + "verdict.none": { "text": "No signs above the measured boundary" }, "analysis.score.with-verdict": { "text": "**{0}/100 — {1}**" }, "analysis.score.without-verdict": { "text": "**{0}/100**" }, "analysis.no-verdict": { "text": "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*" }, diff --git a/src/SignsOfAI.Core/Reporting/report.es.json b/src/SignsOfAI.Core/Reporting/report.es.json index 8a24442..9fd1efd 100644 --- a/src/SignsOfAI.Core/Reporting/report.es.json +++ b/src/SignsOfAI.Core/Reporting/report.es.json @@ -54,21 +54,13 @@ "text": "No se pudieron leer", "sourceHash": "51a02791dcb61eb8846a00d3f0263a9da516b30b6a92a3b08e34d65af4cef259" }, - "verdict.strong": { - "text": "Señales fuertes de escritura con IA", - "sourceHash": "ee17bc5f7ea18e88a5a4a535d8a7d8f86fbaf3559c0b1927c71eb299ca138eef" + "verdict.signs": { + "text": "Señales de escritura con IA", + "sourceHash": "a03ec88d694c5dee62e6b04c9b46b89cdcb6e5d48b2971dba539fe7a5fbef0fc" }, - "verdict.moderate": { - "text": "Señales moderadas de escritura con IA", - "sourceHash": "78304806a705ef996e3115ba84e0256f7838debf03afccb023e91d299d628a5b" - }, - "verdict.light": { - "text": "Señales leves de escritura con IA", - "sourceHash": "99844fce9fa8df4247389342a41e53d0d231b950cc700f2458207dfef79d5d2e" - }, - "verdict.minimal": { - "text": "Parece escrito mayormente por una persona", - "sourceHash": "4dd5c224203bf31595b61831cbd46ec9f2da352ce359b92ce46262e86cda64ce" + "verdict.none": { + "text": "Sin señales por encima del umbral medido", + "sourceHash": "dfdd3c06acb001454e7ec45ae2ffce857964a2531bb41686fc9f61c6abc16003" }, "analysis.no-verdict": { "text": "*Por debajo del umbral que esta compilación puede respaldar, no se emite ningún veredicto. Una puntuación baja no demuestra que una persona haya escrito este texto.*", diff --git a/src/SignsOfAI.UI/Services/Loc.cs b/src/SignsOfAI.UI/Services/Loc.cs index c0bcbaf..a433597 100644 --- a/src/SignsOfAI.UI/Services/Loc.cs +++ b/src/SignsOfAI.UI/Services/Loc.cs @@ -249,12 +249,10 @@ private async Task LogAsync(string message) /// The one-line verdict for an overall score, in the interface's language. The boundary comes /// from ; this only chooses the words for it. /// - public string Verdict(double score) => VerdictBands.Emphasis(score) switch + public string Verdict(double score) => VerdictBands.Holds(score) switch { - VerdictEmphasis.High => this["verdict.strong"], - VerdictEmphasis.Elevated => this["verdict.moderate"], - VerdictEmphasis.Present => this["verdict.light"], - _ => this["verdict.minimal"], + true => this["verdict.signs"], + false => this["verdict.none"], }; /// How to name the language the analyzer settled on. The engine only knows EN and ES. diff --git a/src/SignsOfAI.UI/wwwroot/i18n/en.json b/src/SignsOfAI.UI/wwwroot/i18n/en.json index bfaf507..09428f4 100644 --- a/src/SignsOfAI.UI/wwwroot/i18n/en.json +++ b/src/SignsOfAI.UI/wwwroot/i18n/en.json @@ -46,10 +46,8 @@ "sev.low": "Low", "sev.medium": "Medium", "sev.high": "High", - "verdict.strong": "Strong signs of AI writing", - "verdict.moderate": "Moderate signs of AI writing", - "verdict.light": "Light signs of AI writing", - "verdict.minimal": "Minimal signs of AI writing", + "verdict.signs": "Signs of AI writing", + "verdict.none": "No signs above the measured boundary", "home.pagetitle": "Signs of AI Writing — detect & de-AI-ify your text", "home.tagline": "Paste, upload, or just start typing. We flag the tells of AI writing — overused words, rhetorical crutches, robotic rhythm — live, as you write, and show you how to fix each one.", "home.privacy": "Analysis runs entirely in your browser. Your text never leaves your device.", diff --git a/src/SignsOfAI.UI/wwwroot/i18n/es.json b/src/SignsOfAI.UI/wwwroot/i18n/es.json index 2b0e083..79e0fec 100644 --- a/src/SignsOfAI.UI/wwwroot/i18n/es.json +++ b/src/SignsOfAI.UI/wwwroot/i18n/es.json @@ -46,10 +46,8 @@ "sev.low": "Baja", "sev.medium": "Media", "sev.high": "Alta", - "verdict.strong": "Señales fuertes de escritura IA", - "verdict.moderate": "Señales moderadas de escritura IA", - "verdict.light": "Señales leves de escritura IA", - "verdict.minimal": "Señales mínimas de escritura IA", + "verdict.signs": "Señales de escritura con IA", + "verdict.none": "Sin señales por encima del umbral medido", "home.pagetitle": "Señales de escritura IA — detecta y humaniza tu texto", "home.tagline": "Pega, sube un archivo o simplemente empieza a escribir. Marcamos las señales de escritura IA —palabras sobreusadas, muletillas retóricas, ritmo robótico— en vivo, mientras escribes, y te mostramos cómo corregir cada una.", "home.privacy": "El análisis se ejecuta por completo en tu navegador. Tu texto nunca sale de tu dispositivo.", diff --git a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs index 2f6902f..6d92f8c 100644 --- a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs +++ b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs @@ -272,7 +272,7 @@ public void An_unmeasured_language_gets_no_aggregate_threshold_or_verdict() Assert.Contains("never been measured for language code pt", report); Assert.Contains("aggregate result from other languages is not a substitute", report); - Assert.DoesNotContain("Strong signs of AI writing", report); + Assert.DoesNotContain("Signs of AI writing", report); if (PublishedCalibration.Current is { } calibration) { @@ -328,11 +328,13 @@ public void A_report_language_without_the_mandatory_core_is_rejected() [Fact] public void Withholds_the_verdict_below_the_threshold_it_can_support() { - // "Reads mostly human" printed above "treat the score as saying nothing" is a page arguing - // with itself, and the reader keeps whichever half suits them. + // A verdict printed above "treat the score as saying nothing" is a page arguing with itself, + // and the reader keeps whichever half suits them. Below the boundary the report prints the + // score and the reason, and no verdict line of either kind. var report = Report(); - Assert.DoesNotContain("Reads mostly human", report); + Assert.DoesNotContain("No signs above the measured boundary", report); + Assert.DoesNotContain("Signs of AI writing", report); Assert.Contains("A low score is not evidence that a person wrote this", report); } diff --git a/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs b/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs index 42d37cc..6279123 100644 --- a/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs +++ b/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs @@ -28,7 +28,7 @@ public class LocaleFileTests [ .. Enum.GetValues().Select(c => "cat." + c.ToString().ToLowerInvariant()), .. Enum.GetValues().Select(s => "sev." + s.ToString().ToLowerInvariant()), - "verdict.strong", "verdict.moderate", "verdict.light", "verdict.minimal", + "verdict.signs", "verdict.none", "lang.english", "lang.spanish", ]; diff --git a/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs b/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs index 6b2dc4a..943fb4c 100644 --- a/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs +++ b/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs @@ -73,6 +73,38 @@ public void No_language_is_silenced_merely_because_its_own_corpus_is_small() Assert.Contains("13.3%", report); } + [Fact] + public void The_verdict_reaches_its_reader_in_their_own_language() + { + // Rewording the English retires the SHA-256 pin every translation records for it, and a + // translation whose pin no longer matches is treated as stale — the whole report silently + // falls back to English. That is the correct behaviour and a silent way to undo #36, so the + // wording change and the pins have to travel together. This is the test that says they did. + var result = new AiWritingAnalyzer().Analyze(ObviouslyMachine, "en"); + var spanish = EvidenceReport.ToMarkdown(result, new ReportOptions { InterfaceLanguage = "es" }); + + Assert.Contains("Señales de escritura con IA", spanish); + Assert.DoesNotContain("Signs of AI writing", spanish); + } + + [Fact] + public void A_low_score_says_nothing_about_who_wrote_the_text() + { + // "Reads mostly human" in the report, "Minimal signs of AI writing" in the interface: the + // same state, two claims, and the first was never ours to make. A detector that detects + // nothing also returns zero, and this project has deliberately never measured how much + // machine writing it catches. + var human = new AiWritingAnalyzer().Analyze( + "We measured height, weight and age in three hundred participants recruited at two " + + "hospitals during the winter of 2011. At one site the effect vanished entirely.", "en"); + + Assert.False(VerdictBands.Holds(human.OverallScore), + $"The fixture stopped being a low-scoring text (scored {human.OverallScore:0})."); + + Assert.DoesNotContain("human", human.Verdict, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("person", human.Verdict, StringComparison.OrdinalIgnoreCase); + } + [Fact] public void The_boundary_the_product_draws_is_the_boundary_the_project_publishes() { diff --git a/tools/SignsOfAI.Calibration/Report.cs b/tools/SignsOfAI.Calibration/Report.cs index 69a820d..8d81a0c 100644 --- a/tools/SignsOfAI.Calibration/Report.cs +++ b/tools/SignsOfAI.Calibration/Report.cs @@ -103,6 +103,47 @@ public static string Render(CalibrationResult r, CorpusManifest manifest, string } sb.AppendLine(); + // ---- what the product does with the number ------------------------------------------------ + // Written here rather than left to the reader, because the gap between the measured figure and + // the shipped wording is precisely what this project criticises in other tools, and a page that + // publishes the figure without saying what the product does with it invites the same charge. + sb.AppendLine("## What the product does with this number"); + sb.AppendLine(); + if (r.Overall.ThresholdForTarget is { } recommended) + { + sb.AppendLine( + $"The tool speaks at **{recommended.ToString("0.#", Inv)}/100** and nowhere else, taking the " + + "boundary from the table above rather than from anybody's judgement. Below it a document " + + "gets its score and the reason it gets nothing more: a low score is not evidence that a " + + "person wrote something, since a detector that detects nothing also returns a low score, " + + "and this project has deliberately never measured how much machine writing it catches. " + + "The boundary moves when this page moves — including upward if a larger corpus turns out " + + "to be less flattering."); + sb.AppendLine(); + sb.AppendLine( + "Above it there is **one** verdict, not a scale of them. This corpus can place a boundary " + + "and can say nothing whatever about how much further past it a score has travelled: no " + + "text known to be human came close to the upper reaches, and grading \"moderate\" against " + + "\"strong\" would need machine-written text, which the opening of this page argues against " + + "collecting. Interfaces do shade a high score more urgently than a low one, and those " + + "shades are a display convention — they are not on this page because nothing measured " + + "them."); + sb.AppendLine(); + sb.AppendLine( + "A language absent from the corpus entirely gets no verdict at all, whatever it scores. " + + "A language present but too thin to bound its own rate borrows this boundary and carries " + + "its own figure beside it, so the reader weighs the rate that was measured for the " + + "writing in front of them rather than the pooled one."); + } + else + { + sb.AppendLine( + "This corpus supports no threshold, so the tool renders no verdict at any score. It " + + "reports what it found and says the boundary is unmeasured, which is the honest output " + + "of an unmeasured build and the reason this section is not silently omitted."); + } + sb.AppendLine(); + // ---- rules ------------------------------------------------------------------------------- if (r.RuleFalsePositives.Count > 0) {