diff --git a/src/SignsOfAI.Cli/Program.cs b/src/SignsOfAI.Cli/Program.cs index bf27ad8..745416f 100644 --- a/src/SignsOfAI.Cli/Program.cs +++ b/src/SignsOfAI.Cli/Program.cs @@ -399,8 +399,15 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor) } if (r.Findings.Count > shown.Count) Console.WriteLine(Col($" … and {r.Findings.Count - shown.Count} more (use --top {r.Findings.Count}).", 90)); + // Not "reads mostly human", and not a green tick. Both were the same claim in two forms, and it + // is the one #32 removed from the report and the interface because it was never ours to make: a + // detector that detects nothing also finds nothing, and this project has deliberately never + // measured how much machine writing it catches. It survived here for months because the test + // that guards the wording reads AnalysisResult.Verdict, and this line is a literal. if (r.Findings.Count == 0) - Console.WriteLine(Col(" ✓ No strong AI tells found — reads mostly human.", 32)); + Console.WriteLine(Col( + " No strong AI tells found — nothing above the measured boundary. " + + "That is not evidence a person wrote this.", 90)); Console.WriteLine(); } diff --git a/src/SignsOfAI.Core/Model/Finding.cs b/src/SignsOfAI.Core/Model/Finding.cs index 01065b2..8d8dd3f 100644 --- a/src/SignsOfAI.Core/Model/Finding.cs +++ b/src/SignsOfAI.Core/Model/Finding.cs @@ -28,7 +28,19 @@ public sealed record Finding /// Optional evidence, e.g. "48× more frequent post-ChatGPT". public string? Evidence { get; init; } - /// Contribution of this finding to the overall score (higher = stronger AI signal). + /// + /// How much this finding weighs (higher = stronger AI signal). + /// + /// For everything outside this is its contribution to the + /// score: the scorer sums these into one weighted density and maps it monotonically, so more + /// weight is more score. stat.burstiness is the exception — the scorer excludes that + /// category from the sum and derives its share from the sentence-length distribution instead, so + /// the weight it carries ranks it against the others without being what moved the number. + /// + /// This said "contribution of this finding to the overall score", which was false for that one + /// finding, and it stopped being a harmless inaccuracy the moment the evidence report began + /// sorting by it and telling the reader what the order meant. + /// public double Weight { get; init; } /// diff --git a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs index adfab2a..a751521 100644 --- a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs +++ b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs @@ -48,6 +48,11 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = var title = string.Equals(o.Title, ReportOptions.Default.Title, StringComparison.Ordinal) ? text.Get(ReportMessages.DefaultTitle).Text : o.Title; + // MaxRows is public, settable and unvalidated, and this library is on NuGet. A negative value + // printed "… and 12 more" for two findings — a count of omitted evidence larger than the + // evidence — because the arithmetic is Count minus the limit. Clamped once, here, rather than + // trusted at four call sites. + var rows = Math.Max(0, o.MaxRows); // Through Cell like everything else this report did not write: the title is a caller's string, // and every host that has one builds it from a filename. @@ -56,7 +61,7 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = var fallbackNoticeAt = sb.Length; if (!string.IsNullOrWhiteSpace(o.DocumentName)) AppendBlock(sb, text, ReportMessages.MetaDocument, Cell(o.DocumentName)); - AppendBlock(sb, text, ReportMessages.MetaGenerated, o.GeneratedOn, o.EngineVersion); + AppendBlock(sb, text, ReportMessages.MetaGenerated, Cell(o.GeneratedOn), Cell(o.EngineVersion)); sb.AppendLine(); // ── The reading, and immediately the caveat that makes it usable ────────────────────────── @@ -83,23 +88,31 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = // Named up here rather than left to the section below, because a page that opens "0/100" and // buries four contradictions in its own bibliography further down has chosen the wrong thing // to make salient — and that exact document is the one this project keeps writing about. - if (result.Citations.Issues.Count > 0 || result.Artifacts.Any) + // Contradictions, not issues. CitationIssue draws this line itself — a cited source missing + // from the list is a contradiction, an uncited entry in the list is not, "because people + // legitimately list further reading" — and the headline used to count both. A document whose + // only fault was further reading was announced as "1 source contradiction", and the note + // below the section then asserted in the report's own voice that it "disagrees with itself". + // That is an accusation the checker had explicitly declined to make, printed on the page a + // teacher takes to a committee, and it shipped for months. + var contradictions = result.Citations.ContradictionCount; + if (contradictions > 0 || result.Artifacts.Any) { - if (result.Citations.Issues.Count > 0 && result.Artifacts.Any) + if (contradictions > 0 && result.Artifacts.Any) AppendBlock(sb, text, - result.Citations.Issues.Count == 1 + contradictions == 1 ? result.Artifacts.Count == 1 ? ReportMessages.AnalysisFactsBothOneOne : ReportMessages.AnalysisFactsBothOneOther : result.Artifacts.Count == 1 ? ReportMessages.AnalysisFactsBothOtherOne : ReportMessages.AnalysisFactsBothOtherOther, - result.Citations.Issues.Count, result.Artifacts.Count); - else if (result.Citations.Issues.Count > 0) - AppendBlock(sb, text, result.Citations.Issues.Count == 1 + contradictions, result.Artifacts.Count); + else if (contradictions > 0) + AppendBlock(sb, text, contradictions == 1 ? ReportMessages.AnalysisFactsCitationOne : ReportMessages.AnalysisFactsCitationOther, - result.Citations.Issues.Count); + contradictions); else AppendBlock(sb, text, result.Artifacts.Count == 1 ? ReportMessages.AnalysisFactsArtifactOne @@ -155,6 +168,11 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = // pasted in. Saying so here is the difference between a fact and an insinuation. AppendBlock(sb, text, ReportMessages.CharactersExplanation); sb.AppendLine(); + // Said above the one table the reader is invited to check in an editor. Without it + // the Line column reads 6, 1, 2, 3 and looks like the tool lost its place — the + // same reason the signals list carries its own ordering line. + AppendBlock(sb, text, ReportMessages.CharactersOrdered); + sb.AppendLine(); AppendBlock(sb, text, ReportMessages.CharactersTableHeader); sb.AppendLine("|---|---|---:|---:|"); // Strong kinds first, then by position. A file can hold two hundred soft hyphens — @@ -165,15 +183,15 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = foreach (var occurrence in result.Artifacts.Occurrences .OrderByDescending(a => a.IsStrong) .ThenBy(a => a.Line).ThenBy(a => a.Column) - .Take(o.MaxRows)) + .Take(rows)) sb.Append("| ").Append(Cell(Describe(occurrence.Kind))).Append(" | `") .Append(occurrence.CodePoint).Append("` | ").Append(occurrence.Line) .Append(" | ").Append(occurrence.Column).AppendLine(" |"); - if (result.Artifacts.Occurrences.Count > o.MaxRows) + if (result.Artifacts.Occurrences.Count > rows) { sb.AppendLine(); AppendBlock(sb, text, ReportMessages.MoreRows, - result.Artifacts.Occurrences.Count - o.MaxRows); + result.Artifacts.Occurrences.Count - rows); } sb.AppendLine(); } @@ -184,16 +202,22 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = sb.AppendLine(); sb.AppendLine(Cell(result.Citations.Summary)); sb.AppendLine(); - foreach (var issue in result.Citations.Issues.Take(o.MaxRows)) + // Contradictions before untidiness, then in document order. Cutting this list by + // position was the same defect as the signals list and it was left in place one + // commit ago: forty uncited entries early in a long bibliography would print, and + // the repeated DOI in the last one — the finding that settles anything — would not. + foreach (var issue in result.Citations.Issues + .OrderByDescending(i => i.IsContradiction).ThenBy(i => i.Line) + .Take(rows)) sb.Append("- ").AppendLine(Cell(issue.Message)); // This list used to stop at forty in silence, alone among the four. A reader counting // the contradictions on the page against the number the summary above states would // find the page contradicting itself about a document accused of contradicting itself. - if (result.Citations.Issues.Count > o.MaxRows) + if (result.Citations.Issues.Count > rows) { sb.AppendLine(); AppendBlock(sb, text, ReportMessages.MoreRows, - result.Citations.Issues.Count - o.MaxRows); + result.Citations.Issues.Count - rows); } sb.AppendLine(); // Only claimed when something actually contradicts. The first version printed it @@ -202,7 +226,7 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = // its own voice, a self-contradiction it had explicitly not looked for. In a document // that goes to a committee about a nineteen-year-old, that is the precise harm this // project exists to argue against. - AppendBlock(sb, text, result.Citations.Issues.Count > 0 + AppendBlock(sb, text, result.Citations.ContradictionCount > 0 ? ReportMessages.CitationsIssuesNote : ReportMessages.CitationsNoIssuesNote); sb.AppendLine(); @@ -226,7 +250,17 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = // the visible evidence could not account for — in the one document this project builds // for somebody to take into a room where a decision is made about a person. // - // Weight is the finding's own contribution to the score. Ties are broken by how many + // The line says weight and not "how much it moved the score", which was the first + // wording and is false for one finding. Every non-statistical finding pools into a + // single weighted density that Map turns into the pattern score, and Map is monotone, + // so among those the order by weight *is* the order by contribution. The burstiness + // finding is not one of those: Scorer excludes the Statistical category from that sum + // and derives its share from the sentence-length distribution instead, so the weight it + // carries ranks it on this page without being what moved the number. It sorted to the + // top of the first report generated from this code, which is where the claim would have + // been read and where it would have been wrong. + // + // Ties are broken by how many // times that rule has already appeared, and only then by position: sixteen English rules // share the weight 3.5, and in a tie the reader is better served by one occurrence of // each before any second occurrence than by one rule's run. It reorders strictly within @@ -248,20 +282,23 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = .ThenBy(x => x.Rank) .ThenBy(x => x.Finding.Span.Start) .Select(x => x.Finding) - .Take(o.MaxRows)) + .Take(rows)) { sb.Append("- **").Append(f.Category).Append("** — "); if (!string.IsNullOrWhiteSpace(f.MatchedText)) sb.Append('“').Append(Cell(f.MatchedText)).Append("” — "); sb.Append(Cell(f.Message)).Append(' ').Append("*→ ").Append(Cell(f.Suggestion)).AppendLine("*"); } - if (result.Signals.Count > o.MaxRows) + if (result.Signals.Count > rows) { sb.AppendLine(); // Not the generic "… and N more": what was cut is now a property of the evidence // rather than of where it happened to fall, and the reader is entitled to know that - // nothing stronger than what they are looking at was left out. - AppendBlock(sb, text, ReportMessages.SignalsMore, result.Signals.Count - o.MaxRows); + // nothing stronger than what they are looking at was left out. That clause needs + // something to be looking at — at zero rows there is nothing above for the omitted + // findings to weigh less than, so the generic line, which stays true, is used. + AppendBlock(sb, text, rows > 0 ? ReportMessages.SignalsMore : ReportMessages.MoreRows, + result.Signals.Count - rows); } } sb.AppendLine(); @@ -279,15 +316,15 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = var groups = result.Observations.GroupBy(f => f.RuleId) .OrderByDescending(g => g.Count()).ThenBy(g => g.Key, StringComparer.Ordinal) .ToList(); - foreach (var group in groups.Take(o.MaxRows)) + foreach (var group in groups.Take(rows)) AppendBlock(sb, text, group.Count() == 1 ? ReportMessages.ObservationsRowOne : ReportMessages.ObservationsRowOther, Cell(group.Key), group.Count()); - if (groups.Count > o.MaxRows) + if (groups.Count > rows) { sb.AppendLine(); - AppendBlock(sb, text, ReportMessages.MoreRows, groups.Count - o.MaxRows); + AppendBlock(sb, text, ReportMessages.MoreRows, groups.Count - rows); } sb.AppendLine(); } @@ -361,7 +398,7 @@ public static string FolderToMarkdown( sb.AppendLine(); var fallbackNoticeAt = sb.Length; AppendBlock(sb, text, ReportMessages.MetaFolder, Cell(folderName)); - AppendBlock(sb, text, ReportMessages.MetaGenerated, o.GeneratedOn, o.EngineVersion); + AppendBlock(sb, text, ReportMessages.MetaGenerated, Cell(o.GeneratedOn), Cell(o.EngineVersion)); sb.AppendLine(); AppendBlock(sb, text, unreadable.Count > 0 ? entries.Count == 1 @@ -567,12 +604,18 @@ _ when VerdictBands.Holds(score) => ReportMessages.VerdictSigns, _ => ReportMessages.VerdictNone, }).Text; + /// + /// The two known names, or the code itself. Through , because for anything + /// other than en and es this prints a string the report did not write: Analyze takes the + /// language from its caller, the MCP server takes it as free text from a model, and + /// is a public record any host can construct. + /// private static string LanguageName(ReportText text, string language) => text.Get( language.Equals("en", StringComparison.OrdinalIgnoreCase) ? ReportMessages.LanguageEnglish : language.Equals("es", StringComparison.OrdinalIgnoreCase) ? ReportMessages.LanguageSpanish : ReportMessages.LanguageOther, language.Equals("en", StringComparison.OrdinalIgnoreCase) - || language.Equals("es", StringComparison.OrdinalIgnoreCase) ? [] : [language]).Text; + || language.Equals("es", StringComparison.OrdinalIgnoreCase) ? [] : [Cell(language)]).Text; private static void AppendBlock( StringBuilder sb, ReportText text, string key, params object?[] args) => @@ -666,7 +709,28 @@ private static string Pct(double fraction) => /// followed become report prose — a line beginning "## " arrived as a heading, in the report's own /// voice, from a filename or an extractor's error message; and a <. /// - /// The last one is why the Markdown form needs escaping at all. escapes on + /// Then two of Markdown's own: [, because escaping the angle bracket blocks raw HTML and + /// does nothing whatever to ![alt](url) — the page's last line says nothing here was + /// uploaded anywhere, and an image node fetches from somebody else's host the moment a teacher + /// pastes the report into an LMS, carrying the moment they opened it. Escaping the opening + /// bracket is enough for both images and links; the ! in front of one is only a + /// character. And *, because the signal row is written as the matched text followed by + /// the report's own *→ suggestion*: a single asterisk in the document paired with the + /// report's marker, so emphasis opened at the document's character and closed at the report's, + /// swallowing the report's explanation into the document's content and deleting the matched + /// character on the way. That covers ** with it, which could shift the pairing of the + /// category's own bold. + /// + /// _ and ` are deliberately left alone. Neither can pair with a marker this report + /// writes on the same line, so the worst they do is italicise a document's own text, and + /// escaping them would put a backslash in front of the underscore in every filename a teacher + /// reads. A bare URL still autolinks in some renderers; that is a link, not a request, and it + /// cannot be escaped without mangling the URL the reader needs to see. + /// + /// Nothing here is trimmed. Whitespace a rule matched is part of the span in the document, and + /// this is the page that claims to reproduce what the document said. + /// + /// The angle bracket is why the Markdown form needs escaping at all. escapes on /// its way out, so the HTML was never at risk — but Markdown is the form this file documents for /// pasting into an LMS comment box or a GitHub issue, and both of those render raw HTML embedded /// in Markdown. A document containing <img src=x onerror=…> reached them intact, and @@ -688,7 +752,8 @@ private static string Cell(string? text) => .Replace("\\", "\\\\") .Replace("|", "\\|") .Replace("<", "\\<") - .Trim(); + .Replace("[", "\\[") + .Replace("*", "\\*"); private static string Escape(string s) => s.Replace("&", "&").Replace("<", "<").Replace(">", ">"); @@ -796,7 +861,7 @@ private static List SplitRow(string line) /// Bold, italic and code, applied after escaping so a document cannot inject markup. private static string Inline(string text) { - var s = Escape(Unescape(text)); + var s = EscapeInline(text); s = Wrap(s, "**", "", ""); s = Wrap(s, "`", "", ""); s = Wrap(s, "*", "", ""); @@ -804,29 +869,46 @@ private static string Inline(string text) } /// - /// Undoes what wrote, so the HTML shows the character and not the backslash - /// that protected it in the Markdown. Only the three escapes this file emits: a backslash before - /// anything else came from the document and stays. + /// HTML-escapes a line and resolves the escapes wrote, in one pass, so the + /// page shows the character rather than the backslash that protected it in the Markdown — and so + /// that a character the document supplied can never be read as a marker by . + /// + /// The order is the whole point. Resolving the escapes first and escaping afterwards puts a live + /// asterisk back into the line before the emphasis pass runs, which is the defect this is here to + /// stop: the row is written as the matched text and then the report's own *→ suggestion*, + /// so one asterisk from the document paired with the report's marker and swallowed the report's + /// explanation into the document's content. An asterisk therefore comes back as &#42;, + /// which is a literal asterisk to every renderer and is not a character can + /// see. Nothing else needs the treatment: [ and | are not markers here. /// - /// Table cells already lose their \| in , which has to resolve them - /// before it can tell a real column boundary from a pipe inside a filename; the list items and - /// paragraphs had no such step, so a citation message containing a pipe used to reach the HTML - /// page as \|. + /// Table cells lose their \| earlier, in , which has to resolve them + /// before it can tell a column boundary from a pipe inside a filename. List items and paragraphs + /// had no such step, so a citation message containing a pipe used to reach the page as \|. /// - private static string Unescape(string text) + private static string EscapeInline(string text) { - if (!text.Contains('\\')) return text; - var sb = new StringBuilder(text.Length); for (int i = 0; i < text.Length; i++) { if (text[i] == '\\' && i + 1 < text.Length - && (text[i + 1] is '|' or '<' or '\\')) + && text[i + 1] is '|' or '<' or '[' or '*' or '\\') { - sb.Append(text[++i]); + sb.Append(text[++i] switch + { + '*' => "*", + '<' => "<", + var c => c.ToString(), + }); continue; } - sb.Append(text[i]); + + sb.Append(text[i] switch + { + '&' => "&", + '<' => "<", + '>' => ">", + var c => c.ToString(), + }); } return sb.ToString(); } diff --git a/src/SignsOfAI.Core/Reporting/ReportMessages.cs b/src/SignsOfAI.Core/Reporting/ReportMessages.cs index 6b3bcff..f7bc97d 100644 --- a/src/SignsOfAI.Core/Reporting/ReportMessages.cs +++ b/src/SignsOfAI.Core/Reporting/ReportMessages.cs @@ -78,6 +78,13 @@ public static class ReportMessages public const string CheckableIntro = "checkable.intro"; public const string CharactersExplanation = "characters.explanation"; public const string CharactersTableHeader = "characters.table-header"; + + /// + /// Why the table is not in file order. It prints line and column so the reader can find each + /// occurrence in an editor, which makes it the one table on the page they are invited to verify — + /// and a Line column reading 6, 1, 2, 3 with no explanation looks like a defect. + /// + public const string CharactersOrdered = "characters.ordered"; public const string MoreRows = "common.more-rows"; public const string CitationsIssuesNote = "citations.issues-note"; public const string CitationsNoIssuesNote = "citations.no-issues-note"; @@ -168,6 +175,7 @@ public static class ReportMessages [CheckableIntro] = 0, [CharactersExplanation] = 0, [CharactersTableHeader] = 0, + [CharactersOrdered] = 0, [MoreRows] = 1, [CitationsIssuesNote] = 0, [CitationsNoIssuesNote] = 0, @@ -256,12 +264,13 @@ public static class ReportMessages [CheckableIntro] = "These are not judgements about the writing and they did not move the score. Each is either present in the file or it is not.", [CharactersExplanation] = "Several of these have ordinary explanations — word processors insert soft hyphens and unusual spaces on their own, and any copy-paste can carry them. Invisible characters and letters borrowed from another alphabet are harder to arrive at by accident, though pasting text can do it. This table says what is in the file, not how it got there.", [CharactersTableHeader] = "| Character | Codepoint | Line | Column |", + [CharactersOrdered] = "Listed with the characters hardest to arrive at by accident first, then in the order they appear in the file.", [MoreRows] = "… and {0} more.", [CitationsIssuesNote] = "> None of this needed the internet: the document disagrees with itself. It is a question to ask, not a conclusion — the answer is usually one sentence.", [CitationsNoIssuesNote] = "> Nothing here is a finding. It describes what could and could not be checked.", [SignalsNone] = "None.", - [SignalsOrdered] = "Ordered by how much each one moved the score, strongest first, rather than by where it appears in the text.", - [SignalsMore] = "… and {0} more, none of which moved the score as much as any of the above.", + [SignalsOrdered] = "Ordered by how much weight each one carries, heaviest first, rather than by where it appears in the text.", + [SignalsMore] = "… and {0} more, none of them carrying more weight than what is shown above.", [ObservationsIntro] = "Measured against writing published before generative models existed. Shown because they are real, and counted for nothing because they are ordinary.", [ObservationsRowOne] = "- {0} — {1} occurrence", [ObservationsRowOther] = "- {0} — {1} occurrences", diff --git a/src/SignsOfAI.Core/Reporting/report.en.json b/src/SignsOfAI.Core/Reporting/report.en.json index 5861efa..6a177a6 100644 --- a/src/SignsOfAI.Core/Reporting/report.en.json +++ b/src/SignsOfAI.Core/Reporting/report.en.json @@ -47,13 +47,14 @@ "caveat.aggregate-measured": { "text": "> **A score is not proof.** On {0} texts published before generative models existed, this build's false-positive rate at a threshold of {1}/100 was under {2} — the upper end of a 95% interval, not a guarantee, and measured on published articles rather than student work. Below that threshold, treat the score as saying nothing." }, "checkable.intro": { "text": "These are not judgements about the writing and they did not move the score. Each is either present in the file or it is not." }, "characters.explanation": { "text": "Several of these have ordinary explanations — word processors insert soft hyphens and unusual spaces on their own, and any copy-paste can carry them. Invisible characters and letters borrowed from another alphabet are harder to arrive at by accident, though pasting text can do it. This table says what is in the file, not how it got there." }, + "characters.ordered": { "text": "Listed with the characters hardest to arrive at by accident first, then in the order they appear in the file." }, "characters.table-header": { "text": "| Character | Codepoint | Line | Column |" }, "common.more-rows": { "text": "… and {0} more." }, "citations.issues-note": { "text": "> None of this needed the internet: the document disagrees with itself. It is a question to ask, not a conclusion — the answer is usually one sentence." }, "citations.no-issues-note": { "text": "> Nothing here is a finding. It describes what could and could not be checked." }, "signals.none": { "text": "None." }, - "signals.ordered": { "text": "Ordered by how much each one moved the score, strongest first, rather than by where it appears in the text." }, - "signals.more": { "text": "… and {0} more, none of which moved the score as much as any of the above." }, + "signals.ordered": { "text": "Ordered by how much weight each one carries, heaviest first, rather than by where it appears in the text." }, + "signals.more": { "text": "… and {0} more, none of them carrying more weight than what is shown above." }, "observations.intro": { "text": "Measured against writing published before generative models existed. Shown because they are real, and counted for nothing because they are ordinary." }, "observations.row.one": { "text": "- {0} — {1} occurrence" }, "observations.row.other": { "text": "- {0} — {1} occurrences" }, diff --git a/src/SignsOfAI.Core/Reporting/report.es.json b/src/SignsOfAI.Core/Reporting/report.es.json index 5434333..750ab32 100644 --- a/src/SignsOfAI.Core/Reporting/report.es.json +++ b/src/SignsOfAI.Core/Reporting/report.es.json @@ -42,13 +42,17 @@ "text": "Señales contabilizadas", "sourceHash": "078671a3b913dc8d830dc433445ca4b4cc401debf4b6fb8829ea9376ab658cc2" }, + "characters.ordered": { + "text": "Se listan primero los caracteres más difíciles de obtener por accidente, y después en el orden en que aparecen en el fichero.", + "sourceHash": "268fa7576be22e9731efa7d6e98715ff6d62e83a1af2c40601cdc85036c3cdb7" + }, "signals.ordered": { - "text": "Ordenadas por cuánto movió cada una la puntuación, de mayor a menor, y no por el lugar que ocupan en el texto.", - "sourceHash": "6328aa4d5b8ee1e8742dad32fd9d8a92fe928d606938710e706be22d0be26f15" + "text": "Ordenadas por el peso que carga cada una, de mayor a menor, y no por el lugar que ocupan en el texto.", + "sourceHash": "5a270776bb1082eb80960b1a1b431b1a56fd4d6dd3f422544cd5959eeaf18c04" }, "signals.more": { - "text": "… y {0} más, ninguna de las cuales movió la puntuación tanto como las de arriba.", - "sourceHash": "754e025aefdb15ea0b58eb55aecaa82c9da6799259ed0f2c64d33ccac3cfb558" + "text": "… y {0} más, ninguna con más peso que las que se muestran arriba.", + "sourceHash": "621da36ba6fb9cadbc5a4ded4368137d9bc81254ae34678cc98e0a8978aa2212" }, "section.observations": { "text": "Encontrado, pero a una frecuencia habitual en textos humanos", diff --git a/src/SignsOfAI.UI/wwwroot/i18n/en.json b/src/SignsOfAI.UI/wwwroot/i18n/en.json index 09428f4..5109a21 100644 --- a/src/SignsOfAI.UI/wwwroot/i18n/en.json +++ b/src/SignsOfAI.UI/wwwroot/i18n/en.json @@ -130,11 +130,11 @@ "rhythm.cap.uniform": "Very uniform bars — a machine-like, metronomic cadence.", "rhythm.cap.somewhat": "Somewhat uniform — the rhythm barely varies.", "rhythm.cap.fairly": "Fairly varied — getting closer to human cadence.", - "rhythm.cap.varied": "Nicely varied, jagged rhythm — reads human.", + "rhythm.cap.varied": "Nicely varied, jagged rhythm — the opposite of a metronomic cadence.", "home.annotated": "Your text, annotated", "home.annotated.hint": "Highlighted spans are AI tells — hover for the suggestion.", "home.recommendations": "Recommendations ({0})", - "home.nofindings": "No strong AI tells found. This reads mostly human — nice work.", + "home.nofindings": "No strong AI tells found — nothing above the measured boundary. That is not evidence a person wrote this.", "home.idle": "Start typing, paste, or upload a document — your AI-writing score appears here instantly.", "file.toolarge": "“{0}” is too large (max 5 MB).", "file.loaded": "Loaded “{0}” ({1} KB).", diff --git a/src/SignsOfAI.UI/wwwroot/i18n/es.json b/src/SignsOfAI.UI/wwwroot/i18n/es.json index 79e0fec..74ef2b2 100644 --- a/src/SignsOfAI.UI/wwwroot/i18n/es.json +++ b/src/SignsOfAI.UI/wwwroot/i18n/es.json @@ -130,11 +130,11 @@ "rhythm.cap.uniform": "Barras muy uniformes: una cadencia de metrónomo, de máquina.", "rhythm.cap.somewhat": "Algo uniforme: el ritmo apenas varía.", "rhythm.cap.fairly": "Bastante variado: se acerca a la cadencia humana.", - "rhythm.cap.varied": "Ritmo bien variado e irregular: se lee humano.", + "rhythm.cap.varied": "Ritmo bien variado e irregular: lo contrario de una cadencia de metrónomo.", "home.annotated": "Tu texto, anotado", "home.annotated.hint": "Los fragmentos resaltados son señales de IA; pasa el cursor para ver la sugerencia.", "home.recommendations": "Recomendaciones ({0})", - "home.nofindings": "No se encontraron señales fuertes de IA. Se lee bastante humano; buen trabajo.", + "home.nofindings": "No se encontraron señales fuertes de IA: nada por encima del umbral medido. Eso no demuestra que lo escribiera una persona.", "home.idle": "Empieza a escribir, pega texto o sube un documento: tu puntaje de escritura IA aparece aquí al instante.", "file.toolarge": "“{0}” es demasiado grande (máx. 5 MB).", "file.loaded": "Se cargó “{0}” ({1} KB).", diff --git a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs index 75326a6..d0e6f50 100644 --- a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs +++ b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs @@ -1,5 +1,6 @@ using SignsOfAI.Core; using SignsOfAI.Core.Calibration; +using SignsOfAI.Core.Citations; using SignsOfAI.Core.Model; using SignsOfAI.Core.Reporting; using System.Text.RegularExpressions; @@ -383,7 +384,8 @@ public void A_supported_interface_language_carries_no_apology() /// report prints and a known strongest one, and building that out of real prose would make the /// test a statement about which rules happen to fire rather than about what the report keeps. /// - private static AnalysisResult Synthetic(IReadOnlyList findings) => new() + private static AnalysisResult Synthetic( + IReadOnlyList findings, IReadOnlyList? issues = null) => new() { Language = "en", RulePackLanguage = "en", @@ -391,6 +393,29 @@ public void A_supported_interface_language_carries_no_apology() CategoryScores = [], OverallScore = 30, Statistics = new TextStatistics { WordCount = 4000, SentenceCount = 200 }, + Citations = issues is null ? CitationReport.Empty : new CitationReport + { + References = [], + // One citation, because CitationReport.Any gates the whole section on there being + // something to describe — a report with issues and no sources is not a real state. + Citations = + [ + new InTextCitation + { + Raw = "(Delgado & Ruiz, 2021)", + Span = new TextSpan(0, 22), + Line = 1, + Surname = "Delgado", + Year = 2021, + }, + ], + Issues = issues, + Style = CitationStyle.AuthorYear, + HasReferenceList = true, + Summary = "A bibliography with one contradiction and a great deal of untidiness.", + Advice = "Ask about the repeated identifier.", + ContradictionCount = issues.Count(i => i.IsContradiction), + }, }; private static Finding Signal(string id, double weight, int start, string matched) => new() @@ -436,8 +461,94 @@ public void Says_what_it_left_out_and_that_none_of_it_outweighed_what_is_shown() var report = EvidenceReport.ToMarkdown(Synthetic(findings)); - Assert.Contains("Ordered by how much each one moved the score", report); - Assert.Contains("21 more, none of which moved the score as much as any of the above", report); + Assert.Contains("Ordered by how much weight each one carries", report); + Assert.Contains("21 more, none of them carrying more weight than what is shown above", report); + } + + [Theory] + [InlineData(0)] + [InlineData(-10)] + public void A_row_limit_below_one_cannot_invent_omitted_evidence(int maxRows) + { + // MaxRows is public, settable and unvalidated, and this library is on NuGet. The omission + // count is Count minus the limit, so a negative limit reported more omitted findings than + // the document had; and at zero rows the strong clause claimed nothing omitted outweighed + // what was shown, with nothing shown. + var findings = Enumerable.Range(0, 2) + .Select(i => Signal("lex.weak", 1.0, i * 100, $"weak-{i:000}")) + .ToList(); + + var report = EvidenceReport.ToMarkdown(Synthetic(findings), + new ReportOptions { MaxRows = maxRows }); + + Assert.Contains("… and 2 more.", report); + Assert.DoesNotContain("carrying more weight than what is shown above", report); + } + + [Fact] + public void A_contradiction_is_not_pushed_out_of_the_list_by_untidiness() + { + // The same defect as the signals list, left in place one commit ago. The checker draws the + // line itself with IsContradiction, and a bibliography can carry forty uncited entries and + // one repeated DOI — the finding that settles anything — in its last one. + var issues = Enumerable.Range(0, 45) + .Select(i => new CitationIssue + { + Kind = CitationIssueKind.ListedButNotCited, + Span = new TextSpan(i * 10, 5), + Line = i + 1, + Subject = $"Entry {i}", + Message = $"untidy-{i:000} is listed but never cited.", + IsContradiction = false, + }) + .Append(new CitationIssue + { + Kind = CitationIssueKind.RepeatedDoi, + Span = new TextSpan(9_000, 5), + Line = 999, + Subject = "10.1080/jem.2019.4471", + Message = "the-same-doi on two different works.", + IsContradiction = true, + }) + .ToList(); + + var report = EvidenceReport.ToMarkdown(Synthetic([], issues)); + + Assert.Contains("the-same-doi", report); + Assert.DoesNotContain("untidy-044", report); + Assert.Contains("… and 6 more.", report); + } + + [Fact] + public void An_untidy_bibliography_is_not_announced_as_a_contradiction() + { + // People legitimately list further reading, which is why CitationIssue draws the line itself + // with IsContradiction. The headline counted every issue instead, so a document whose only + // fault is an uncited entry was announced as "1 source contradiction" over a note asserting + // it "disagrees with itself" — an accusation the checker had explicitly declined to make, + // on the page a teacher takes to a committee. + const string tidy = """ + Formative assessment improves retention. Delgado and Ruiz (2021) report gains across two + cohorts, and the effect held after a year. + + ## References + + Delgado, M., & Ruiz, C. (2021). Formative assessment and retention. Studies in Higher + Education, 46(4), 512-538. + + Whitfield, J. (2020). Feedback timing in large cohorts. Assessment Review, 12(1), 33-51. + """; + + var result = new AiWritingAnalyzer().Analyze(tidy, "en"); + var report = EvidenceReport.ToMarkdown(result); + + // Guard the premise: the fixture must actually produce an untidiness and no contradiction, + // or the assertions below would pass without testing anything. + Assert.Contains(result.Citations.Issues, i => !i.IsContradiction); + Assert.Equal(0, result.Citations.ContradictionCount); + + Assert.DoesNotContain("source contradiction", report); + Assert.DoesNotContain("disagrees with itself", report); } [Fact] @@ -453,8 +564,8 @@ public void A_spanish_reader_is_told_the_order_in_Spanish() var report = EvidenceReport.ToMarkdown(Synthetic(findings), new ReportOptions { InterfaceLanguage = "es" }); - Assert.Contains("Ordenadas por cuánto movió cada una la puntuación", report); - Assert.Contains("21 más, ninguna de las cuales movió la puntuación", report); + Assert.Contains("Ordenadas por el peso que carga cada una", report); + Assert.Contains("21 más, ninguna con más peso que las que se muestran arriba", report); } [Fact] @@ -490,6 +601,82 @@ public void The_markdown_form_does_not_carry_live_html_into_a_comment_box() Assert.DoesNotContain("→ Say it another way.", html); + } + + [Fact] + public void A_language_code_from_a_host_cannot_carry_markup() + { + // The MCP server takes language strings as free text from a model, and the public analyzer + // API takes one from any host. It reaches the page through "language code {0}". + var result = Synthetic([]) with { Language = "" }; + + var markdown = EvidenceReport.ToMarkdown(result); + + Assert.DoesNotContain(" + /// Wordings this project decided it may not use, in any locale, because they are claims about the + /// writer rather than about the tool. + /// + /// A blacklist is a blunt instrument and this one is deliberate. The decision was taken in #32 and + /// applied to the report and to the score card; it survived for months in the empty state of the + /// home page — "This reads mostly human — nice work" — and in the rhythm caption, because the test + /// that guards it reads AnalysisResult.Verdict and these are locale data. Translations + /// arrive as community pull requests, so the guard has to live where the data does. + /// + private static readonly string[] RetiredClaims = + [ + "reads mostly human", "reads human", "mostly human", + "se lee humano", "se lee bastante humano", "bastante humano", + ]; + + [Fact] + public void No_locale_tells_the_reader_a_person_wrote_the_text() + { + var problems = new List(); + + foreach (var file in LocaleFiles()) + { + var name = Path.GetFileName(file); + foreach (var (key, value) in ReadLocale(Path.GetFileNameWithoutExtension(file))) + { + foreach (var claim in RetiredClaims) + { + if (value.Contains(claim, StringComparison.OrdinalIgnoreCase)) + problems.Add($"{name} → \"{key}\": says \"{claim}\""); + } + } + } + + Assert.True(problems.Count == 0, + "A low score is a fact about this tool, not about who wrote the text: a detector that " + + "detects nothing also returns a low score, and this project has deliberately never " + + "measured how much machine writing it catches. Say what was measured instead — see " + + "verdict.none. Found:\n " + string.Join("\n ", problems)); + } + [Fact] public void Translations_use_only_known_keys() {