diff --git a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs index 94c7d38..adfab2a 100644 --- a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs +++ b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs @@ -49,7 +49,9 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = ? text.Get(ReportMessages.DefaultTitle).Text : o.Title; - sb.Append("# ").Append(title).AppendLine(); + // Through Cell like everything else this report did not write: the title is a caller's string, + // and every host that has one builds it from a filename. + sb.Append("# ").Append(Cell(title)).AppendLine(); sb.AppendLine(); var fallbackNoticeAt = sb.Length; if (!string.IsNullOrWhiteSpace(o.DocumentName)) @@ -144,7 +146,7 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = { AppendHeading(sb, text, 3, ReportMessages.SectionCharacters); sb.AppendLine(); - sb.AppendLine(result.Artifacts.Summary); + sb.AppendLine(Cell(result.Artifacts.Summary)); sb.AppendLine(); // The heading used to say "characters writing does not produce", which is false for // half of what this table lists: Word makes soft hyphens on its own and a stray @@ -155,7 +157,15 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = sb.AppendLine(); AppendBlock(sb, text, ReportMessages.CharactersTableHeader); sb.AppendLine("|---|---|---:|---:|"); - foreach (var occurrence in result.Artifacts.Occurrences.Take(o.MaxRows)) + // Strong kinds first, then by position. A file can hold two hundred soft hyphens — + // Word inserts them unprompted — and one letter borrowed from another alphabet. In + // document order the innocent two hundred fill the table and the one occurrence that + // is hard to arrive at by accident falls off the end. IsStrong is the scanner's own + // published distinction, not a new judgement invented for the page. + foreach (var occurrence in result.Artifacts.Occurrences + .OrderByDescending(a => a.IsStrong) + .ThenBy(a => a.Line).ThenBy(a => a.Column) + .Take(o.MaxRows)) sb.Append("| ").Append(Cell(Describe(occurrence.Kind))).Append(" | `") .Append(occurrence.CodePoint).Append("` | ").Append(occurrence.Line) .Append(" | ").Append(occurrence.Column).AppendLine(" |"); @@ -172,10 +182,19 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = { AppendHeading(sb, text, 3, ReportMessages.SectionCitations); sb.AppendLine(); - sb.AppendLine(result.Citations.Summary); + sb.AppendLine(Cell(result.Citations.Summary)); sb.AppendLine(); foreach (var issue in result.Citations.Issues.Take(o.MaxRows)) sb.Append("- ").AppendLine(Cell(issue.Message)); + // This list used to stop at forty in silence, alone among the four. A reader counting + // the contradictions on the page against the number the summary above states would + // find the page contradicting itself about a document accused of contradicting itself. + if (result.Citations.Issues.Count > o.MaxRows) + { + sb.AppendLine(); + AppendBlock(sb, text, ReportMessages.MoreRows, + result.Citations.Issues.Count - o.MaxRows); + } sb.AppendLine(); // Only claimed when something actually contradicts. The first version printed it // whenever there was anything to say about sources at all — including "no reference @@ -199,7 +218,37 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = } else { - foreach (var f in result.Signals.Take(o.MaxRows)) + // Strongest first, and the page says so. The analyser returns findings in the order they + // occur in the text, because that is what highlighting and the rewriter need; printing + // them that way and then cutting at forty meant a long document could spend the whole + // list on weak hits in its opening pages while the finding that did most to produce the + // headline number sat in the last paragraph, omitted. The reader was then given a score + // the visible evidence could not account for — in the one document this project builds + // for somebody to take into a room where a decision is made about a person. + // + // Weight is the finding's own contribution to the score. Ties are broken by how many + // times that rule has already appeared, and only then by position: sixteen English rules + // share the weight 3.5, and in a tie the reader is better served by one occurrence of + // each before any second occurrence than by one rule's run. It reorders strictly within + // equal weight, so it costs the guarantee nothing — everything omitted still weighs no + // more than everything shown, which is what the line below is entitled to say. + // + // A rule that genuinely outweighs the rest still fills the list with its own repeats, + // and that is the honest picture: fourteen occurrences of one word are where such a + // document's number actually comes from. Whether the section should collapse them into + // a count, as the observations section does, is a question about its shape rather than + // about which evidence it drops. + AppendBlock(sb, text, ReportMessages.SignalsOrdered); + sb.AppendLine(); + foreach (var f in result.Signals + .GroupBy(f => f.RuleId) + .SelectMany(g => g.OrderBy(f => f.Span.Start) + .Select((f, rank) => (Finding: f, Rank: rank))) + .OrderByDescending(x => x.Finding.Weight) + .ThenBy(x => x.Rank) + .ThenBy(x => x.Finding.Span.Start) + .Select(x => x.Finding) + .Take(o.MaxRows)) { sb.Append("- **").Append(f.Category).Append("** — "); if (!string.IsNullOrWhiteSpace(f.MatchedText)) @@ -209,7 +258,10 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = if (result.Signals.Count > o.MaxRows) { sb.AppendLine(); - AppendBlock(sb, text, ReportMessages.MoreRows, result.Signals.Count - o.MaxRows); + // Not the generic "… and N more": what was cut is now a property of the evidence + // rather than of where it happened to fall, and the reader is entitled to know that + // nothing stronger than what they are looking at was left out. + AppendBlock(sb, text, ReportMessages.SignalsMore, result.Signals.Count - o.MaxRows); } } sb.AppendLine(); @@ -220,11 +272,23 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = sb.AppendLine(); AppendBlock(sb, text, ReportMessages.ObservationsIntro); sb.AppendLine(); - foreach (var group in result.Observations.GroupBy(f => f.RuleId).Take(o.MaxRows)) + // Most frequent first, for the same reason as the signals list: a rule used thirty times + // is the one the reader wants to see, and in rule-id order it can be pushed out by + // twenty rules that fired once. The id comes from a rule pack, which is JSON anyone can + // contribute, so it goes through Cell like any other text this report did not write. + var groups = result.Observations.GroupBy(f => f.RuleId) + .OrderByDescending(g => g.Count()).ThenBy(g => g.Key, StringComparer.Ordinal) + .ToList(); + foreach (var group in groups.Take(o.MaxRows)) AppendBlock(sb, text, group.Count() == 1 ? ReportMessages.ObservationsRowOne : ReportMessages.ObservationsRowOther, - group.Key, group.Count()); + Cell(group.Key), group.Count()); + if (groups.Count > o.MaxRows) + { + sb.AppendLine(); + AppendBlock(sb, text, ReportMessages.MoreRows, groups.Count - o.MaxRows); + } sb.AppendLine(); } @@ -293,7 +357,7 @@ public static string FolderToMarkdown( var unreadable = entries.Where(e => e.Error is not null).ToList(); var scored = entries.Where(e => e.Error is null && e.Score is not null).ToList(); - sb.Append("# ").AppendLine(title); + sb.Append("# ").AppendLine(Cell(title)); sb.AppendLine(); var fallbackNoticeAt = sb.Length; AppendBlock(sb, text, ReportMessages.MetaFolder, Cell(folderName)); @@ -596,19 +660,35 @@ private static string Pct(double fraction) => CultureInfo.InvariantCulture) + "%"; /// - /// User content on its way into a Markdown line. Two things it must survive being given: a pipe, + /// User content on its way into a Markdown line. Three things it must survive being given: a pipe, /// which would open an extra table cell and shift every number one column to the right in a table - /// a teacher reads scores from; and a newline, which would end the list item and let whatever + /// a teacher reads scores from; a newline, which would end the list item and let whatever /// followed become report prose — a line beginning "## " arrived as a heading, in the report's own - /// voice, from a filename or an extractor's error message. + /// voice, from a filename or an extractor's error message; and a <. + /// + /// The last one is why the Markdown form needs escaping at all. escapes on + /// its way out, so the HTML was never at risk — but Markdown is the form this file documents for + /// pasting into an LMS comment box or a GitHub issue, and both of those render raw HTML embedded + /// in Markdown. A document containing <img src=x onerror=…> reached them intact, and + /// the person pasting it is a teacher who has been told the report is the safe thing to forward. + /// Backslash rather than an entity, so the character survives one escaping and exactly one: + /// undoes it before escaping for HTML, and the reader sees what the document + /// actually said. Reproducing the matched text exactly is the claim this product rests on. /// - /// Matched text is user content by definition, and so is anything a community rule pack matches, - /// which is JSON anybody can contribute. + /// Matched text is user content by definition, and so is anything a community rule pack matches + /// or names, which is JSON anybody can contribute. /// private static string Cell(string? text) => string.IsNullOrEmpty(text) ? "" - : text.ReplaceLineEndings(" ").Replace("|", "\\|").Trim(); + // The backslash goes first, and it is not decoration. Escaping only the bracket turns a + // document that already contains \< into \\< , which Markdown reads as an escaped + // backslash followed by a live bracket — the escape defeated with one extra character. + : text.ReplaceLineEndings(" ") + .Replace("\\", "\\\\") + .Replace("|", "\\|") + .Replace("<", "\\<") + .Trim(); private static string Escape(string s) => s.Replace("&", "&").Replace("<", "<").Replace(">", ">"); @@ -716,13 +796,41 @@ private static List SplitRow(string line) /// Bold, italic and code, applied after escaping so a document cannot inject markup. private static string Inline(string text) { - var s = Escape(text); + var s = Escape(Unescape(text)); s = Wrap(s, "**", "", ""); s = Wrap(s, "`", "", ""); s = Wrap(s, "*", "", ""); return s; } + /// + /// Undoes what wrote, so the HTML shows the character and not the backslash + /// that protected it in the Markdown. Only the three escapes this file emits: a backslash before + /// anything else came from the document and stays. + /// + /// Table cells already lose their \| in , which has to resolve them + /// before it can tell a real column boundary from a pipe inside a filename; the list items and + /// paragraphs had no such step, so a citation message containing a pipe used to reach the HTML + /// page as \|. + /// + private static string Unescape(string text) + { + if (!text.Contains('\\')) return text; + + var sb = new StringBuilder(text.Length); + for (int i = 0; i < text.Length; i++) + { + if (text[i] == '\\' && i + 1 < text.Length + && (text[i + 1] is '|' or '<' or '\\')) + { + sb.Append(text[++i]); + continue; + } + sb.Append(text[i]); + } + return sb.ToString(); + } + private static string Wrap(string text, string marker, string open, string close) { var parts = text.Split(marker); @@ -762,6 +870,12 @@ public sealed record ReportOptions /// /// Where each list stops. A report meant to be read by a person is worth less at four hundred rows /// than at forty, and the count of what was left out is printed rather than the rows themselves. + /// + /// Every list that this cuts is sorted strongest first before it is cut, and every one of them + /// says how many it left out. Truncating a list the analyser returns in document order silently + /// discards the evidence with the most claim to be on the page — see the signals section. The + /// folder table is deliberately exempt and prints every file: a scan of two hundred essays that + /// omitted the low scorers would be withholding the result that settles a suspicion. /// public int MaxRows { get; init; } = 40; diff --git a/src/SignsOfAI.Core/Reporting/ReportMessages.cs b/src/SignsOfAI.Core/Reporting/ReportMessages.cs index 792920c..6b3bcff 100644 --- a/src/SignsOfAI.Core/Reporting/ReportMessages.cs +++ b/src/SignsOfAI.Core/Reporting/ReportMessages.cs @@ -82,6 +82,20 @@ public static class ReportMessages public const string CitationsIssuesNote = "citations.issues-note"; public const string CitationsNoIssuesNote = "citations.no-issues-note"; public const string SignalsNone = "signals.none"; + + /// + /// Why the list is not in the order of the text. Printed whenever there are signals at all, not + /// only when the list is cut: the order changed for every reader, and one following the report + /// through their student's document would otherwise think the tool had lost its place. + /// + public const string SignalsOrdered = "signals.ordered"; + + /// + /// What the cut left out. Separate from because here it can say something + /// the generic line cannot: not merely that there is more, but that none of it outweighs what is + /// on the page. That is only true because the list is sorted first. + /// + public const string SignalsMore = "signals.more"; public const string ObservationsIntro = "observations.intro"; public const string ObservationsRowOne = "observations.row.one"; public const string ObservationsRowOther = "observations.row.other"; @@ -158,6 +172,8 @@ public static class ReportMessages [CitationsIssuesNote] = 0, [CitationsNoIssuesNote] = 0, [SignalsNone] = 0, + [SignalsOrdered] = 0, + [SignalsMore] = 1, // {0} how many were left out [ObservationsIntro] = 0, [ObservationsRowOne] = 2, [ObservationsRowOther] = 2, @@ -244,6 +260,8 @@ public static class ReportMessages [CitationsIssuesNote] = "> None of this needed the internet: the document disagrees with itself. It is a question to ask, not a conclusion — the answer is usually one sentence.", [CitationsNoIssuesNote] = "> Nothing here is a finding. It describes what could and could not be checked.", [SignalsNone] = "None.", + [SignalsOrdered] = "Ordered by how much each one moved the score, strongest first, rather than by where it appears in the text.", + [SignalsMore] = "… and {0} more, none of which moved the score as much as any of the above.", [ObservationsIntro] = "Measured against writing published before generative models existed. Shown because they are real, and counted for nothing because they are ordinary.", [ObservationsRowOne] = "- {0} — {1} occurrence", [ObservationsRowOther] = "- {0} — {1} occurrences", diff --git a/src/SignsOfAI.Core/Reporting/report.en.json b/src/SignsOfAI.Core/Reporting/report.en.json index 6887e14..5861efa 100644 --- a/src/SignsOfAI.Core/Reporting/report.en.json +++ b/src/SignsOfAI.Core/Reporting/report.en.json @@ -52,6 +52,8 @@ "citations.issues-note": { "text": "> None of this needed the internet: the document disagrees with itself. It is a question to ask, not a conclusion — the answer is usually one sentence." }, "citations.no-issues-note": { "text": "> Nothing here is a finding. It describes what could and could not be checked." }, "signals.none": { "text": "None." }, + "signals.ordered": { "text": "Ordered by how much each one moved the score, strongest first, rather than by where it appears in the text." }, + "signals.more": { "text": "… and {0} more, none of which moved the score as much as any of the above." }, "observations.intro": { "text": "Measured against writing published before generative models existed. Shown because they are real, and counted for nothing because they are ordinary." }, "observations.row.one": { "text": "- {0} — {1} occurrence" }, "observations.row.other": { "text": "- {0} — {1} occurrences" }, diff --git a/src/SignsOfAI.Core/Reporting/report.es.json b/src/SignsOfAI.Core/Reporting/report.es.json index 9fd1efd..5434333 100644 --- a/src/SignsOfAI.Core/Reporting/report.es.json +++ b/src/SignsOfAI.Core/Reporting/report.es.json @@ -42,6 +42,14 @@ "text": "Señales contabilizadas", "sourceHash": "078671a3b913dc8d830dc433445ca4b4cc401debf4b6fb8829ea9376ab658cc2" }, + "signals.ordered": { + "text": "Ordenadas por cuánto movió cada una la puntuación, de mayor a menor, y no por el lugar que ocupan en el texto.", + "sourceHash": "6328aa4d5b8ee1e8742dad32fd9d8a92fe928d606938710e706be22d0be26f15" + }, + "signals.more": { + "text": "… y {0} más, ninguna de las cuales movió la puntuación tanto como las de arriba.", + "sourceHash": "754e025aefdb15ea0b58eb55aecaa82c9da6799259ed0f2c64d33ccac3cfb558" + }, "section.observations": { "text": "Encontrado, pero a una frecuencia habitual en textos humanos", "sourceHash": "828c9cc6ac44b392c9e7a358d18a7c9771519012e0c878ac3c02e367c5f75338" diff --git a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs index 6d92f8c..75326a6 100644 --- a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs +++ b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs @@ -1,5 +1,6 @@ using SignsOfAI.Core; using SignsOfAI.Core.Calibration; +using SignsOfAI.Core.Model; using SignsOfAI.Core.Reporting; using System.Text.RegularExpressions; @@ -377,6 +378,168 @@ public void A_supported_interface_language_carries_no_apology() Assert.DoesNotContain("no está disponible en", report); } + /// + /// A result assembled by hand. The ordering defects need a document with more findings than the + /// report prints and a known strongest one, and building that out of real prose would make the + /// test a statement about which rules happen to fire rather than about what the report keeps. + /// + private static AnalysisResult Synthetic(IReadOnlyList findings) => new() + { + Language = "en", + RulePackLanguage = "en", + Findings = findings, + CategoryScores = [], + OverallScore = 30, + Statistics = new TextStatistics { WordCount = 4000, SentenceCount = 200 }, + }; + + private static Finding Signal(string id, double weight, int start, string matched) => new() + { + RuleId = id, + Category = SignCategory.Lexical, + Severity = Severity.Medium, + Span = new TextSpan(start, matched.Length), + MatchedText = matched, + Message = "Reads as a machine tell.", + Suggestion = "Say it another way.", + Weight = weight, + }; + + [Fact] + public void The_strongest_evidence_survives_a_document_longer_than_the_report() + { + // The defect this replaces: findings arrive in the order they occur in the text, the report + // cut the list at forty, and so a long document spent the whole list on weak hits in its + // opening pages while the finding that did most to produce the headline number — in the last + // paragraph — was dropped. The reader was handed a score the visible evidence could not + // account for, in the document this project builds for a room where somebody is judged. + var findings = Enumerable.Range(0, 60) + .Select(i => Signal("lex.weak", 1.0, i * 100, $"weak-{i:000}")) + .Append(Signal("lex.strong", 9.0, 99_000, "unmistakable-tell")) + .ToList(); + + var report = EvidenceReport.ToMarkdown(Synthetic(findings)); + + Assert.Contains("unmistakable-tell", report); + Assert.DoesNotContain("weak-059", report); + } + + [Fact] + public void Says_what_it_left_out_and_that_none_of_it_outweighed_what_is_shown() + { + // "… and 21 more" over a list in document order says the report stopped reading. Over a list + // in weight order it can say something stronger and true: there is more, and none of it is + // heavier than what you are looking at. The claim is only earned by sorting first. + var findings = Enumerable.Range(0, 61) + .Select(i => Signal("lex.weak", 1.0, i * 100, $"weak-{i:000}")) + .ToList(); + + var report = EvidenceReport.ToMarkdown(Synthetic(findings)); + + Assert.Contains("Ordered by how much each one moved the score", report); + Assert.Contains("21 more, none of which moved the score as much as any of the above", report); + } + + [Fact] + public void A_spanish_reader_is_told_the_order_in_Spanish() + { + // The line that explains why the list is not in the order of their student's document is the + // one a reader most needs in their own language, and a stale pin would drop it back to + // English without saying anything was wrong with it. + var findings = Enumerable.Range(0, 61) + .Select(i => Signal("lex.weak", 1.0, i * 100, $"weak-{i:000}")) + .ToList(); + + var report = EvidenceReport.ToMarkdown(Synthetic(findings), + new ReportOptions { InterfaceLanguage = "es" }); + + Assert.Contains("Ordenadas por cuánto movió cada una la puntuación", report); + Assert.Contains("21 más, ninguna de las cuales movió la puntuación", report); + } + + [Fact] + public void A_strong_character_is_not_pushed_out_of_the_table_by_soft_hyphens() + { + // Word inserts soft hyphens unprompted, so a real file can hold hundreds of them and one + // letter borrowed from another alphabet. In file order the innocent ones fill the table and + // the one occurrence that is hard to arrive at by accident falls off the end — of the table + // this project points at when it says a character is a fact rather than an opinion. + var text = string.Concat(Enumerable.Repeat("sepa­ration of the parts. ", 60)) + + "The final delveе stands alone."; + + var report = EvidenceReport.ToMarkdown(new AiWritingAnalyzer().Analyze(text, "en")); + + Assert.Contains("Letter from another alphabet", report); + } + + [Fact] + public void The_markdown_form_does_not_carry_live_html_into_a_comment_box() + { + // Markdown is the form documented here for pasting into an LMS comment box or a GitHub issue, + // and both render raw HTML embedded in Markdown. ToHtml escaped on its way out so the HTML + // page was never at risk; the Markdown was, and it is the one a teacher is told to forward. + var findings = new[] { Signal("lex.x", 3.0, 0, "") }; + + var markdown = EvidenceReport.ToMarkdown(Synthetic(findings)); + + // Neutralised where a renderer reads it, and still legible where a person does: the escape + // is a backslash, so the teacher reading the raw file sees what the document actually said. + Assert.Contains(@"\", markdown); + + // And no bracket that opens a tag survived unescaped anywhere on the page. + Assert.DoesNotContain(" becomes \\") }; + + var markdown = EvidenceReport.ToMarkdown(Synthetic(findings)); + + Assert.DoesNotContain("", markdown); + + // The HTML resolves both escapes and shows exactly what the document said, once. + var html = EvidenceReport.ToHtml(Synthetic(findings)); + Assert.Contains(@"\<script>alert(1)</script>", html); + Assert.DoesNotContain("") }; + + var html = EvidenceReport.ToHtml(Synthetic(findings)); + + Assert.Contains("<img src=x>", html); + Assert.DoesNotContain("\\<", html); + Assert.DoesNotContain("&lt;", html); + } + + [Fact] + public void A_pipe_in_a_list_item_is_shown_as_a_pipe() + { + // Table cells lost their backslash in SplitRow, which has to resolve it before it can tell a + // column boundary from a pipe inside a filename. List items had no such step, so the escape + // that protected the table leaked onto the page everywhere else. + var findings = new[] { Signal("lex.x", 3.0, 0, "either|or") }; + + var html = EvidenceReport.ToHtml(Synthetic(findings)); + + Assert.Contains("either|or", html); + Assert.DoesNotContain("either\\|or", html); + } + [Fact] public void Puts_the_checkable_facts_in_the_headline() {