Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/SignsOfAI.Cli/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,12 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor)
string Col(string s, int code) => useColor ? $"[{code}m{s}" : s;
string Bold(string s) => useColor ? $"{s}" : s;

int scoreColor = r.OverallScore switch { >= 70 => 31, >= 45 => 33, >= 20 => 33, _ => 32 };
int scoreColor = VerdictBands.Emphasis(r.OverallScore) switch
{
VerdictEmphasis.High => 31,
VerdictEmphasis.Elevated or VerdictEmphasis.Present => 33,
_ => 32,
};
Console.WriteLine();
Console.WriteLine(Bold($" ✍ Signs of AI Writing — {Path.GetFileName(path)}"));
Console.WriteLine($" {Col($"{r.OverallScore:0}/100", scoreColor)} {Bold(r.Verdict)} " +
Expand Down
17 changes: 12 additions & 5 deletions src/SignsOfAI.Core/Model/AnalysisResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,19 @@ public sealed record AnalysisResult
/// </summary>
public CitationReport Citations { get; init; } = CitationReport.Empty;

/// <summary>Human-readable one-line verdict derived from <see cref="OverallScore"/>.</summary>
public string Verdict => OverallScore switch
/// <summary>
/// Human-readable one-line verdict derived from <see cref="OverallScore"/>, in English.
///
/// English-only on purpose: this is what a machine consumer gets — the CLI's `--json`, the MCP
/// tool's payload — where a stable string is more use than a translated one. Anything shown to a
/// person goes through the interface's localiser or the report's own resources, both of which
/// take their boundary from <see cref="VerdictBands"/> exactly as this does.
/// </summary>
public string Verdict => VerdictBands.Emphasis(OverallScore) switch
{
>= 70 => "Strong signs of AI writing",
>= 45 => "Moderate signs of AI writing",
>= 20 => "Light signs of AI writing",
VerdictEmphasis.High => "Strong signs of AI writing",
VerdictEmphasis.Elevated => "Moderate signs of AI writing",
VerdictEmphasis.Present => "Light signs of AI writing",
_ => "Reads mostly human",
};
}
90 changes: 90 additions & 0 deletions src/SignsOfAI.Core/Model/VerdictBands.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
using SignsOfAI.Core.Calibration;

namespace SignsOfAI.Core.Model;

/// <summary>
/// Where the verdict changes, and the only place that decides it.
///
/// It used to be decided in eight: the analysis result, the report, the interface's localiser, the
/// CLI's colour picker, two switches in the web page, the batch page and the live-rewrite panel —
/// each with the numbers written out again, kept in step by a comment reading "mirrors the bands in
/// AnalysisResult.Verdict". They did not stay in step, and the drift was not cosmetic: the report
/// withheld the verdict from every document ever analysed while the interface and the CLI printed
/// one for every document ever analysed. One engine gave three answers about the same text.
/// </summary>
public static class VerdictBands
{
/// <summary>
/// The score at which this build is willing to say something about a text, taken from the
/// calibration it ships with rather than chosen.
///
/// Null when no calibration is embedded — a fork that has never measured itself. That case must
/// stay silent rather than inherit a boundary somebody else measured on somebody else's corpus,
/// which is the same rule the report already follows for the error rate.
/// </summary>
public static double? Threshold => PublishedCalibration.Current?.RecommendedThreshold;

/// <summary>
/// Whether a score has earned a verdict at all. Below the boundary the number stands alone: a
/// low score is not evidence that a person wrote something, and the wording must not imply it.
/// </summary>
public static bool Holds(double score) => Threshold is { } threshold && score >= threshold;

/// <summary>
/// The same question for a document in a named language, which is stricter and has to be.
///
/// There are three states here, not two, and collapsing them is how this went wrong before:
///
/// <list type="bullet">
/// <item>A language <b>in the corpus</b> — English, Spanish — has a measured false-positive bound
/// of its own, even when its sample is too small to set its own boundary. It borrows the pooled
/// boundary and the page prints its bound beside the verdict, so the reader weighs the right
/// number.</item>
/// <item>A language <b>absent from the corpus</b> has no bound to print. A verdict there would
/// imply a reliability nobody measured, and there would be nothing on the page to correct the
/// impression. It gets the score and the reason it gets nothing else.</item>
/// <item>No calibration at all — a fork that has never measured itself — speaks about nothing.</item>
/// </list>
/// </summary>
public static bool Holds(double score, string? language) => Holds(score) && Measured(language);

/// <summary>Whether the corpus contains this language at all, however thinly.</summary>
public static bool Measured(string? language) =>
PublishedCalibration.Current?.For(language) is not null;

/// <summary>
/// How loudly to present a score that has earned a verdict.
///
/// The two upper cuts are a **display convention and nothing more**. No text in the calibration
/// corpus came within twenty points of them — the highest scoring human text reached 23.4 — so
/// the corpus can locate <see cref="Threshold"/> and can say nothing whatever about 45 or 70.
/// Separating "moderate" from "strong" would need machine-written text, and Docs/CALIBRATION.md
/// argues at length against collecting any: it dates badly and flatters whoever assembles it.
///
/// They survive here to colour a reading, never to make a claim. See issue #32.
/// </summary>
public static VerdictEmphasis Emphasis(double score) => score switch
{
_ when !Holds(score) => VerdictEmphasis.None,
>= 70 => VerdictEmphasis.High,
>= 45 => VerdictEmphasis.Elevated,
_ => VerdictEmphasis.Present,
};
}

/// <summary>
/// How prominently a verdict is shown. Not a measurement, and deliberately not a number: anything
/// that reads as a quantity here would be read as one that was measured, and only the boundary
/// between <see cref="None"/> and the rest was.
/// </summary>
public enum VerdictEmphasis
{
/// <summary>Below the boundary this build can support. The score stands without a verdict.</summary>
None,

Present,

Elevated,

High,
}
36 changes: 21 additions & 15 deletions src/SignsOfAI.Core/Reporting/EvidenceReport.cs
Original file line number Diff line number Diff line change
Expand Up @@ -373,19 +373,25 @@ public static string FolderToHtml(
}

/// <summary>
/// Whether the score has earned the right to carry a verdict: only at or above the threshold
/// measured for the language actually analysed. Everywhere else the number stands alone.
/// Whether the score has earned the right to carry a verdict.
///
/// This asked for the threshold measured for the language actually analysed, refusing to borrow
/// the aggregate. That reading was too strict by a wide margin, and the cost was not theoretical:
/// no language in the corpus supports its own threshold — English has 65 texts and Spanish 25,
/// against the ~75 the interval needs — so the condition was false for every document in every
/// language, and the exported report never carried a verdict at all. A tool that renders nothing
/// has not been careful, it has been switched off, and Spanish would have stayed switched off
/// for years while the page still promised a reading.
///
/// The distinction that resolves it: borrowing the aggregate *error rate* would misstate how
/// often this build is wrong about Spanish (13.3% measured, against 5.6% for English and 4.1%
/// pooled), and that remains forbidden — the caveat below still quotes the language's own bound
/// and never the pooled one. Borrowing the aggregate *boundary* states nothing about reliability;
/// it decides when the tool speaks, and it is published, measured and printed on the page beside
/// the language's own figure. See issue #32.
/// </summary>
private static bool VerdictHolds(AnalysisResult result)
{
var c = PublishedCalibration.Current;
if (c is null) return false;

// Never borrow the aggregate. A language absent from the corpus has no supported verdict,
// even when the combined EN/ES sample happens to support one.
return c.For(result.Language)?.RecommendedThreshold is { } threshold
&& result.OverallScore >= threshold;
}
private static bool VerdictHolds(AnalysisResult result) =>
VerdictBands.Holds(result.OverallScore, result.Language);

/// <summary>
/// The sentence that has to appear on every report. Written from the embedded calibration so it
Expand Down Expand Up @@ -493,9 +499,9 @@ private static void HowOftenWrong(StringBuilder sb, ReportText text, string? lan

private static string Verdict(ReportText text, double score) => text.Get(score switch
{
>= 70 => ReportMessages.VerdictStrong,
>= 45 => ReportMessages.VerdictModerate,
>= 20 => ReportMessages.VerdictLight,
_ when VerdictBands.Emphasis(score) is VerdictEmphasis.High => ReportMessages.VerdictStrong,
_ when VerdictBands.Emphasis(score) is VerdictEmphasis.Elevated => ReportMessages.VerdictModerate,
_ when VerdictBands.Emphasis(score) is VerdictEmphasis.Present => ReportMessages.VerdictLight,
_ => ReportMessages.VerdictMinimal,
}).Text;

Expand Down
8 changes: 4 additions & 4 deletions src/SignsOfAI.UI/Components/LiveRewritePanel.razor
Original file line number Diff line number Diff line change
Expand Up @@ -269,11 +269,11 @@
if (cursor < Text.Length) yield return new Seg(Text[cursor..], null);
}

private static string ScoreClass(double score) => score switch
private static string ScoreClass(double score) => VerdictBands.Emphasis(score) switch
{
>= 70 => "danger",
>= 45 => "warn",
>= 20 => "notice",
VerdictEmphasis.High => "danger",
VerdictEmphasis.Elevated => "warn",
VerdictEmphasis.Present => "notice",
_ => "good",
};
}
8 changes: 5 additions & 3 deletions src/SignsOfAI.UI/Pages/Batch.razor
Original file line number Diff line number Diff line change
Expand Up @@ -214,10 +214,12 @@ else
}
}

private static string BandClass(int score) => score switch
// This said 40 where every other surface said 45 — the drift the single source exists to stop,
// and it had already happened: one document could be coloured two ways by the same build.
private static string BandClass(int score) => VerdictBands.Emphasis(score) switch
{
>= 70 => "high",
>= 40 => "medium",
VerdictEmphasis.High => "high",
VerdictEmphasis.Elevated => "medium",
_ => "low",
};
}
16 changes: 8 additions & 8 deletions src/SignsOfAI.UI/Pages/Home.razor
Original file line number Diff line number Diff line change
Expand Up @@ -715,19 +715,19 @@ else
$"signsofai-report-{stamp}.html", EvidenceReport.ToHtml(r, options));
}

private static string ScoreClass(double score) => score switch
private static string ScoreClass(double score) => VerdictBands.Emphasis(score) switch
{
>= 70 => "danger",
>= 45 => "warn",
>= 20 => "notice",
VerdictEmphasis.High => "danger",
VerdictEmphasis.Elevated => "warn",
VerdictEmphasis.Present => "notice",
_ => "good",
};

private static string ScoreHex(double score) => score switch
private static string ScoreHex(double score) => VerdictBands.Emphasis(score) switch
{
>= 70 => "#dc2626",
>= 45 => "#ea580c",
>= 20 => "#ca8a04",
VerdictEmphasis.High => "#dc2626",
VerdictEmphasis.Elevated => "#ea580c",
VerdictEmphasis.Present => "#ca8a04",
_ => "#16a34a",
};

Expand Down
13 changes: 8 additions & 5 deletions src/SignsOfAI.UI/Services/Loc.cs
Original file line number Diff line number Diff line change
Expand Up @@ -245,12 +245,15 @@ private async Task LogAsync(string message)

public string Sev(Severity severity) => this["sev." + severity.ToString().ToLowerInvariant()];

/// <summary>The one-line verdict for an overall score. Mirrors the bands in <c>AnalysisResult.Verdict</c>.</summary>
public string Verdict(double score) => score switch
/// <summary>
/// The one-line verdict for an overall score, in the interface's language. The boundary comes
/// from <see cref="VerdictBands"/>; this only chooses the words for it.
/// </summary>
public string Verdict(double score) => VerdictBands.Emphasis(score) switch
{
>= 70 => this["verdict.strong"],
>= 45 => this["verdict.moderate"],
>= 20 => this["verdict.light"],
VerdictEmphasis.High => this["verdict.strong"],
VerdictEmphasis.Elevated => this["verdict.moderate"],
VerdictEmphasis.Present => this["verdict.light"],
_ => this["verdict.minimal"],
};

Expand Down
88 changes: 88 additions & 0 deletions tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
using SignsOfAI.Core.Calibration;
using SignsOfAI.Core.Model;
using SignsOfAI.Core.Reporting;

namespace SignsOfAI.Core.Tests;

/// <summary>
/// One text, one engine, one answer.
///
/// Nothing in this suite used to notice that the three surfaces disagreed, which is why they were
/// allowed to disagree for weeks. The exported report withheld the verdict from every document ever
/// analysed — its gate demanded a per-language threshold, and no language in the corpus has one —
/// while the CLI and the web interface printed a hand-picked band on every document ever analysed.
/// A text scoring 90/100 was called "Strong signs of AI writing" on screen and given no verdict at
/// all in the document a teacher would print and carry to a committee.
///
/// These tests exist so that the next disagreement fails a build instead of shipping.
/// </summary>
public class VerdictAgreementTests
{
/// <summary>Dense with the tells the packs describe: a score at the top of the scale.</summary>
private const string ObviouslyMachine =
"In today's rapidly evolving digital landscape, it is important to note that artificial " +
"intelligence has fundamentally transformed how we approach problem-solving. Moreover, this " +
"comprehensive framework facilitates a robust understanding of the underlying mechanisms. " +
"Furthermore, it is worth noting that such systems not only enhance productivity but also " +
"streamline operations. In order to leverage these capabilities, organizations must delve " +
"into the intricacies of implementation. Additionally, the multifaceted nature of these " +
"tools underscores their pivotal role.";

[Fact]
public void The_report_speaks_about_a_text_the_product_calls_the_strongest_case_it_has()
{
var result = new AiWritingAnalyzer().Analyze(ObviouslyMachine, "en");

// Not a claim that the score is right — only that a document scoring this high is the case
// the tool exists for. If the report stays silent here it stays silent everywhere.
Assert.True(result.OverallScore >= 70,
$"The fixture stopped being an extreme case (scored {result.OverallScore:0}); " +
"pick a stronger one rather than lowering this bar.");

var report = EvidenceReport.ToMarkdown(result);

Assert.DoesNotContain("no verdict is given", report);
}

[Fact]
public void No_language_is_silenced_merely_because_its_own_corpus_is_small()
{
// The per-language threshold needs roughly seventy-five texts. Spanish has twenty-five and
// will not reach seventy-five for a long time. A gate that waits for it does not protect a
// Spanish writer; it withholds the tool from them while serving everyone else — and on this
// corpus it withholds it from everyone, because English has sixty-five.
var calibration = PublishedCalibration.Current;
Assert.NotNull(calibration);

foreach (var language in calibration!.Languages)
Assert.True(language.RecommendedThreshold is null,
$"'{language.Language}' now supports its own threshold. That is good news and it " +
"makes this test's premise obsolete — keep the fallback, but re-read it.");

var spanish = new AiWritingAnalyzer().Analyze(
"En el panorama actual, es importante destacar que la inteligencia artificial ha " +
"transformado fundamentalmente nuestro enfoque. Además, este marco integral facilita " +
"una comprensión robusta de los mecanismos subyacentes. Asimismo, cabe señalar que " +
"dichos sistemas no solo mejoran la productividad sino que también optimizan las " +
"operaciones.", "es");

var report = EvidenceReport.ToMarkdown(spanish);

// Whatever it says about Spanish, it must not be silence justified by a missing number that
// the aggregate already supplies.
Assert.Contains("13.3%", report);
}

[Fact]
public void The_boundary_the_product_draws_is_the_boundary_the_project_publishes()
{
// The product used to say "light signs of AI writing" from 20, five points below the only
// threshold this project publishes, and below the highest-scoring human text in the corpus
// (23.4). Publishing a measured figure while shipping an unmeasured boundary is the
// inconsistency this project exists to complain about in other tools.
var threshold = PublishedCalibration.Current?.RecommendedThreshold;
Assert.NotNull(threshold);

Assert.Equal(threshold!.Value, VerdictBands.Threshold);
}
}
Loading