diff --git a/docs/cli/command-reference.md b/docs/cli/command-reference.md index 0c39c4f..b25ac5a 100644 --- a/docs/cli/command-reference.md +++ b/docs/cli/command-reference.md @@ -238,6 +238,17 @@ it eases towards 0.632 as the history lengthens. On a window shorter than eight slice narrows to a single run and one run's absence never reaches the bar, so the kind is silent there — see [known limitations](../known-limitations.md). +It also asks only the runs that were in a position to answer. A run under a `dotnet test --filter` +did not fail to see the tests it excluded; it never looked for them, and counting its silence makes +every unselected test look deleted. A run whose distinct tests are under half those of the largest +run in the window is therefore set aside, the rest are re-split into their own earlier and current +runs, and the finding's denominators count only those — the sentence says `full runs` and a +`set aside` metric gives the number left out. The summary line reports how many runs covered part of +the suite. Only this kind sets them aside; every other kind still counts them in full, because a +filtered run's outcomes are as true as any other run's. The cost is that a deletion removing more +than half a suite is indistinguishable from a filter and is not reported — see +[known limitations](../known-limitations.md). + ### Finding ids The `f_…` on each finding is a short, stable identity for that finding — a hash of what the @@ -328,7 +339,7 @@ Every finding carries a `headline` — the same sentence the rendered report pri ```json { - "schemaVersion": "1.12", + "schemaVersion": "1.13", "window": { "sessionCount": 20, "resolution": "default", "currentSliceSize": 3 }, "context": { "sha": "a3f9c2e", "branch": "main", "assembly": "Checkout.Tests" }, "summary": { @@ -337,7 +348,8 @@ Every finding carries a `headline` — the same sentence the rendered report pri "counts": { "high": 1, "medium": 2, "low": 0 }, "healthy": 409, "excludedLowEvidence": 41, - "excludedNotSignificant": 6 + "excludedNotSignificant": 6, + "partialSessions": 0 }, "findings": [ { diff --git a/docs/known-limitations.md b/docs/known-limitations.md index e76c2c2..4e472c5 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -319,6 +319,44 @@ anything stopped — but it does mean a fresh store says nothing about deleted t eight runs in it. The run rate and the p-value are published with every finding that does clear the bar. +--- + +### `Vanished` Cannot Tell A Filtered Run From A Deletion, So It Trusts Neither Below Half A Suite + +**Impact**: a run that covered less than half the suite is counted on neither side of an absence, so +a deletion that removed more than half a suite is never reported. A `--filter` selecting *more* than +half the suite still produces one false `stopped running` per unselected test. + +**Reason**: a run under a `dotnet test --filter` did not fail to see the tests it excluded — it never +looked for them. Counting its silence makes every unselected test look deleted, which is what an +ordinary inner loop produces in about ten minutes: a handful of full runs, then a stream of filtered +ones, then a report claiming most of the suite has stopped running. Every statement in it is true of +the data and false about the world. + +So the report classifies each run by how much of the suite it covered — its distinct tests against +the largest run in the window, which is the best evidence the window holds of how big the suite is, +and the only anchor a store of mostly filtered runs does not corrupt. The median does not work: four +full runs and sixteen filtered ones has a median of one test, and every run in it measures as +typical. A report is scoped to exactly one assembly, so the runs being compared are always runs of +the same suite. Runs covering less than half are set aside, the remaining runs are re-split into +their own "now" and "before", and the counts on the finding are of those runs alone — `full runs` in +the sentence, with a `set aside` metric saying how many were left out. + +**The trade, stated rather than discovered**: a count cannot separate the two cases, and no threshold +makes it able to. Nine tests missing from a suite of seventeen is the same table whether they were +excluded or removed. The line is placed at a half — the point at which a run stopped being a run of +the suite and became a run of part of it — and it is deliberately biased towards silence: `Vanished` +is capped at `Severity.Low` because a disappearance is usually something the developer did on purpose +a minute ago, so a missed one costs little, whereas the false positive arrives once per unselected +test on every filtered run for as long as it stays in the window. + +**Related**: setting runs aside shortens the history this kind measures against, so a store whose +runs are mostly filtered can fall below the eight runs the section above requires and report nothing +at all. Only the kinds that read absence set these runs aside — every other kind still counts them in +full, because a filtered run's *outcomes* are as true as any other run's and it is only its silences +that mean nothing. The summary line says how many runs covered part of the suite, so the distinction +is visible rather than inferred. + ### `RetryExhausted` Is Observed, And The Declared Retry Limit Is Not Interpreted **Impact**: a test whose retry attribute allows three retries but which only ever recorded two @@ -491,3 +529,4 @@ When reporting, please include: | 1.7.0 | Documented what `TimeSensitive` now charges for searching three axes, and what that costs | | 1.8.0 | Documented what `ParallelSensitive` now measures, and the duration confound it cannot correct | | 1.9.0 | Documented the run rate `Vanished` now requires, and the window size below which it is silent | +| 1.10.0 | Documented how `Vanished` treats a run that covered part of the suite, and what that trade costs | diff --git a/src/Xping.Cli/Report/AnalysisContext.cs b/src/Xping.Cli/Report/AnalysisContext.cs index c7b5672..5a79569 100644 --- a/src/Xping.Cli/Report/AnalysisContext.cs +++ b/src/Xping.Cli/Report/AnalysisContext.cs @@ -44,9 +44,28 @@ public AnalysisContext(AnalysisWindow window, RevisionContext? revision) for (int position = 0; position < window.Sessions.Count; position++) views.Add(SessionView.For(window.Sessions[position], position)); + // Partiality is a comparison and not a measurement, so it needs the whole window before any + // one session can be classified. The anchor is the largest run in it: the best evidence the + // window holds of how big the suite is, and the only one a store of mostly filtered runs + // does not corrupt — see LocalAnalysisConstants.PartialSessionShare. + int suiteSize = 0; + foreach (SessionView view in views) + suiteSize = Math.Max(suiteSize, view.Tests); + + if (suiteSize > 0) + { + double floor = LocalAnalysisConstants.PartialSessionShare * suiteSize; + for (int position = 0; position < views.Count; position++) + { + if (views[position].Tests < floor) + views[position] = views[position] with { IsPartial = true }; + } + } + SessionViews = views; _viewsBySession = views.ToDictionary(v => v.Session.SessionId); EnvironmentalSessionCount = views.Count(v => v.IsLikelyEnvironmental); + PartialSessionCount = views.Count(v => v.IsPartial); } /// Gets the sessions under analysis and the boundaries that produced them. @@ -71,6 +90,17 @@ public AnalysisContext(AnalysisWindow window, RevisionContext? revision) /// public int EnvironmentalSessionCount { get; } + /// + /// Gets how many analysed sessions covered only part of the suite. + /// + /// + /// Reported in the summary as an observation rather than a discount. Only the kinds that read + /// absence set these sessions aside — every other kind reads outcomes of executions that + /// happened, and a filtered run's outcomes are as true as any other's — so the line says what + /// the window contains and does not claim the numbers were adjusted. + /// + public int PartialSessionCount { get; } + /// /// Gets the health of one analysed session. /// diff --git a/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs b/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs index 7e3fa7e..c2b1378 100644 --- a/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs +++ b/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs @@ -84,6 +84,7 @@ public static ReportEnvelope Build( result.ExcludedNotSignificant, context.EnvironmentalSessionCount, + context.PartialSessionCount, incompleteSessions, unreadableSessions, result.FailedProviders), diff --git a/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs b/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs index 329f67f..c9b329a 100644 --- a/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs +++ b/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs @@ -536,27 +536,57 @@ private static (string, IReadOnlyList) TimeSensitive(TimeSensitiveEvi /// The evidence. /// The headline and its metrics. /// + /// /// The headline carries the denominator and not the p-value, because here the denominator is the /// discriminating figure: "3 of 17" and "17 of 17" are visibly different claims to a reader /// skimming the fence, where the two arms of a split are not. The p-value is a metric, for the /// reader who opens the finding to check how much belief the sentence earned. + /// + /// + /// Both denominators count only the runs that covered the suite, so where any were set aside + /// both clauses say "full runs" and a metric says how many were left out. Both, because "the + /// last 3" and "the last 3 full runs" are not the same three runs once anything has been set + /// aside, and a clause that is only true given the one before it is the kind a reader pastes + /// into a chat window on its own. Qualified rather than worded that way always, because in a + /// store with no filtered runs there is nothing for the word to distinguish the runs from and + /// the shorter sentence is the true one. The wording cannot destabilise the finding's id, which + /// hashes the kind and the subject and nothing else — see . + /// /// - private static (string, IReadOnlyList) Vanished(VanishedEvidence e) => - ( - $"ran in {e.BaselineSessions} of {e.BaselineSessionCount} earlier runs, " + - $"absent from the last {e.CurrentSessionCount}", + private static (string, IReadOnlyList) Vanished(VanishedEvidence e) + { + bool anySetAside = e.PartialSessionsSetAside > 0; + string runs = anySetAside ? "full runs" : "runs"; + + List metrics = [ new( "ran in", - $"{e.BaselineSessions} of {e.BaselineSessionCount} earlier runs " + + $"{e.BaselineSessions} of {e.BaselineSessionCount} earlier {runs} " + $"({Percent(e.BaselineRunRate)})"), - new("absent from", $"the last {e.CurrentSessionCount} runs"), + new("absent from", $"the last {e.CurrentSessionCount} {runs}"), new("executions", e.ExecutionsInWindow.ToString(CultureInfo.InvariantCulture)), // One-sided, and legitimately so: the kind only ever forms a table for a test already // absent, so the direction was fixed before the counts were. new("significance", $"p {Probability(e.PValue)} one-sided") - ]); + ]; + + // Only where there were any. A "0 set aside" line on every finding in every ordinary store + // would be noise standing in for the absence of a caveat. + if (anySetAside) + metrics.Add(new("set aside", $"{Runs(e.PartialSessionsSetAside)} that covered part of the suite")); + + // The trailing noun only where it discriminates. On an ordinary store "the last 3" can mean + // nothing but the last three runs, and the sentence is the one this kind has always printed. + string absence = anySetAside + ? $"absent from the last {e.CurrentSessionCount} {runs}" + : $"absent from the last {e.CurrentSessionCount}"; + + return ( + $"ran in {e.BaselineSessions} of {e.BaselineSessionCount} earlier {runs}, {absence}", + metrics); + } private static string Times(int count) => count == 1 ? "once" : $"{count.ToString(CultureInfo.InvariantCulture)} times"; diff --git a/src/Xping.Cli/Report/Contract/ReportEnvelope.cs b/src/Xping.Cli/Report/Contract/ReportEnvelope.cs index 5999aaf..c0d39d1 100644 --- a/src/Xping.Cli/Report/Contract/ReportEnvelope.cs +++ b/src/Xping.Cli/Report/Contract/ReportEnvelope.cs @@ -39,14 +39,12 @@ internal sealed record ReportEnvelope( /// Moves whenever anything a consumer reads changes shape, and the per-kind evidence payloads /// are part of that even though this document describes them as opaque: a script that reached /// into evidence for a field this build no longer emits is reading a contract, and - /// leaving the number still would tell it nothing had moved. 1.12 is where every evidence - /// record renamed the fields a rate is taken over — executionsConsidered and - /// sessionsConsidered where discounting applied, executionsInWindow and - /// sessions where it did not — split discountedExecutions into its environmental - /// and clustered halves, and where the finding gained population, which says which of - /// those the numbers beside it describe. + /// leaving the number still would tell it nothing had moved. 1.13 is where the summary gained + /// partialSessions and VanishedEvidence gained partialSessionsSetAside — + /// the runs that covered only part of the suite, which an absence is counted on neither side + /// of. /// - public const string CurrentSchemaVersion = "1.12"; + public const string CurrentSchemaVersion = "1.13"; } /// @@ -95,6 +93,12 @@ internal sealed record ContextDto(string? Sha, string? Branch, string? Assembly) /// reader given only an empty block cannot tell that from a suite with nothing to report. /// /// Sessions discounted as environment failures. +/// +/// Sessions that covered only part of the suite — a dotnet test --filter run, or anything +/// else that ran a fraction of the tests the window's largest run did. An observation and not a +/// discount: only the kinds that read absence set such a session aside, because a filtered run's +/// outcomes are as true as any other run's and it is only its silences that mean nothing. +/// /// Sessions found but not finalised. /// Session files that could not be read. /// Metrics that threw and produced nothing. @@ -106,6 +110,7 @@ internal sealed record SummaryDto( int ExcludedLowEvidence, int ExcludedNotSignificant, int EnvironmentalSessions, + int PartialSessions, int IncompleteSessions, int UnreadableSessions, IReadOnlyList FailedProviders); diff --git a/src/Xping.Cli/Report/Indexes/SessionView.cs b/src/Xping.Cli/Report/Indexes/SessionView.cs index 952cb59..d8d9a35 100644 --- a/src/Xping.Cli/Report/Indexes/SessionView.cs +++ b/src/Xping.Cli/Report/Indexes/SessionView.cs @@ -21,6 +21,11 @@ namespace Xping.Cli.Report.Indexes; /// It is a classification against two thresholds, not a claim about what went wrong. The report says /// a session looks environmental; it never says why, and it never says so about a test. /// +/// +/// is the other flag, and it answers a different question: not whether the +/// session's outcomes can be trusted, but whether its silences can. It is window-relative, so +/// cannot decide it — see . +/// /// /// The session itself. /// Its position in the window; 0 is the newest. @@ -38,6 +43,30 @@ internal sealed record SessionView( double FailureRate, bool IsLikelyEnvironmental) { + /// + /// Gets a value indicating whether the session covered only part of the suite. + /// + /// + /// + /// A run under a dotnet test --filter is not a run in which the tests it excluded failed + /// to appear — it is a run that never asked about them. A kind reading absence has to set such a + /// session aside, or every unselected test looks deleted. + /// + /// + /// Not a positional member, because it cannot be measured from one session: it is + /// against of the + /// largest run in the window, and the window is what does not have. + /// sets it once the whole window is measured, which is also the + /// only place it means anything. + /// + /// + /// A classification against a threshold and not a claim about what happened, in the same way + /// is. The report says a session covered part of the suite; + /// it never says a filter was the reason, because a deletion produces the same count. + /// + /// + public bool IsPartial { get; init; } + /// /// Measures one session. /// diff --git a/src/Xping.Cli/Report/LocalAnalysisConstants.cs b/src/Xping.Cli/Report/LocalAnalysisConstants.cs index 964d2ed..a2c07b8 100644 --- a/src/Xping.Cli/Report/LocalAnalysisConstants.cs +++ b/src/Xping.Cli/Report/LocalAnalysisConstants.cs @@ -77,6 +77,43 @@ internal static class LocalAnalysisConstants /// public const int SmallWindowSessionCount = 8; + /// + /// Share of the window's largest run a session must cover to count as a run of the suite (0.50). + /// + /// + /// + /// A dotnet test --filter run is an ordinary thing to find in a local store, and a + /// session that ran one test of seventeen says nothing whatever about the other sixteen. Absence + /// is only evidence where the run was in a position to produce a presence, so a session covering + /// a small part of the suite is set aside by the kinds that read absence rather than outcome — + /// today that is alone. + /// + /// + /// The anchor is the largest session in the window and deliberately not its median. A store of + /// four full runs and sixteen filtered ones — which is what ten minutes of an inner loop + /// produces — has a median of one test, and every session in it would measure as typical + /// against that. The largest run is the best evidence the window holds of how big the suite + /// actually is, and a report is scoped to exactly one assembly, so the sessions being compared + /// are always runs of the same suite. + /// + /// + /// A count cannot tell a filtered run from a deletion, and no threshold makes it able to: a run + /// of eight tests where the suite has seventeen is the same table whether nine tests were + /// excluded or removed. So the line is a trade rather than a discovery, and it is placed at a + /// half — the point at which a run stopped being a run of the suite and became a run of part of + /// it. What that costs, in both directions: a filter selecting more than half the suite still + /// produces false absences, and a deletion of more than half a suite is never reported at all. + /// + /// + /// Biased towards silence on purpose. is capped at + /// because a disappearance is usually something the developer + /// did on purpose thirty seconds ago, so a missed one costs almost nothing; whereas the false + /// positive arrives one per unselected test, on every filtered run, for as long as the run stays + /// in the window. + /// + /// + public const double PartialSessionShare = 0.50; + /// /// Failure rate at or above which a test is broken rather than flaky (0.90). /// diff --git a/src/Xping.Cli/Report/Providers/VanishedProvider.cs b/src/Xping.Cli/Report/Providers/VanishedProvider.cs index 8d27ac1..869f60c 100644 --- a/src/Xping.Cli/Report/Providers/VanishedProvider.cs +++ b/src/Xping.Cli/Report/Providers/VanishedProvider.cs @@ -6,6 +6,7 @@ using Xping.Cli.Report.Indexes; using Xping.Cli.Report.Model; using Xping.Cli.Report.Scoring; +using Xping.Cli.Report.Windowing; using Xping.Sdk.Core.Models; namespace Xping.Cli.Report.Providers; @@ -16,6 +17,13 @@ namespace Xping.Cli.Report.Providers; /// Sessions in the baseline slice the test appeared in. /// Sessions in the baseline slice. /// Sessions in the current slice it is absent from. +/// +/// Sessions in the window that covered only part of the suite and were counted on neither side. A +/// run under a filter never asked about this test, so its silence is not an absence. Published +/// because the two denominators above are otherwise unexplainable to a reader who asked for twenty +/// runs and is being shown a claim about four: the sentence is true of the runs that ran the suite, +/// and this is how many did not. +/// /// /// Share of the baseline sessions it appeared in — the habit itself, as a point estimate. /// @@ -40,6 +48,7 @@ internal sealed record VanishedEvidence( int BaselineSessions, int BaselineSessionCount, int CurrentSessionCount, + int PartialSessionsSetAside, double BaselineRunRate, double PValue, int ExecutionsInWindow, @@ -66,6 +75,16 @@ internal sealed record VanishedEvidence( /// table is formed: this kind only ever looks at a test already known to be absent, and there is no /// finding for one that started running. /// +/// +/// And absence is only meaningful in a session that asked. A run under a dotnet test --filter +/// did not fail to see the tests it excluded; it never looked for them, and counting its silence +/// makes every unselected test look deleted. So this kind reads the window as the sequence of +/// sessions that covered the suite — — and re-splits +/// that sequence into its own "now" and "before". A re-split rather than a filter of the window's +/// own slices, because the two differ on a store that interleaves full and filtered runs: dropping +/// the partial sessions out of a current slice of three filtered runs leaves nothing to ask about, +/// where taking the three most recent full runs still finds a test that genuinely stopped. +/// /// internal sealed class VanishedProvider : IFindingProvider { @@ -84,7 +103,10 @@ public ProviderReport Analyze(AnalysisContext context) AnalysisWindowSlices slices = AnalysisWindowSlices.From(context); // Nothing to compare against: with no baseline every test looks new, and with no current - // slice every test looks vanished. + // slice every test looks vanished. This is also where a window holding too few full runs + // stops — and it stops with an empty family rather than merely with no candidates, so the + // coordinator's Benjamini-Hochberg pass is handed a kind that asked nothing instead of a + // kind that asked three hundred questions and liked none of the answers. if (slices.BaselineCount == 0 || slices.CurrentCount == 0) return Report(candidates, tested); @@ -136,6 +158,7 @@ public ProviderReport Analyze(AnalysisContext context) appearances, slices.BaselineCount, slices.CurrentCount, + slices.PartialSessionsSetAside, FindingOrder.Round((double)appearances / slices.BaselineCount), FindingOrder.RoundProbability(pValue), executions.Count, @@ -181,17 +204,27 @@ private static ProviderReport Report(IReadOnlyList candidates, } /// -/// The fingerprints present on each side of a window's split. +/// The fingerprints present on each side of a split, taken over the runs that covered the suite. /// +/// +/// Not the window's own and +/// . Those are every session in order, and a session that +/// ran a tenth of the suite belongs in neither side of a question about absence — it did not fail to +/// see the tests it excluded, it never looked. So the sessions that covered the suite are taken in +/// order and split again, on , which is the same rule +/// the window itself narrows by. +/// /// Fingerprints seen anywhere in the current slice. /// Baseline sessions each fingerprint appeared in. /// Sessions in the current slice. /// Sessions in the baseline slice. +/// Sessions left out of both for covering part of the suite. internal sealed record AnalysisWindowSlices( IReadOnlySet Current, IReadOnlyDictionary BaselineAppearances, int CurrentCount, - int BaselineCount) + int BaselineCount, + int PartialSessionsSetAside) { /// /// Derives the split for a window. @@ -200,17 +233,29 @@ internal sealed record AnalysisWindowSlices( /// The fingerprints on each side. public static AnalysisWindowSlices From(AnalysisContext context) { + // Newest first, because AnalysisContext builds its views in window order and the window is + // ordered newest first. Dropping the partial ones preserves that, so the head of what is + // left is still the most recent thing that ran the suite. + var covering = new List(context.SessionViews.Count); + foreach (SessionView view in context.SessionViews) + { + if (!view.IsPartial) + covering.Add(view.Session); + } + + int sliceSize = AnalysisWindow.SliceSizeFor(covering.Count); + var current = new HashSet(StringComparer.Ordinal); - foreach (TestSession session in context.Window.CurrentSlice) + for (int position = 0; position < sliceSize; position++) { - foreach (string fingerprint in TestIndex.FingerprintsIn(session)) + foreach (string fingerprint in TestIndex.FingerprintsIn(covering[position])) current.Add(fingerprint); } var baseline = new Dictionary(StringComparer.Ordinal); - foreach (TestSession session in context.Window.BaselineSlice) + for (int position = sliceSize; position < covering.Count; position++) { - foreach (string fingerprint in TestIndex.FingerprintsIn(session)) + foreach (string fingerprint in TestIndex.FingerprintsIn(covering[position])) { baseline.TryGetValue(fingerprint, out int seen); baseline[fingerprint] = seen + 1; @@ -220,7 +265,8 @@ public static AnalysisWindowSlices From(AnalysisContext context) return new AnalysisWindowSlices( current, baseline, - context.Window.CurrentSlice.Count, - context.Window.BaselineSlice.Count); + sliceSize, + covering.Count - sliceSize, + context.SessionViews.Count - covering.Count); } } diff --git a/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs b/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs index c583773..c617b3a 100644 --- a/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs +++ b/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs @@ -142,6 +142,14 @@ private void WriteCaveats(StringBuilder builder, ReportEnvelope envelope, string $"{summary.EnvironmentalSessions} {RunWord(summary.EnvironmentalSessions)} " + "discounted as environmental"); + // Worded as what the window holds, not as what was done about it. Only the kinds that read + // absence set these aside; every other kind still counts them in full, and "discounted" + // beside the environmental line would claim a symmetry that does not exist. + if (summary.PartialSessions > 0) + caveats.Add( + $"{summary.PartialSessions} {RunWord(summary.PartialSessions)} " + + "covered part of the suite"); + if (summary.FailedProviders.Count > 0) caveats.Add($"metrics unavailable: {string.Join(", ", summary.FailedProviders)}"); diff --git a/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs b/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs index 02a281d..e2748b4 100644 --- a/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs +++ b/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs @@ -89,13 +89,7 @@ public static AnalysisWindow Create( WindowResolution resolution, string? argument) { - // In a small window three sessions would be most of the history, leaving a baseline too thin - // to compare against. One session is a worse "now" but leaves a usable "before". - int sliceSize = sessions.Count < LocalAnalysisConstants.SmallWindowSessionCount - ? 1 - : LocalAnalysisConstants.CurrentSliceSize; - - sliceSize = Math.Min(sliceSize, sessions.Count); + int sliceSize = SliceSizeFor(sessions.Count); return new AnalysisWindow( sessions, @@ -106,4 +100,29 @@ public static AnalysisWindow Create( [.. sessions.Take(sliceSize)], [.. sessions.Skip(sliceSize)]); } + + /// + /// Returns how many of a run of sessions form the "now" side of a delta. + /// + /// Sessions available to split. + /// The size of the current slice, never larger than what there is. + /// + /// In a small window three sessions would be most of the history, leaving a baseline too thin to + /// compare against. One session is a worse "now" but leaves a usable "before". + /// + /// Shared rather than inlined at the split, because a kind that re-slices over a subset of the + /// window has to narrow on the same rule or the two drift — one would call three sessions "now" + /// on a history the other had already decided was too short to have one. The only such kind is + /// , which re-slices over the sessions that covered the + /// suite. + /// + /// + public static int SliceSizeFor(int sessionCount) + { + int sliceSize = sessionCount < LocalAnalysisConstants.SmallWindowSessionCount + ? 1 + : LocalAnalysisConstants.CurrentSliceSize; + + return Math.Min(sliceSize, sessionCount); + } } diff --git a/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs b/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs index df56bcd..50e55ec 100644 --- a/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs +++ b/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs @@ -328,7 +328,7 @@ public void JsonEmitsTheVersionedEnvelope() using JsonDocument doc = JsonDocument.Parse(output); JsonElement root = doc.RootElement; - Assert.Equal("1.12", root.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", root.GetProperty("schemaVersion").GetString()); Assert.Equal(6, root.GetProperty("window").GetProperty("sessionCount").GetInt32()); Assert.Equal("default", root.GetProperty("window").GetProperty("resolution").GetString()); Assert.Equal(1, root.GetProperty("summary").GetProperty("tests").GetInt32()); @@ -344,7 +344,7 @@ public void JsonIsStillTheEnvelopeWhenSelectedByTheLegacyFlag() Assert.Equal(0, code); using JsonDocument doc = JsonDocument.Parse(output); - Assert.Equal("1.12", doc.RootElement.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", doc.RootElement.GetProperty("schemaVersion").GetString()); } [Fact] diff --git a/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs b/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs index 14bdc12..6dcf702 100644 --- a/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs +++ b/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs @@ -7,6 +7,7 @@ using System.Text; using System.Text.Json; using Xping.Cli.Commands; +using Xping.Cli.Report; using Xping.Cli.Report.Model; using Xping.Sdk.Core.Models; using Xping.Sdk.Core.Models.Executions; @@ -51,13 +52,34 @@ public void Dispose() /// /// Writes sessions in which several tests stop running, so the report has findings to rank. /// + /// Tests that stop. + /// Sessions to write. + /// How many of the oldest sessions run the vanishing tests. + /// + /// + /// The suite is sized from rather than fixed, because these + /// tests want findings and the kind is squeezed from both sides at once. + /// + /// + /// Too few stable tests and the later sessions drop below half the suite, which reads as runs + /// that covered part of it — see — and + /// a kind reading absence rightly declines them. Too many and the Benjamini-Hochberg pass, whose + /// bar is times the discoveries over the + /// family, tightens past the 1/56 these eight sessions can produce. One more stable test than + /// vanishing ones sits comfortably inside both, at every count seeded here. Vanished is + /// only the most convenient finding to make several of; neither bound is what is under test. + /// + /// private static void SeedVanishing(int vanishingTests = 1, int total = 8, int presentIn = 5) { ILocalSessionStore store = LocalSessionStore.Create(); for (int i = 0; i < total; i++) { - var executions = new List { TestSessionFactory.Execution("Stable") }; + var executions = new List(); + + for (int t = 0; t <= vanishingTests; t++) + executions.Add(TestSessionFactory.Execution($"Stable{t}")); if (i < presentIn) { @@ -104,7 +126,7 @@ public void TheEnvelopeCarriesEveryDocumentedSection() JsonElement root = RunJson(); - Assert.Equal("1.12", root.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", root.GetProperty("schemaVersion").GetString()); JsonElement window = root.GetProperty("window"); foreach (string key in @@ -238,10 +260,10 @@ public void HealthyCountsTestsNoFindingNamed() JsonElement summary = RunJson().GetProperty("summary"); - // Three distinct tests in the window; two vanished. - Assert.Equal(3, summary.GetProperty("tests").GetInt32()); + // Five distinct tests in the window; two vanished. + Assert.Equal(5, summary.GetProperty("tests").GetInt32()); Assert.Equal(2, summary.GetProperty("findings").GetInt32()); - Assert.Equal(1, summary.GetProperty("healthy").GetInt32()); + Assert.Equal(3, summary.GetProperty("healthy").GetInt32()); } [Fact] @@ -287,7 +309,7 @@ public void WarningsGoToStandardErrorSoJsonStaysParsable() // Would throw if a warning had been interleaved into stdout. using JsonDocument document = JsonDocument.Parse(output); - Assert.Equal("1.12", document.RootElement.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", document.RootElement.GetProperty("schemaVersion").GetString()); } [Fact] diff --git a/tests/Xping.Cli.Tests/Report/SessionViewTests.cs b/tests/Xping.Cli.Tests/Report/SessionViewTests.cs index 67e804a..b6499c7 100644 --- a/tests/Xping.Cli.Tests/Report/SessionViewTests.cs +++ b/tests/Xping.Cli.Tests/Report/SessionViewTests.cs @@ -3,6 +3,7 @@ * License: [MIT] */ +using Xping.Cli.Report; using Xping.Cli.Report.Indexes; using Xping.Sdk.Core.Models; using Xping.Sdk.Core.Models.Executions; @@ -118,4 +119,56 @@ public void AnEmptySessionHasNoFailureRateRatherThanADivisionByZero() Assert.Equal(0, view.FailureRate); Assert.False(view.IsLikelyEnvironmental); } + + [Fact] + public void PartialityIsAPropertyOfTheWindowAndNotOfTheSession() + { + // The same run, measured twice. Six tests is the whole suite in one window and a tenth of it + // in another, and only the window can say which — which is why SessionView.For leaves the + // flag alone and AnalysisContext sets it. + TestSession small = TestSessionFactory.Session(1, [.. Names(6)]); + + AnalysisContext alone = TestSessionFactory.Context( + TestSessionFactory.Session(0, [.. Names(6)]), small); + + AnalysisContext beside = TestSessionFactory.Context( + TestSessionFactory.Session(0, [.. Names(60)]), small); + + Assert.False(ViewOf(alone, small).IsPartial); + Assert.Equal(0, alone.PartialSessionCount); + + Assert.True(ViewOf(beside, small).IsPartial); + Assert.Equal(1, beside.PartialSessionCount); + } + + [Fact] + public void ARunCoveringExactlyHalfTheSuiteIsStillARunOfIt() + { + // The threshold is a floor the session has to fall below, not one it has to clear. Half is + // the last size that still counts, so the boundary is pinned rather than left to a rounding + // argument in a review. + TestSession half = TestSessionFactory.Session(1, [.. Names(5)]); + + AnalysisContext context = TestSessionFactory.Context( + TestSessionFactory.Session(0, [.. Names(10)]), half); + + Assert.False(ViewOf(context, half).IsPartial); + } + + [Fact] + public void AWindowInWhichNothingRanHasNoSuiteToBePartOf() + { + // Every session empty makes the anchor zero, and a share of zero is a division nobody wants + // to be surprised by. + AnalysisContext context = TestSessionFactory.Context( + TestSessionFactory.Session(0, []), TestSessionFactory.Session(1, [])); + + Assert.Equal(0, context.PartialSessionCount); + } + + private static IEnumerable Names(int count) => + Enumerable.Range(0, count).Select(i => $"T{i:00}"); + + private static SessionView ViewOf(AnalysisContext context, TestSession session) => + context.SessionViewFor(session.SessionId)!; } diff --git a/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs b/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs index 47d8673..554faf9 100644 --- a/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs +++ b/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs @@ -193,7 +193,7 @@ [new ClusterMember("fp", "MyApp.Tests.A", 4)], FindingKind.Vanished => new VanishedEvidence( - 12, 17, 3, 0.706, 0.0491, 40, + 12, 17, 3, 0, 0.706, 0.0491, 40, new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc), "a3f9c2e"), _ => throw new ArgumentOutOfRangeException(nameof(kind)) @@ -1030,6 +1030,7 @@ private static ReportEnvelope Envelope( 0, 0, 0, + 0, []), findings, new TruncationDto(shown, total, "xping report --all")); diff --git a/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs b/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs index e514eda..5192840 100644 --- a/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs +++ b/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs @@ -4,6 +4,7 @@ */ using Xping.Cli.Report; +using Xping.Cli.Report.Contract; using Xping.Cli.Report.Model; using Xping.Cli.Report.Providers; using Xping.Sdk.Core.Models; @@ -291,4 +292,202 @@ public void TheProviderReachesTheReportEndToEnd() Assert.StartsWith("f_", finding.Id, StringComparison.Ordinal); Assert.Contains("--kind Vanished", finding.DrillDownCommand, StringComparison.Ordinal); } + + /// + /// Builds a suite of tests, the newest sessions running only some. + /// + /// Sessions to build. + /// How many of the newest sessions run a reduced set. + /// Tests the full runs execute. + /// Tests the reduced runs execute. + private static AnalysisContext Suite(int total, int filtered, int suiteSize, int selected) + { + string[] suite = [.. Enumerable.Range(0, suiteSize).Select(i => $"T{i:00}")]; + + var sessions = new List(); + for (int i = 0; i < total; i++) + { + // Ordinal 0 is the oldest, so the filtered runs are the last ones built. + bool reduced = i >= total - filtered; + sessions.Add(TestSessionFactory.Session(i, reduced ? suite[..selected] : suite)); + } + + return TestSessionFactory.Context([.. sessions]); + } + + [Fact] + public void AFilteredRunIsNotASessionEveryUnselectedTestVanishedFrom() + { + // The defect, exactly as reported: seventeen full runs of a suite, then three runs under a + // `dotnet test --filter` naming one test. Every other test is absent from all three, ran in + // all seventeen, and scores p = 8.8e-4 — the best any twenty-run window can do. The absence + // is real and the conclusion is false: those runs never asked about the other sixteen. + Assert.Empty(Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 1))); + } + + [Fact] + public void TheFamilyIsTheQuestionsTheFullRunsCouldAnswer() + { + // Setting the filtered runs aside does not empty the family, and should not. Seventeen full + // runs remain, every test in the suite is still asked whether it stopped, and seventeen + // askings that answered no is what the Benjamini-Hochberg pass has to be charged for — the + // multiplicity is real even though none of it became a finding. + Assert.Equal(17, Family(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 1))); + } + + [Fact] + public void AWindowWithTooFewFullRunsToSplitAsksNothingAtAll() + { + // Where the filtered runs leave a single run covering the suite there is a "now" and no + // "before", so the kind returns an empty family rather than no candidates. The difference + // matters to the coordinator: a family of seventeen would tighten the bar for a comparison + // that was never actually made. + AnalysisContext context = Suite(total: 20, filtered: 19, suiteSize: 17, selected: 1); + + Assert.Empty(Analyze(context)); + Assert.Equal(0, Family(context)); + } + + [Fact] + public void ARunThatStillCoversTheSuiteIsNotSetAside() + { + // The control for the two above. Same shape, same denominators, but the last three runs + // execute the whole suite bar the one test that genuinely stopped — so the absence stands. + IReadOnlyList candidates = + Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 16)); + + FindingCandidate candidate = Assert.Single(candidates); + var evidence = Assert.IsType(candidate.Evidence); + + Assert.Equal(0, evidence.PartialSessionsSetAside); + Assert.Equal(17, evidence.BaselineSessionCount); + Assert.Equal(3, evidence.CurrentSessionCount); + } + + [Fact] + public void TheNowIsTheMostRecentRunsThatCoveredTheSuiteAndNotTheMostRecentRuns() + { + // A re-split, not a filter of the window's own slices. Three filtered runs sit at the head + // of this window; dropping them from a current slice of three would leave nothing to ask + // about and the kind would go silent. Taking the three most recent runs that covered the + // suite instead still finds the test that stopped before them. + string[] suite = ["A", "B", "C", "D"]; + + var sessions = new List(); + for (int i = 0; i < 17; i++) + sessions.Add(TestSessionFactory.Session(i, suite)); // 0-16: the whole suite + for (int i = 17; i < 20; i++) + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C")); // 17-19: D has stopped + for (int i = 20; i < 23; i++) + sessions.Add(TestSessionFactory.Session(i, "A")); // 20-22: under a filter + + FindingCandidate candidate = Assert.Single( + Analyze(TestSessionFactory.Context([.. sessions]))); + + var evidence = Assert.IsType(candidate.Evidence); + + Assert.Equal(3, evidence.PartialSessionsSetAside); + Assert.Equal(3, evidence.CurrentSessionCount); + Assert.Equal(17, evidence.BaselineSessionCount); + Assert.Equal(17, evidence.BaselineSessions); + } + + [Fact] + public void AFilteredBaselineIsNotAHabitTheTestFailedToKeep() + { + // The other half. Interleave the filtered runs through the baseline and a test that ran in + // every run that asked for it reads as a 5-of-17 occasional visitor, which is the one shape + // the p-value gate exists to decline. Counted over the runs that covered the suite it is + // 5 of 5, and the absence carries. + // + // Eight runs cover the suite, at every second ordinal; the five oldest of them run "F" and + // the three newest do not. The twelve between them name one test and are set aside, so the + // table is 5 of 5 against 3 — one deal in fifty-six — rather than 5 of 17 against 3. + var sessions = new List(); + for (int i = 0; i < 20; i++) + { + if (i % 2 != 0 || i > 14) + sessions.Add(TestSessionFactory.Session(i, "A")); + else if (i <= 8) + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C", "D", "E", "F")); + else + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C", "D", "E")); + } + + FindingCandidate candidate = Assert.Single( + Analyze(TestSessionFactory.Context([.. sessions]))); + + Assert.Equal("fp-F", Assert.IsType(candidate.Subject).Test.TestFingerprint); + + var evidence = Assert.IsType(candidate.Evidence); + + Assert.Equal(5, evidence.BaselineSessions); + Assert.Equal(5, evidence.BaselineSessionCount); + Assert.Equal(3, evidence.CurrentSessionCount); + Assert.Equal(12, evidence.PartialSessionsSetAside); + Assert.Equal(1.0, evidence.BaselineRunRate); + } + + [Fact] + public void TheEvidenceSaysHowManyRunsCoveredOnlyPartOfTheSuite() + { + // Otherwise the denominators are unexplainable: a reader who asked for twenty-three runs is + // being shown a claim about twenty, and nothing on the finding says which twenty or why. + string[] suite = ["A", "B", "C", "D"]; + + var sessions = new List(); + for (int i = 0; i < 17; i++) + sessions.Add(TestSessionFactory.Session(i, suite)); + for (int i = 17; i < 20; i++) + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C")); + for (int i = 20; i < 23; i++) + sessions.Add(TestSessionFactory.Session(i, "A")); + + FindingCandidate candidate = Assert.Single( + Analyze(TestSessionFactory.Context([.. sessions]))); + + (string headline, IReadOnlyList metrics) = + EvidenceHeadline.For(FindingKind.Vanished, candidate.Evidence); + + // Both clauses, not just the first. "the last 3" and "the last 3 full runs" are different + // runs once anything has been set aside, and a headline is read a clause at a time. + Assert.Equal( + "ran in 17 of 17 earlier full runs, absent from the last 3 full runs", + headline); + + Assert.Contains( + metrics, + m => m.Label == "set aside" && m.Value == "3 runs that covered part of the suite"); + } + + [Fact] + public void AnOrdinaryStoreIsNotToldAboutRunsItDoesNotHave() + { + // The qualification is earned, not standing. With nothing set aside there is nothing for + // the word "full" to distinguish the runs from, and the shorter sentence is the true one. + FindingCandidate candidate = Assert.Single(Analyze(Context(total: 20, presentIn: 17))); + + (string headline, IReadOnlyList metrics) = + EvidenceHeadline.For(FindingKind.Vanished, candidate.Evidence); + + // Byte for byte the sentence this kind has always printed: with nothing set aside, "the + // last 3" can mean nothing but the last three runs. + Assert.Equal("ran in 17 of 17 earlier runs, absent from the last 3", headline); + Assert.DoesNotContain(metrics, m => m.Label == "set aside"); + } + + [Fact] + public void ADeletionOfMostOfASuiteIsNotReported() + { + // The cost of deciding this on counts, pinned rather than left to be discovered. Sixteen of + // seventeen tests removed and one kept is arithmetically indistinguishable from a filter + // selecting that one, so the report says nothing. Deliberate: the finding is capped at + // Severity.Low because a disappearance is usually something the developer just did, and the + // false positive it trades against arrives once per unselected test on every filtered run. + Assert.Empty(Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 1))); + + // A deletion that leaves most of the suite standing still reports, which is the case the + // kind is actually for. + Assert.Single(Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 16))); + } }