From 708c6ff374f58918e1fbe8e8c1935681d4ac64c9 Mon Sep 17 00:00:00 2001 From: xping-admin Date: Mon, 7 Sep 2026 12:56:24 +0200 Subject: [PATCH 1/2] fix(cli): ask only the runs that covered the suite whether a test stopped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `VanishedProvider` decided absence with one membership test — is this fingerprint in the current slice? — and had no notion of whether the sessions in that slice tried to run it. A `dotnet test --filter` run in the inner loop therefore produced one false `stopped running` per unselected test, and #191's Fisher gate did not fix that: it only helps where the filtered runs dominate the *baseline* too. The commonest shape still fired at full strength. Measured on a real store — seventeen full runs of `SampleApp.MSTest` then three `--filter` runs of one test — the report produced **twenty** `stopped running` findings, each reading "ran in 17 of 17 earlier runs" at p = 8.8e-4, which is the best any twenty-run window can do. It now produces none. Absence is only evidence in a session that asked. So each run is classified by how much of the suite it covered — its distinct tests against `PartialSessionShare = 0.50` of the largest run in the window — and the kinds that read absence set the small ones aside. The anchor is the largest run and deliberately not the median: four full runs and sixteen filtered ones, which is what ten minutes of an inner loop produces, has a median of one test and every run in it measures as typical against that. A report is scoped to exactly one assembly, so the runs being compared are always runs of the same suite. `SessionView.Tests`, not `QuickStatistics.distinctTests` as the issue proposes. `SessionAssemblies.Project` nulls that field for any multi-assembly session, so it is absent exactly on solution-wide runs; the count the CLI already derives from executions is the same number and needs no plumbing. A re-split rather than a filter of the window's own slices. The two differ on a store that interleaves full and filtered runs: dropping partial sessions out of a current slice of three filtered runs leaves nothing to ask about, where taking the three most recent full runs still finds a test that genuinely stopped. Both the "now" and the "before" are re-derived over the covering runs, which also fixes the baseline half — a test present in every run that asked for it no longer reads as a 5-of-17 occasional visitor. `AnalysisWindow.SliceSizeFor` is extracted so the re-split narrows on the same rule the window does and the two cannot drift. The trade, recorded deliberately rather than left to be discovered. A count cannot separate a filter from a deletion and no threshold makes it able to: nine tests missing from seventeen is the same table whether they were excluded or removed. The line is at a half, and it is biased towards silence — the kind is capped at `Severity.Low` because a disappearance is usually something the developer did on purpose a minute ago, whereas the false positive arrives once per unselected test on every filtered run for as long as it stays in the window. So a deletion removing more than half a suite is not reported, and a filter selecting more than half still produces false absences. Both are in the command reference and in known limitations. Not suppressed silently. `VanishedEvidence` publishes `partialSessionsSetAside`, the headline says "full runs" and a `set aside` metric gives the number where any were, and the summary carries `partialSessions` with a caveat line worded as an observation rather than a discount — only this kind sets them aside, because a filtered run's *outcomes* are as true as any other run's and it is only its silences that mean nothing. The schema moves to 1.13. Verified end to end: on a store of full runs only the envelope is identical apart from the schema version and the new field. Item 3 of the issue — populating `TestSession.TotalTestsExpected` — is deliberately not done here, and cannot be done as written. MSTest exposes no test count to any hook the adapter occupies; xUnit needs a `CreateDiscoverer` override and NUnit's `ITest.TestCaseCount` silently becomes post-filter under the adapter's `PreFilter` setting. And the field is modelled host-wide, so `SessionAssemblies.Project` nulls it before the CLI sees a session. A follow-up will file it as a per-assembly discovered count. Refs #140 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TMQcZLBGFTd8wz3mcfn9vc --- docs/cli/command-reference.md | 16 +- docs/known-limitations.md | 39 ++++ src/Xping.Cli/Report/AnalysisContext.cs | 30 +++ .../Report/Contract/EnvelopeBuilder.cs | 1 + .../Report/Contract/EvidenceHeadline.cs | 37 +++- .../Report/Contract/ReportEnvelope.cs | 19 +- src/Xping.Cli/Report/Indexes/SessionView.cs | 29 +++ .../Report/LocalAnalysisConstants.cs | 37 ++++ .../Report/Providers/VanishedProvider.cs | 64 +++++- .../Report/Rendering/TextReportRenderer.cs | 8 + .../Report/Windowing/AnalysisWindow.cs | 33 ++- .../Commands/CliSurfaceTests.cs | 4 +- .../Report/ReportEnvelopeTests.cs | 37 +++- .../Report/SessionViewTests.cs | 53 +++++ .../Report/ShareableOutputTests.cs | 3 +- .../Report/VanishedProviderTests.cs | 193 ++++++++++++++++++ 16 files changed, 562 insertions(+), 41 deletions(-) diff --git a/docs/cli/command-reference.md b/docs/cli/command-reference.md index 0c39c4f2..b25ac5a4 100644 --- a/docs/cli/command-reference.md +++ b/docs/cli/command-reference.md @@ -238,6 +238,17 @@ it eases towards 0.632 as the history lengthens. On a window shorter than eight slice narrows to a single run and one run's absence never reaches the bar, so the kind is silent there — see [known limitations](../known-limitations.md). +It also asks only the runs that were in a position to answer. A run under a `dotnet test --filter` +did not fail to see the tests it excluded; it never looked for them, and counting its silence makes +every unselected test look deleted. A run whose distinct tests are under half those of the largest +run in the window is therefore set aside, the rest are re-split into their own earlier and current +runs, and the finding's denominators count only those — the sentence says `full runs` and a +`set aside` metric gives the number left out. The summary line reports how many runs covered part of +the suite. Only this kind sets them aside; every other kind still counts them in full, because a +filtered run's outcomes are as true as any other run's. The cost is that a deletion removing more +than half a suite is indistinguishable from a filter and is not reported — see +[known limitations](../known-limitations.md). + ### Finding ids The `f_…` on each finding is a short, stable identity for that finding — a hash of what the @@ -328,7 +339,7 @@ Every finding carries a `headline` — the same sentence the rendered report pri ```json { - "schemaVersion": "1.12", + "schemaVersion": "1.13", "window": { "sessionCount": 20, "resolution": "default", "currentSliceSize": 3 }, "context": { "sha": "a3f9c2e", "branch": "main", "assembly": "Checkout.Tests" }, "summary": { @@ -337,7 +348,8 @@ Every finding carries a `headline` — the same sentence the rendered report pri "counts": { "high": 1, "medium": 2, "low": 0 }, "healthy": 409, "excludedLowEvidence": 41, - "excludedNotSignificant": 6 + "excludedNotSignificant": 6, + "partialSessions": 0 }, "findings": [ { diff --git a/docs/known-limitations.md b/docs/known-limitations.md index e76c2c22..4e472c5e 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -319,6 +319,44 @@ anything stopped — but it does mean a fresh store says nothing about deleted t eight runs in it. The run rate and the p-value are published with every finding that does clear the bar. +--- + +### `Vanished` Cannot Tell A Filtered Run From A Deletion, So It Trusts Neither Below Half A Suite + +**Impact**: a run that covered less than half the suite is counted on neither side of an absence, so +a deletion that removed more than half a suite is never reported. A `--filter` selecting *more* than +half the suite still produces one false `stopped running` per unselected test. + +**Reason**: a run under a `dotnet test --filter` did not fail to see the tests it excluded — it never +looked for them. Counting its silence makes every unselected test look deleted, which is what an +ordinary inner loop produces in about ten minutes: a handful of full runs, then a stream of filtered +ones, then a report claiming most of the suite has stopped running. Every statement in it is true of +the data and false about the world. + +So the report classifies each run by how much of the suite it covered — its distinct tests against +the largest run in the window, which is the best evidence the window holds of how big the suite is, +and the only anchor a store of mostly filtered runs does not corrupt. The median does not work: four +full runs and sixteen filtered ones has a median of one test, and every run in it measures as +typical. A report is scoped to exactly one assembly, so the runs being compared are always runs of +the same suite. Runs covering less than half are set aside, the remaining runs are re-split into +their own "now" and "before", and the counts on the finding are of those runs alone — `full runs` in +the sentence, with a `set aside` metric saying how many were left out. + +**The trade, stated rather than discovered**: a count cannot separate the two cases, and no threshold +makes it able to. Nine tests missing from a suite of seventeen is the same table whether they were +excluded or removed. The line is placed at a half — the point at which a run stopped being a run of +the suite and became a run of part of it — and it is deliberately biased towards silence: `Vanished` +is capped at `Severity.Low` because a disappearance is usually something the developer did on purpose +a minute ago, so a missed one costs little, whereas the false positive arrives once per unselected +test on every filtered run for as long as it stays in the window. + +**Related**: setting runs aside shortens the history this kind measures against, so a store whose +runs are mostly filtered can fall below the eight runs the section above requires and report nothing +at all. Only the kinds that read absence set these runs aside — every other kind still counts them in +full, because a filtered run's *outcomes* are as true as any other run's and it is only its silences +that mean nothing. The summary line says how many runs covered part of the suite, so the distinction +is visible rather than inferred. + ### `RetryExhausted` Is Observed, And The Declared Retry Limit Is Not Interpreted **Impact**: a test whose retry attribute allows three retries but which only ever recorded two @@ -491,3 +529,4 @@ When reporting, please include: | 1.7.0 | Documented what `TimeSensitive` now charges for searching three axes, and what that costs | | 1.8.0 | Documented what `ParallelSensitive` now measures, and the duration confound it cannot correct | | 1.9.0 | Documented the run rate `Vanished` now requires, and the window size below which it is silent | +| 1.10.0 | Documented how `Vanished` treats a run that covered part of the suite, and what that trade costs | diff --git a/src/Xping.Cli/Report/AnalysisContext.cs b/src/Xping.Cli/Report/AnalysisContext.cs index c7b5672b..5a79569a 100644 --- a/src/Xping.Cli/Report/AnalysisContext.cs +++ b/src/Xping.Cli/Report/AnalysisContext.cs @@ -44,9 +44,28 @@ public AnalysisContext(AnalysisWindow window, RevisionContext? revision) for (int position = 0; position < window.Sessions.Count; position++) views.Add(SessionView.For(window.Sessions[position], position)); + // Partiality is a comparison and not a measurement, so it needs the whole window before any + // one session can be classified. The anchor is the largest run in it: the best evidence the + // window holds of how big the suite is, and the only one a store of mostly filtered runs + // does not corrupt — see LocalAnalysisConstants.PartialSessionShare. + int suiteSize = 0; + foreach (SessionView view in views) + suiteSize = Math.Max(suiteSize, view.Tests); + + if (suiteSize > 0) + { + double floor = LocalAnalysisConstants.PartialSessionShare * suiteSize; + for (int position = 0; position < views.Count; position++) + { + if (views[position].Tests < floor) + views[position] = views[position] with { IsPartial = true }; + } + } + SessionViews = views; _viewsBySession = views.ToDictionary(v => v.Session.SessionId); EnvironmentalSessionCount = views.Count(v => v.IsLikelyEnvironmental); + PartialSessionCount = views.Count(v => v.IsPartial); } /// Gets the sessions under analysis and the boundaries that produced them. @@ -71,6 +90,17 @@ public AnalysisContext(AnalysisWindow window, RevisionContext? revision) /// public int EnvironmentalSessionCount { get; } + /// + /// Gets how many analysed sessions covered only part of the suite. + /// + /// + /// Reported in the summary as an observation rather than a discount. Only the kinds that read + /// absence set these sessions aside — every other kind reads outcomes of executions that + /// happened, and a filtered run's outcomes are as true as any other's — so the line says what + /// the window contains and does not claim the numbers were adjusted. + /// + public int PartialSessionCount { get; } + /// /// Gets the health of one analysed session. /// diff --git a/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs b/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs index 7e3fa7ef..c2b13786 100644 --- a/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs +++ b/src/Xping.Cli/Report/Contract/EnvelopeBuilder.cs @@ -84,6 +84,7 @@ public static ReportEnvelope Build( result.ExcludedNotSignificant, context.EnvironmentalSessionCount, + context.PartialSessionCount, incompleteSessions, unreadableSessions, result.FailedProviders), diff --git a/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs b/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs index 329f67fe..c9c88ceb 100644 --- a/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs +++ b/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs @@ -536,27 +536,50 @@ private static (string, IReadOnlyList) TimeSensitive(TimeSensitiveEvi /// The evidence. /// The headline and its metrics. /// + /// /// The headline carries the denominator and not the p-value, because here the denominator is the /// discriminating figure: "3 of 17" and "17 of 17" are visibly different claims to a reader /// skimming the fence, where the two arms of a split are not. The p-value is a metric, for the /// reader who opens the finding to check how much belief the sentence earned. + /// + /// + /// Both denominators count only the runs that covered the suite, so where any were set aside the + /// sentence says "full runs" and a metric says how many were left out. Qualified rather than + /// worded that way always, because in a store with no filtered runs in it there is nothing for + /// the word to distinguish the runs from, and the shorter sentence is the true one. The wording + /// cannot destabilise the finding's id, which hashes the kind and the subject and nothing else + /// — see . + /// /// - private static (string, IReadOnlyList) Vanished(VanishedEvidence e) => - ( - $"ran in {e.BaselineSessions} of {e.BaselineSessionCount} earlier runs, " + - $"absent from the last {e.CurrentSessionCount}", + private static (string, IReadOnlyList) Vanished(VanishedEvidence e) + { + bool anySetAside = e.PartialSessionsSetAside > 0; + string runs = anySetAside ? "full runs" : "runs"; + + List metrics = [ new( "ran in", - $"{e.BaselineSessions} of {e.BaselineSessionCount} earlier runs " + + $"{e.BaselineSessions} of {e.BaselineSessionCount} earlier {runs} " + $"({Percent(e.BaselineRunRate)})"), - new("absent from", $"the last {e.CurrentSessionCount} runs"), + new("absent from", $"the last {e.CurrentSessionCount} {runs}"), new("executions", e.ExecutionsInWindow.ToString(CultureInfo.InvariantCulture)), // One-sided, and legitimately so: the kind only ever forms a table for a test already // absent, so the direction was fixed before the counts were. new("significance", $"p {Probability(e.PValue)} one-sided") - ]); + ]; + + // Only where there were any. A "0 set aside" line on every finding in every ordinary store + // would be noise standing in for the absence of a caveat. + if (anySetAside) + metrics.Add(new("set aside", $"{Runs(e.PartialSessionsSetAside)} that covered part of the suite")); + + return ( + $"ran in {e.BaselineSessions} of {e.BaselineSessionCount} earlier {runs}, " + + $"absent from the last {e.CurrentSessionCount}", + metrics); + } private static string Times(int count) => count == 1 ? "once" : $"{count.ToString(CultureInfo.InvariantCulture)} times"; diff --git a/src/Xping.Cli/Report/Contract/ReportEnvelope.cs b/src/Xping.Cli/Report/Contract/ReportEnvelope.cs index 5999aafb..c0d39d10 100644 --- a/src/Xping.Cli/Report/Contract/ReportEnvelope.cs +++ b/src/Xping.Cli/Report/Contract/ReportEnvelope.cs @@ -39,14 +39,12 @@ internal sealed record ReportEnvelope( /// Moves whenever anything a consumer reads changes shape, and the per-kind evidence payloads /// are part of that even though this document describes them as opaque: a script that reached /// into evidence for a field this build no longer emits is reading a contract, and - /// leaving the number still would tell it nothing had moved. 1.12 is where every evidence - /// record renamed the fields a rate is taken over — executionsConsidered and - /// sessionsConsidered where discounting applied, executionsInWindow and - /// sessions where it did not — split discountedExecutions into its environmental - /// and clustered halves, and where the finding gained population, which says which of - /// those the numbers beside it describe. + /// leaving the number still would tell it nothing had moved. 1.13 is where the summary gained + /// partialSessions and VanishedEvidence gained partialSessionsSetAside — + /// the runs that covered only part of the suite, which an absence is counted on neither side + /// of. /// - public const string CurrentSchemaVersion = "1.12"; + public const string CurrentSchemaVersion = "1.13"; } /// @@ -95,6 +93,12 @@ internal sealed record ContextDto(string? Sha, string? Branch, string? Assembly) /// reader given only an empty block cannot tell that from a suite with nothing to report. /// /// Sessions discounted as environment failures. +/// +/// Sessions that covered only part of the suite — a dotnet test --filter run, or anything +/// else that ran a fraction of the tests the window's largest run did. An observation and not a +/// discount: only the kinds that read absence set such a session aside, because a filtered run's +/// outcomes are as true as any other run's and it is only its silences that mean nothing. +/// /// Sessions found but not finalised. /// Session files that could not be read. /// Metrics that threw and produced nothing. @@ -106,6 +110,7 @@ internal sealed record SummaryDto( int ExcludedLowEvidence, int ExcludedNotSignificant, int EnvironmentalSessions, + int PartialSessions, int IncompleteSessions, int UnreadableSessions, IReadOnlyList FailedProviders); diff --git a/src/Xping.Cli/Report/Indexes/SessionView.cs b/src/Xping.Cli/Report/Indexes/SessionView.cs index 952cb59b..d8d9a351 100644 --- a/src/Xping.Cli/Report/Indexes/SessionView.cs +++ b/src/Xping.Cli/Report/Indexes/SessionView.cs @@ -21,6 +21,11 @@ namespace Xping.Cli.Report.Indexes; /// It is a classification against two thresholds, not a claim about what went wrong. The report says /// a session looks environmental; it never says why, and it never says so about a test. /// +/// +/// is the other flag, and it answers a different question: not whether the +/// session's outcomes can be trusted, but whether its silences can. It is window-relative, so +/// cannot decide it — see . +/// /// /// The session itself. /// Its position in the window; 0 is the newest. @@ -38,6 +43,30 @@ internal sealed record SessionView( double FailureRate, bool IsLikelyEnvironmental) { + /// + /// Gets a value indicating whether the session covered only part of the suite. + /// + /// + /// + /// A run under a dotnet test --filter is not a run in which the tests it excluded failed + /// to appear — it is a run that never asked about them. A kind reading absence has to set such a + /// session aside, or every unselected test looks deleted. + /// + /// + /// Not a positional member, because it cannot be measured from one session: it is + /// against of the + /// largest run in the window, and the window is what does not have. + /// sets it once the whole window is measured, which is also the + /// only place it means anything. + /// + /// + /// A classification against a threshold and not a claim about what happened, in the same way + /// is. The report says a session covered part of the suite; + /// it never says a filter was the reason, because a deletion produces the same count. + /// + /// + public bool IsPartial { get; init; } + /// /// Measures one session. /// diff --git a/src/Xping.Cli/Report/LocalAnalysisConstants.cs b/src/Xping.Cli/Report/LocalAnalysisConstants.cs index 964d2ed2..a2c07b8c 100644 --- a/src/Xping.Cli/Report/LocalAnalysisConstants.cs +++ b/src/Xping.Cli/Report/LocalAnalysisConstants.cs @@ -77,6 +77,43 @@ internal static class LocalAnalysisConstants /// public const int SmallWindowSessionCount = 8; + /// + /// Share of the window's largest run a session must cover to count as a run of the suite (0.50). + /// + /// + /// + /// A dotnet test --filter run is an ordinary thing to find in a local store, and a + /// session that ran one test of seventeen says nothing whatever about the other sixteen. Absence + /// is only evidence where the run was in a position to produce a presence, so a session covering + /// a small part of the suite is set aside by the kinds that read absence rather than outcome — + /// today that is alone. + /// + /// + /// The anchor is the largest session in the window and deliberately not its median. A store of + /// four full runs and sixteen filtered ones — which is what ten minutes of an inner loop + /// produces — has a median of one test, and every session in it would measure as typical + /// against that. The largest run is the best evidence the window holds of how big the suite + /// actually is, and a report is scoped to exactly one assembly, so the sessions being compared + /// are always runs of the same suite. + /// + /// + /// A count cannot tell a filtered run from a deletion, and no threshold makes it able to: a run + /// of eight tests where the suite has seventeen is the same table whether nine tests were + /// excluded or removed. So the line is a trade rather than a discovery, and it is placed at a + /// half — the point at which a run stopped being a run of the suite and became a run of part of + /// it. What that costs, in both directions: a filter selecting more than half the suite still + /// produces false absences, and a deletion of more than half a suite is never reported at all. + /// + /// + /// Biased towards silence on purpose. is capped at + /// because a disappearance is usually something the developer + /// did on purpose thirty seconds ago, so a missed one costs almost nothing; whereas the false + /// positive arrives one per unselected test, on every filtered run, for as long as the run stays + /// in the window. + /// + /// + public const double PartialSessionShare = 0.50; + /// /// Failure rate at or above which a test is broken rather than flaky (0.90). /// diff --git a/src/Xping.Cli/Report/Providers/VanishedProvider.cs b/src/Xping.Cli/Report/Providers/VanishedProvider.cs index 8d27ac19..869f60ce 100644 --- a/src/Xping.Cli/Report/Providers/VanishedProvider.cs +++ b/src/Xping.Cli/Report/Providers/VanishedProvider.cs @@ -6,6 +6,7 @@ using Xping.Cli.Report.Indexes; using Xping.Cli.Report.Model; using Xping.Cli.Report.Scoring; +using Xping.Cli.Report.Windowing; using Xping.Sdk.Core.Models; namespace Xping.Cli.Report.Providers; @@ -16,6 +17,13 @@ namespace Xping.Cli.Report.Providers; /// Sessions in the baseline slice the test appeared in. /// Sessions in the baseline slice. /// Sessions in the current slice it is absent from. +/// +/// Sessions in the window that covered only part of the suite and were counted on neither side. A +/// run under a filter never asked about this test, so its silence is not an absence. Published +/// because the two denominators above are otherwise unexplainable to a reader who asked for twenty +/// runs and is being shown a claim about four: the sentence is true of the runs that ran the suite, +/// and this is how many did not. +/// /// /// Share of the baseline sessions it appeared in — the habit itself, as a point estimate. /// @@ -40,6 +48,7 @@ internal sealed record VanishedEvidence( int BaselineSessions, int BaselineSessionCount, int CurrentSessionCount, + int PartialSessionsSetAside, double BaselineRunRate, double PValue, int ExecutionsInWindow, @@ -66,6 +75,16 @@ internal sealed record VanishedEvidence( /// table is formed: this kind only ever looks at a test already known to be absent, and there is no /// finding for one that started running. /// +/// +/// And absence is only meaningful in a session that asked. A run under a dotnet test --filter +/// did not fail to see the tests it excluded; it never looked for them, and counting its silence +/// makes every unselected test look deleted. So this kind reads the window as the sequence of +/// sessions that covered the suite — — and re-splits +/// that sequence into its own "now" and "before". A re-split rather than a filter of the window's +/// own slices, because the two differ on a store that interleaves full and filtered runs: dropping +/// the partial sessions out of a current slice of three filtered runs leaves nothing to ask about, +/// where taking the three most recent full runs still finds a test that genuinely stopped. +/// /// internal sealed class VanishedProvider : IFindingProvider { @@ -84,7 +103,10 @@ public ProviderReport Analyze(AnalysisContext context) AnalysisWindowSlices slices = AnalysisWindowSlices.From(context); // Nothing to compare against: with no baseline every test looks new, and with no current - // slice every test looks vanished. + // slice every test looks vanished. This is also where a window holding too few full runs + // stops — and it stops with an empty family rather than merely with no candidates, so the + // coordinator's Benjamini-Hochberg pass is handed a kind that asked nothing instead of a + // kind that asked three hundred questions and liked none of the answers. if (slices.BaselineCount == 0 || slices.CurrentCount == 0) return Report(candidates, tested); @@ -136,6 +158,7 @@ public ProviderReport Analyze(AnalysisContext context) appearances, slices.BaselineCount, slices.CurrentCount, + slices.PartialSessionsSetAside, FindingOrder.Round((double)appearances / slices.BaselineCount), FindingOrder.RoundProbability(pValue), executions.Count, @@ -181,17 +204,27 @@ private static ProviderReport Report(IReadOnlyList candidates, } /// -/// The fingerprints present on each side of a window's split. +/// The fingerprints present on each side of a split, taken over the runs that covered the suite. /// +/// +/// Not the window's own and +/// . Those are every session in order, and a session that +/// ran a tenth of the suite belongs in neither side of a question about absence — it did not fail to +/// see the tests it excluded, it never looked. So the sessions that covered the suite are taken in +/// order and split again, on , which is the same rule +/// the window itself narrows by. +/// /// Fingerprints seen anywhere in the current slice. /// Baseline sessions each fingerprint appeared in. /// Sessions in the current slice. /// Sessions in the baseline slice. +/// Sessions left out of both for covering part of the suite. internal sealed record AnalysisWindowSlices( IReadOnlySet Current, IReadOnlyDictionary BaselineAppearances, int CurrentCount, - int BaselineCount) + int BaselineCount, + int PartialSessionsSetAside) { /// /// Derives the split for a window. @@ -200,17 +233,29 @@ internal sealed record AnalysisWindowSlices( /// The fingerprints on each side. public static AnalysisWindowSlices From(AnalysisContext context) { + // Newest first, because AnalysisContext builds its views in window order and the window is + // ordered newest first. Dropping the partial ones preserves that, so the head of what is + // left is still the most recent thing that ran the suite. + var covering = new List(context.SessionViews.Count); + foreach (SessionView view in context.SessionViews) + { + if (!view.IsPartial) + covering.Add(view.Session); + } + + int sliceSize = AnalysisWindow.SliceSizeFor(covering.Count); + var current = new HashSet(StringComparer.Ordinal); - foreach (TestSession session in context.Window.CurrentSlice) + for (int position = 0; position < sliceSize; position++) { - foreach (string fingerprint in TestIndex.FingerprintsIn(session)) + foreach (string fingerprint in TestIndex.FingerprintsIn(covering[position])) current.Add(fingerprint); } var baseline = new Dictionary(StringComparer.Ordinal); - foreach (TestSession session in context.Window.BaselineSlice) + for (int position = sliceSize; position < covering.Count; position++) { - foreach (string fingerprint in TestIndex.FingerprintsIn(session)) + foreach (string fingerprint in TestIndex.FingerprintsIn(covering[position])) { baseline.TryGetValue(fingerprint, out int seen); baseline[fingerprint] = seen + 1; @@ -220,7 +265,8 @@ public static AnalysisWindowSlices From(AnalysisContext context) return new AnalysisWindowSlices( current, baseline, - context.Window.CurrentSlice.Count, - context.Window.BaselineSlice.Count); + sliceSize, + covering.Count - sliceSize, + context.SessionViews.Count - covering.Count); } } diff --git a/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs b/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs index c5837737..c617b3af 100644 --- a/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs +++ b/src/Xping.Cli/Report/Rendering/TextReportRenderer.cs @@ -142,6 +142,14 @@ private void WriteCaveats(StringBuilder builder, ReportEnvelope envelope, string $"{summary.EnvironmentalSessions} {RunWord(summary.EnvironmentalSessions)} " + "discounted as environmental"); + // Worded as what the window holds, not as what was done about it. Only the kinds that read + // absence set these aside; every other kind still counts them in full, and "discounted" + // beside the environmental line would claim a symmetry that does not exist. + if (summary.PartialSessions > 0) + caveats.Add( + $"{summary.PartialSessions} {RunWord(summary.PartialSessions)} " + + "covered part of the suite"); + if (summary.FailedProviders.Count > 0) caveats.Add($"metrics unavailable: {string.Join(", ", summary.FailedProviders)}"); diff --git a/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs b/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs index 02a281de..e2748b4d 100644 --- a/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs +++ b/src/Xping.Cli/Report/Windowing/AnalysisWindow.cs @@ -89,13 +89,7 @@ public static AnalysisWindow Create( WindowResolution resolution, string? argument) { - // In a small window three sessions would be most of the history, leaving a baseline too thin - // to compare against. One session is a worse "now" but leaves a usable "before". - int sliceSize = sessions.Count < LocalAnalysisConstants.SmallWindowSessionCount - ? 1 - : LocalAnalysisConstants.CurrentSliceSize; - - sliceSize = Math.Min(sliceSize, sessions.Count); + int sliceSize = SliceSizeFor(sessions.Count); return new AnalysisWindow( sessions, @@ -106,4 +100,29 @@ public static AnalysisWindow Create( [.. sessions.Take(sliceSize)], [.. sessions.Skip(sliceSize)]); } + + /// + /// Returns how many of a run of sessions form the "now" side of a delta. + /// + /// Sessions available to split. + /// The size of the current slice, never larger than what there is. + /// + /// In a small window three sessions would be most of the history, leaving a baseline too thin to + /// compare against. One session is a worse "now" but leaves a usable "before". + /// + /// Shared rather than inlined at the split, because a kind that re-slices over a subset of the + /// window has to narrow on the same rule or the two drift — one would call three sessions "now" + /// on a history the other had already decided was too short to have one. The only such kind is + /// , which re-slices over the sessions that covered the + /// suite. + /// + /// + public static int SliceSizeFor(int sessionCount) + { + int sliceSize = sessionCount < LocalAnalysisConstants.SmallWindowSessionCount + ? 1 + : LocalAnalysisConstants.CurrentSliceSize; + + return Math.Min(sliceSize, sessionCount); + } } diff --git a/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs b/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs index df56bcd1..50e55ecc 100644 --- a/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs +++ b/tests/Xping.Cli.Tests/Commands/CliSurfaceTests.cs @@ -328,7 +328,7 @@ public void JsonEmitsTheVersionedEnvelope() using JsonDocument doc = JsonDocument.Parse(output); JsonElement root = doc.RootElement; - Assert.Equal("1.12", root.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", root.GetProperty("schemaVersion").GetString()); Assert.Equal(6, root.GetProperty("window").GetProperty("sessionCount").GetInt32()); Assert.Equal("default", root.GetProperty("window").GetProperty("resolution").GetString()); Assert.Equal(1, root.GetProperty("summary").GetProperty("tests").GetInt32()); @@ -344,7 +344,7 @@ public void JsonIsStillTheEnvelopeWhenSelectedByTheLegacyFlag() Assert.Equal(0, code); using JsonDocument doc = JsonDocument.Parse(output); - Assert.Equal("1.12", doc.RootElement.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", doc.RootElement.GetProperty("schemaVersion").GetString()); } [Fact] diff --git a/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs b/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs index 14bdc127..a3b12028 100644 --- a/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs +++ b/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs @@ -7,6 +7,7 @@ using System.Text; using System.Text.Json; using Xping.Cli.Commands; +using Xping.Cli.Report; using Xping.Cli.Report.Model; using Xping.Sdk.Core.Models; using Xping.Sdk.Core.Models.Executions; @@ -51,13 +52,37 @@ public void Dispose() /// /// Writes sessions in which several tests stop running, so the report has findings to rank. /// + /// + /// Seeds a store in which the newest sessions stop running some of the tests. + /// + /// Tests that stop. + /// Sessions to write. + /// How many of the oldest sessions run the vanishing tests. + /// + /// + /// The suite is sized from rather than fixed, because these + /// tests want findings and the kind is squeezed from both sides at once. + /// + /// + /// Too few stable tests and the later sessions drop below half the suite, which reads as runs + /// that covered part of it — see — and + /// a kind reading absence rightly declines them. Too many and the Benjamini-Hochberg pass, whose + /// bar is times the discoveries over the + /// family, tightens past the 1/56 these eight sessions can produce. One more stable test than + /// vanishing ones sits comfortably inside both, at every count seeded here. Vanished is + /// only the most convenient finding to make several of; neither bound is what is under test. + /// + /// private static void SeedVanishing(int vanishingTests = 1, int total = 8, int presentIn = 5) { ILocalSessionStore store = LocalSessionStore.Create(); for (int i = 0; i < total; i++) { - var executions = new List { TestSessionFactory.Execution("Stable") }; + var executions = new List(); + + for (int t = 0; t <= vanishingTests; t++) + executions.Add(TestSessionFactory.Execution($"Stable{t}")); if (i < presentIn) { @@ -104,7 +129,7 @@ public void TheEnvelopeCarriesEveryDocumentedSection() JsonElement root = RunJson(); - Assert.Equal("1.12", root.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", root.GetProperty("schemaVersion").GetString()); JsonElement window = root.GetProperty("window"); foreach (string key in @@ -238,10 +263,10 @@ public void HealthyCountsTestsNoFindingNamed() JsonElement summary = RunJson().GetProperty("summary"); - // Three distinct tests in the window; two vanished. - Assert.Equal(3, summary.GetProperty("tests").GetInt32()); + // Five distinct tests in the window; two vanished. + Assert.Equal(5, summary.GetProperty("tests").GetInt32()); Assert.Equal(2, summary.GetProperty("findings").GetInt32()); - Assert.Equal(1, summary.GetProperty("healthy").GetInt32()); + Assert.Equal(3, summary.GetProperty("healthy").GetInt32()); } [Fact] @@ -287,7 +312,7 @@ public void WarningsGoToStandardErrorSoJsonStaysParsable() // Would throw if a warning had been interleaved into stdout. using JsonDocument document = JsonDocument.Parse(output); - Assert.Equal("1.12", document.RootElement.GetProperty("schemaVersion").GetString()); + Assert.Equal("1.13", document.RootElement.GetProperty("schemaVersion").GetString()); } [Fact] diff --git a/tests/Xping.Cli.Tests/Report/SessionViewTests.cs b/tests/Xping.Cli.Tests/Report/SessionViewTests.cs index 67e804a3..b6499c76 100644 --- a/tests/Xping.Cli.Tests/Report/SessionViewTests.cs +++ b/tests/Xping.Cli.Tests/Report/SessionViewTests.cs @@ -3,6 +3,7 @@ * License: [MIT] */ +using Xping.Cli.Report; using Xping.Cli.Report.Indexes; using Xping.Sdk.Core.Models; using Xping.Sdk.Core.Models.Executions; @@ -118,4 +119,56 @@ public void AnEmptySessionHasNoFailureRateRatherThanADivisionByZero() Assert.Equal(0, view.FailureRate); Assert.False(view.IsLikelyEnvironmental); } + + [Fact] + public void PartialityIsAPropertyOfTheWindowAndNotOfTheSession() + { + // The same run, measured twice. Six tests is the whole suite in one window and a tenth of it + // in another, and only the window can say which — which is why SessionView.For leaves the + // flag alone and AnalysisContext sets it. + TestSession small = TestSessionFactory.Session(1, [.. Names(6)]); + + AnalysisContext alone = TestSessionFactory.Context( + TestSessionFactory.Session(0, [.. Names(6)]), small); + + AnalysisContext beside = TestSessionFactory.Context( + TestSessionFactory.Session(0, [.. Names(60)]), small); + + Assert.False(ViewOf(alone, small).IsPartial); + Assert.Equal(0, alone.PartialSessionCount); + + Assert.True(ViewOf(beside, small).IsPartial); + Assert.Equal(1, beside.PartialSessionCount); + } + + [Fact] + public void ARunCoveringExactlyHalfTheSuiteIsStillARunOfIt() + { + // The threshold is a floor the session has to fall below, not one it has to clear. Half is + // the last size that still counts, so the boundary is pinned rather than left to a rounding + // argument in a review. + TestSession half = TestSessionFactory.Session(1, [.. Names(5)]); + + AnalysisContext context = TestSessionFactory.Context( + TestSessionFactory.Session(0, [.. Names(10)]), half); + + Assert.False(ViewOf(context, half).IsPartial); + } + + [Fact] + public void AWindowInWhichNothingRanHasNoSuiteToBePartOf() + { + // Every session empty makes the anchor zero, and a share of zero is a division nobody wants + // to be surprised by. + AnalysisContext context = TestSessionFactory.Context( + TestSessionFactory.Session(0, []), TestSessionFactory.Session(1, [])); + + Assert.Equal(0, context.PartialSessionCount); + } + + private static IEnumerable Names(int count) => + Enumerable.Range(0, count).Select(i => $"T{i:00}"); + + private static SessionView ViewOf(AnalysisContext context, TestSession session) => + context.SessionViewFor(session.SessionId)!; } diff --git a/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs b/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs index 47d86737..554faf95 100644 --- a/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs +++ b/tests/Xping.Cli.Tests/Report/ShareableOutputTests.cs @@ -193,7 +193,7 @@ [new ClusterMember("fp", "MyApp.Tests.A", 4)], FindingKind.Vanished => new VanishedEvidence( - 12, 17, 3, 0.706, 0.0491, 40, + 12, 17, 3, 0, 0.706, 0.0491, 40, new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc), "a3f9c2e"), _ => throw new ArgumentOutOfRangeException(nameof(kind)) @@ -1030,6 +1030,7 @@ private static ReportEnvelope Envelope( 0, 0, 0, + 0, []), findings, new TruncationDto(shown, total, "xping report --all")); diff --git a/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs b/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs index e514eda2..7a369658 100644 --- a/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs +++ b/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs @@ -4,6 +4,7 @@ */ using Xping.Cli.Report; +using Xping.Cli.Report.Contract; using Xping.Cli.Report.Model; using Xping.Cli.Report.Providers; using Xping.Sdk.Core.Models; @@ -291,4 +292,196 @@ public void TheProviderReachesTheReportEndToEnd() Assert.StartsWith("f_", finding.Id, StringComparison.Ordinal); Assert.Contains("--kind Vanished", finding.DrillDownCommand, StringComparison.Ordinal); } + + /// + /// Builds a suite of tests, the newest sessions running only some. + /// + /// Sessions to build. + /// How many of the newest sessions run a reduced set. + /// Tests the full runs execute. + /// Tests the reduced runs execute. + private static AnalysisContext Suite(int total, int filtered, int suiteSize, int selected) + { + string[] suite = [.. Enumerable.Range(0, suiteSize).Select(i => $"T{i:00}")]; + + var sessions = new List(); + for (int i = 0; i < total; i++) + { + // Ordinal 0 is the oldest, so the filtered runs are the last ones built. + bool reduced = i >= total - filtered; + sessions.Add(TestSessionFactory.Session(i, reduced ? suite[..selected] : suite)); + } + + return TestSessionFactory.Context([.. sessions]); + } + + [Fact] + public void AFilteredRunIsNotASessionEveryUnselectedTestVanishedFrom() + { + // The defect, exactly as reported: seventeen full runs of a suite, then three runs under a + // `dotnet test --filter` naming one test. Every other test is absent from all three, ran in + // all seventeen, and scores p = 8.8e-4 — the best any twenty-run window can do. The absence + // is real and the conclusion is false: those runs never asked about the other sixteen. + Assert.Empty(Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 1))); + } + + [Fact] + public void TheFamilyIsTheQuestionsTheFullRunsCouldAnswer() + { + // Setting the filtered runs aside does not empty the family, and should not. Seventeen full + // runs remain, every test in the suite is still asked whether it stopped, and seventeen + // askings that answered no is what the Benjamini-Hochberg pass has to be charged for — the + // multiplicity is real even though none of it became a finding. + Assert.Equal(17, Family(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 1))); + } + + [Fact] + public void AWindowWithTooFewFullRunsToSplitAsksNothingAtAll() + { + // Where the filtered runs leave a single run covering the suite there is a "now" and no + // "before", so the kind returns an empty family rather than no candidates. The difference + // matters to the coordinator: a family of seventeen would tighten the bar for a comparison + // that was never actually made. + AnalysisContext context = Suite(total: 20, filtered: 19, suiteSize: 17, selected: 1); + + Assert.Empty(Analyze(context)); + Assert.Equal(0, Family(context)); + } + + [Fact] + public void ARunThatStillCoversTheSuiteIsNotSetAside() + { + // The control for the two above. Same shape, same denominators, but the last three runs + // execute the whole suite bar the one test that genuinely stopped — so the absence stands. + IReadOnlyList candidates = + Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 16)); + + FindingCandidate candidate = Assert.Single(candidates); + var evidence = Assert.IsType(candidate.Evidence); + + Assert.Equal(0, evidence.PartialSessionsSetAside); + Assert.Equal(17, evidence.BaselineSessionCount); + Assert.Equal(3, evidence.CurrentSessionCount); + } + + [Fact] + public void TheNowIsTheMostRecentRunsThatCoveredTheSuiteAndNotTheMostRecentRuns() + { + // A re-split, not a filter of the window's own slices. Three filtered runs sit at the head + // of this window; dropping them from a current slice of three would leave nothing to ask + // about and the kind would go silent. Taking the three most recent runs that covered the + // suite instead still finds the test that stopped before them. + string[] suite = ["A", "B", "C", "D"]; + + var sessions = new List(); + for (int i = 0; i < 17; i++) + sessions.Add(TestSessionFactory.Session(i, suite)); // 0-16: the whole suite + for (int i = 17; i < 20; i++) + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C")); // 17-19: D has stopped + for (int i = 20; i < 23; i++) + sessions.Add(TestSessionFactory.Session(i, "A")); // 20-22: under a filter + + FindingCandidate candidate = Assert.Single( + Analyze(TestSessionFactory.Context([.. sessions]))); + + var evidence = Assert.IsType(candidate.Evidence); + + Assert.Equal(3, evidence.PartialSessionsSetAside); + Assert.Equal(3, evidence.CurrentSessionCount); + Assert.Equal(17, evidence.BaselineSessionCount); + Assert.Equal(17, evidence.BaselineSessions); + } + + [Fact] + public void AFilteredBaselineIsNotAHabitTheTestFailedToKeep() + { + // The other half. Interleave the filtered runs through the baseline and a test that ran in + // every run that asked for it reads as a 5-of-17 occasional visitor, which is the one shape + // the p-value gate exists to decline. Counted over the runs that covered the suite it is + // 5 of 5, and the absence carries. + // + // Eight runs cover the suite, at every second ordinal; the five oldest of them run "F" and + // the three newest do not. The twelve between them name one test and are set aside, so the + // table is 5 of 5 against 3 — one deal in fifty-six — rather than 5 of 17 against 3. + var sessions = new List(); + for (int i = 0; i < 20; i++) + { + if (i % 2 != 0 || i > 14) + sessions.Add(TestSessionFactory.Session(i, "A")); + else if (i <= 8) + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C", "D", "E", "F")); + else + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C", "D", "E")); + } + + FindingCandidate candidate = Assert.Single( + Analyze(TestSessionFactory.Context([.. sessions]))); + + Assert.Equal("fp-F", Assert.IsType(candidate.Subject).Test.TestFingerprint); + + var evidence = Assert.IsType(candidate.Evidence); + + Assert.Equal(5, evidence.BaselineSessions); + Assert.Equal(5, evidence.BaselineSessionCount); + Assert.Equal(3, evidence.CurrentSessionCount); + Assert.Equal(12, evidence.PartialSessionsSetAside); + Assert.Equal(1.0, evidence.BaselineRunRate); + } + + [Fact] + public void TheEvidenceSaysHowManyRunsCoveredOnlyPartOfTheSuite() + { + // Otherwise the denominators are unexplainable: a reader who asked for twenty-three runs is + // being shown a claim about twenty, and nothing on the finding says which twenty or why. + string[] suite = ["A", "B", "C", "D"]; + + var sessions = new List(); + for (int i = 0; i < 17; i++) + sessions.Add(TestSessionFactory.Session(i, suite)); + for (int i = 17; i < 20; i++) + sessions.Add(TestSessionFactory.Session(i, "A", "B", "C")); + for (int i = 20; i < 23; i++) + sessions.Add(TestSessionFactory.Session(i, "A")); + + FindingCandidate candidate = Assert.Single( + Analyze(TestSessionFactory.Context([.. sessions]))); + + (string headline, IReadOnlyList metrics) = + EvidenceHeadline.For(FindingKind.Vanished, candidate.Evidence); + + Assert.Contains("earlier full runs", headline, StringComparison.Ordinal); + Assert.Contains( + metrics, + m => m.Label == "set aside" && m.Value == "3 runs that covered part of the suite"); + } + + [Fact] + public void AnOrdinaryStoreIsNotToldAboutRunsItDoesNotHave() + { + // The qualification is earned, not standing. With nothing set aside there is nothing for + // the word "full" to distinguish the runs from, and the shorter sentence is the true one. + FindingCandidate candidate = Assert.Single(Analyze(Context(total: 20, presentIn: 17))); + + (string headline, IReadOnlyList metrics) = + EvidenceHeadline.For(FindingKind.Vanished, candidate.Evidence); + + Assert.Contains("earlier runs", headline, StringComparison.Ordinal); + Assert.DoesNotContain("full", headline, StringComparison.Ordinal); + Assert.DoesNotContain(metrics, m => m.Label == "set aside"); + } + + [Fact] + public void ADeletionOfMostOfASuiteIsNotReported() + { + // The cost of deciding this on counts, pinned rather than left to be discovered. Sixteen of + // seventeen tests removed and one kept is arithmetically indistinguishable from a filter + // selecting that one, so the report says nothing. Deliberate: the finding is capped at + // Severity.Low because a disappearance is usually something the developer just did, and the + // false positive it trades against arrives once per unselected test on every filtered run. + Assert.Empty(Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 1))); + + // A deletion that leaves most of the suite standing still reports, which is the case the + // kind is actually for. + Assert.Single(Analyze(Suite(total: 20, filtered: 3, suiteSize: 17, selected: 16))); + } } From 33ab0a2ada3cf4751559deb36de1c3bd523165b1 Mon Sep 17 00:00:00 2001 From: xping-admin Date: Mon, 7 Sep 2026 13:25:16 +0200 Subject: [PATCH 2/2] fix(cli): carry the noun into both clauses of the vanished headline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from review. "absent from the last 3" was unambiguous only while every run in the window was one this kind counted. Once runs are set aside it is not: the last three runs and the last three *full* runs are different runs, and the first clause saying "earlier full runs" does not repair the second. A headline is read, and pasted, a clause at a time, so a clause that is only true given the one before it is a clause that will be quoted false. Both clauses carry the noun where anything was set aside, and the sentence stays byte for byte what it was where nothing was — on an ordinary store "the last 3" can mean nothing but the last three runs, and the shorter sentence is the true one there. The two headline tests now pin the whole sentence rather than asserting a substring of it, which is what let the second clause drift in the first place. `SeedVanishing` had two consecutive `` blocks: the edit that added the sizing rationale kept the original summary and added a second rather than merging them. One summary, and the sizing argument stays in the remarks where it was. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TMQcZLBGFTd8wz3mcfn9vc --- .../Report/Contract/EvidenceHeadline.cs | 23 ++++++++++++------- .../Report/ReportEnvelopeTests.cs | 3 --- .../Report/VanishedProviderTests.cs | 12 +++++++--- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs b/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs index c9c88ceb..c9b329a4 100644 --- a/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs +++ b/src/Xping.Cli/Report/Contract/EvidenceHeadline.cs @@ -543,12 +543,14 @@ private static (string, IReadOnlyList) TimeSensitive(TimeSensitiveEvi /// reader who opens the finding to check how much belief the sentence earned. /// /// - /// Both denominators count only the runs that covered the suite, so where any were set aside the - /// sentence says "full runs" and a metric says how many were left out. Qualified rather than - /// worded that way always, because in a store with no filtered runs in it there is nothing for - /// the word to distinguish the runs from, and the shorter sentence is the true one. The wording - /// cannot destabilise the finding's id, which hashes the kind and the subject and nothing else - /// — see . + /// Both denominators count only the runs that covered the suite, so where any were set aside + /// both clauses say "full runs" and a metric says how many were left out. Both, because "the + /// last 3" and "the last 3 full runs" are not the same three runs once anything has been set + /// aside, and a clause that is only true given the one before it is the kind a reader pastes + /// into a chat window on its own. Qualified rather than worded that way always, because in a + /// store with no filtered runs there is nothing for the word to distinguish the runs from and + /// the shorter sentence is the true one. The wording cannot destabilise the finding's id, which + /// hashes the kind and the subject and nothing else — see . /// /// private static (string, IReadOnlyList) Vanished(VanishedEvidence e) @@ -575,9 +577,14 @@ private static (string, IReadOnlyList) Vanished(VanishedEvidence e) if (anySetAside) metrics.Add(new("set aside", $"{Runs(e.PartialSessionsSetAside)} that covered part of the suite")); + // The trailing noun only where it discriminates. On an ordinary store "the last 3" can mean + // nothing but the last three runs, and the sentence is the one this kind has always printed. + string absence = anySetAside + ? $"absent from the last {e.CurrentSessionCount} {runs}" + : $"absent from the last {e.CurrentSessionCount}"; + return ( - $"ran in {e.BaselineSessions} of {e.BaselineSessionCount} earlier {runs}, " + - $"absent from the last {e.CurrentSessionCount}", + $"ran in {e.BaselineSessions} of {e.BaselineSessionCount} earlier {runs}, {absence}", metrics); } diff --git a/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs b/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs index a3b12028..6dcf7021 100644 --- a/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs +++ b/tests/Xping.Cli.Tests/Report/ReportEnvelopeTests.cs @@ -52,9 +52,6 @@ public void Dispose() /// /// Writes sessions in which several tests stop running, so the report has findings to rank. /// - /// - /// Seeds a store in which the newest sessions stop running some of the tests. - /// /// Tests that stop. /// Sessions to write. /// How many of the oldest sessions run the vanishing tests. diff --git a/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs b/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs index 7a369658..5192840b 100644 --- a/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs +++ b/tests/Xping.Cli.Tests/Report/VanishedProviderTests.cs @@ -449,7 +449,12 @@ public void TheEvidenceSaysHowManyRunsCoveredOnlyPartOfTheSuite() (string headline, IReadOnlyList metrics) = EvidenceHeadline.For(FindingKind.Vanished, candidate.Evidence); - Assert.Contains("earlier full runs", headline, StringComparison.Ordinal); + // Both clauses, not just the first. "the last 3" and "the last 3 full runs" are different + // runs once anything has been set aside, and a headline is read a clause at a time. + Assert.Equal( + "ran in 17 of 17 earlier full runs, absent from the last 3 full runs", + headline); + Assert.Contains( metrics, m => m.Label == "set aside" && m.Value == "3 runs that covered part of the suite"); @@ -465,8 +470,9 @@ public void AnOrdinaryStoreIsNotToldAboutRunsItDoesNotHave() (string headline, IReadOnlyList metrics) = EvidenceHeadline.For(FindingKind.Vanished, candidate.Evidence); - Assert.Contains("earlier runs", headline, StringComparison.Ordinal); - Assert.DoesNotContain("full", headline, StringComparison.Ordinal); + // Byte for byte the sentence this kind has always printed: with nothing set aside, "the + // last 3" can mean nothing but the last three runs. + Assert.Equal("ran in 17 of 17 earlier runs, absent from the last 3", headline); Assert.DoesNotContain(metrics, m => m.Label == "set aside"); }