diff --git a/docs/CLI.md b/docs/CLI.md index 3e68738..3dade29 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -107,7 +107,7 @@ or `-Report` file is refused before any file changes (exit 3) and left as it is. | Option | Meaning | |---|---| | `-DetectOnly` | Report detected encodings; do not modify files. | -| `-Validate ` | Strictly validate complete files against an allowed encoding list; do not modify files. | +| `-Validate ` | Strictly validate complete files against an allowed encoding list; do not modify files. `utf-8` also accepts ASCII, which is already UTF-8 without a BOM; `utf-8-bom` does not. | | `-FailOnChanges` | Return exit code 2 when files need conversion or fail validation. Useful in CI. | `-DetectOnly` cannot be combined with `-Validate`, `-Target`, `-From`, diff --git a/docs/DEFECT-BACKLOG.md b/docs/DEFECT-BACKLOG.md index 35d1447..39bc1df 100644 --- a/docs/DEFECT-BACKLOG.md +++ b/docs/DEFECT-BACKLOG.md @@ -4,9 +4,9 @@ This is the current ledger for defects and review findings in EncodingChecker. It is organised by status, not discovery date, so the open work is visible in one place. Longer evidence and history follow the ledger. - + -**Derived count: 81 findings — 71 fixed, 6 open, 1 not reproduced, 1 withdrawn, +**Derived count: 82 findings — 72 fixed, 6 open, 1 not reproduced, 1 withdrawn, 1 intentional behavior, and 1 design decision.** Recompute and check these figures with: @@ -88,6 +88,7 @@ the 2026-09-08 reformat to findings that previously had only a sentence. | BL-35 | Plans, journals, reports and settings replaced a read-only or linked file | Fixed | — | — | [BL-35](#bl-35) | | BL-36 | A conversion refused for a changed file still replaced its earlier backup | Fixed | — | — | [BL-36](#bl-36) | | BL-37 | A contradicting source choice that was also the target was not refused | Fixed | — | — | [BL-37](#bl-37) | +| BL-38 | `-Validate utf-8` rejected ASCII files that conversion treats as already UTF-8 | Fixed | — | — | [BL-38](#bl-38) | | EC-32 | A cancellation timeout could hide repeated refusals to press Cancel | Fixed | — | — | [EC-32](#ec-32) | | EC-15 | Plans and journals showed fixed safety flags as if they recorded checks | Fixed | — | — | [EC-15](#ec-15) | | BL-18 | BOM-less UTF-16 could be detected and converted as UTF-32 | Fixed | — | — | [BL-18](#bl-18) | @@ -1385,6 +1386,23 @@ Semantics stay at 8. The change only turns some `Unchanged` decisions into refusals, which revalidation of an applied plan already permits, and semantics 8 has not been released. +### BL-38 + +**`-Validate` now accepts ASCII wherever BOM-less UTF-8 is allowed.** After BL-34, +conversion to UTF-8 without a BOM leaves ASCII files unchanged, but `-Validate` +still compared labels, so the same file failed `-Validate "utf-8"` as +`CharsetNotAllowed`, exit 2 under `-FailOnChanges`. A pipeline that converted with +`-Target utf-8` and then checked with `-Validate "utf-8,utf-8-bom"` failed every +file whose text was plain English. Reproduced on 2026-09-28 with the `master` build. + +`-Validate` now uses the same rule conversion uses, from `ConversionPolicy`, and +explains the pass with "ASCII is already valid UTF-8 without a BOM." Only the +BOM-less `utf-8` label accepts ASCII; a list allowing only `utf-8-bom` still +rejects it, because an ASCII file has no BOM. The whole file is still validated +as ASCII, so a non-ASCII byte past the detection sample is reported as invalid, +as conversion reports it. Tests cover each list, the explanation and the late +byte. + ## Decisions and mistakes that must remain visible ### A known defect shipped after being reported closed diff --git a/sources/EncodingChecker.Tests/AsciiAlreadyUtf8Tests.cs b/sources/EncodingChecker.Tests/AsciiAlreadyUtf8Tests.cs index 1e9dcbd..70661d5 100644 --- a/sources/EncodingChecker.Tests/AsciiAlreadyUtf8Tests.cs +++ b/sources/EncodingChecker.Tests/AsciiAlreadyUtf8Tests.cs @@ -199,6 +199,55 @@ public void AnAsciiFileSaysWhyItNeedsNoConversion() AssertNoBackupOrRecord(path); } + // -Validate accepts ASCII wherever BOM-less UTF-8 is allowed, as conversion does, and + // still reads the whole file. utf-8-bom alone does not accept it: ASCII has no BOM. + [Theory] + [InlineData("utf-8", "Unchanged,,ASCII is already valid UTF-8 without a BOM.", ExpectedClean)] + [InlineData("utf-8,utf-8-bom", "Unchanged,,ASCII is already valid UTF-8 without a BOM.", ExpectedClean)] + [InlineData("us-ascii", "Unchanged,,", ExpectedClean)] + [InlineData("utf-8-bom", "Invalid,CharsetNotAllowed,", ExpectedChangesNeeded)] + public void ValidateAcceptsAsciiWhereBomlessUtf8IsAllowed( + string allowed, string expectedOutcome, int expectedExit) + { + string path = WriteAscii(); + + (int exit, string output, _) = RunCaptured( + "-BasePath", _root, "-Validate", allowed, "-FailOnChanges"); + + Assert.Equal(expectedExit, exit); + Assert.Contains( + output.Split(Environment.NewLine), + line => line.StartsWith(path + ",us-ascii,No,", StringComparison.Ordinal) + && line.Contains("," + expectedOutcome, StringComparison.Ordinal)); + } + + [Fact] + public void ValidateStillReadsTheWholeFileOfAnAsciiSample() + { + // ASCII in the 64 KiB detection sample, then a UTF-8 "é". Accepting it under utf-8 + // must not skip the full-file check, which conversion applies to the same file. + var text = new StringBuilder(); + + while (text.Length < 70 * 1024) + text.Append("the quick brown fox jumps over the lazy dog. "); + + text.Append("café\n"); + + string path = PathOf("late.txt"); + File.WriteAllBytes(path, new UTF8Encoding(false).GetBytes(text.ToString())); + + (int exit, string output, _) = RunCaptured( + "-BasePath", _root, "-Validate", "utf-8", "-FailOnChanges"); + + Assert.Equal(ExpectedChangesNeeded, exit); + Assert.Contains( + output.Split(Environment.NewLine), + line => line.StartsWith(path + ",us-ascii,", StringComparison.Ordinal) + && line.Contains( + $",Invalid,{ConversionReasonCodes.StrictValidationFailed},", + StringComparison.Ordinal)); + } + [Fact] public void AFileAlreadyInTheTargetCodecGetsNoExplanation() { diff --git a/sources/EncodingChecker/ConversionPolicy.cs b/sources/EncodingChecker/ConversionPolicy.cs index caf975b..3556226 100644 --- a/sources/EncodingChecker/ConversionPolicy.cs +++ b/sources/EncodingChecker/ConversionPolicy.cs @@ -14,6 +14,24 @@ internal static class ConversionPolicy private const int AsciiCodePage = 20127; private const int Utf8CodePage = 65001; + /// Why an ASCII file needs nothing to become UTF-8 without a BOM. + internal const string AsciiAlreadyUtf8Reason = "ASCII is already valid UTF-8 without a BOM."; + + /// + /// Whether the source is ASCII and the target is UTF-8, both without a BOM, so the bytes + /// already are the target encoding. + /// + /// + /// Kept here so conversion and -Validate accept the same files. Either way the whole + /// file is still validated as ASCII afterwards. + /// + internal static bool IsAsciiAlreadyUtf8( + int sourceCodePage, bool sourceHasBom, int targetCodePage, bool targetHasBom) => + sourceCodePage == AsciiCodePage + && !sourceHasBom + && targetCodePage == Utf8CodePage + && !targetHasBom; + /// /// Decides what to do with one file, given what is known about its encoding. /// @@ -94,13 +112,10 @@ internal static PlannedAction Decide( // and, with backups on, a repeat run would replace the backup of an earlier original // with a copy of the current file. The unchanged path validates the whole file as // ASCII, so a later non-ASCII byte is reported as an error. - if (sourceCodePage == AsciiCodePage - && !sourceHasBom - && targetCodePage == Utf8CodePage - && !targetHasBom) + if (IsAsciiAlreadyUtf8(sourceCodePage, sourceHasBom, targetCodePage, targetHasBom)) { sourceInterpretation = SourceInterpretation.NotApplicable; - reason = "ASCII is already valid UTF-8 without a BOM."; + reason = AsciiAlreadyUtf8Reason; return PlannedAction.Unchanged; } diff --git a/sources/EncodingChecker/ScanEngine.cs b/sources/EncodingChecker/ScanEngine.cs index d7ae52e..fb10286 100644 --- a/sources/EncodingChecker/ScanEngine.cs +++ b/sources/EncodingChecker/ScanEngine.cs @@ -554,12 +554,21 @@ private static ConversionReportEntry ProcessFileForScan( string label = FormatCharsetLabel(sourceCharset, hasBom); - bool isValid = - sourceCharset != UnknownCharset && + // ASCII bytes are already BOM-less UTF-8, so a list allowing that allows ASCII. + // Only the BOM-less label: an ASCII file has no BOM, so utf-8-bom still fails it. + bool allowedAsUtf8 = + detected is not null && + options.ValidCharsets is not null && + options.ValidCharsets.Contains("utf-8", StringComparer.OrdinalIgnoreCase) && + ConversionPolicy.IsAsciiAlreadyUtf8( + detected.CodePage, hasBom, Encoding.UTF8.CodePage, targetHasBom: false); + + bool listed = options.ValidCharsets is not null && - options.ValidCharsets.Contains( - label, - StringComparer.OrdinalIgnoreCase); + options.ValidCharsets.Contains(label, StringComparer.OrdinalIgnoreCase); + + bool isValid = + sourceCharset != UnknownCharset && (listed || allowedAsUtf8); string? validationDiagnostic = null; @@ -606,6 +615,11 @@ options.ValidCharsets is not null && + $" The label '{label}' is in the allowed list, but EC cannot" + " confirm this file belongs to it."; } + else if (!listed) + { + // Passed only through the UTF-8 rule, so say why a us-ascii row passes. + entry.Diagnostic = ConversionPolicy.AsciiAlreadyUtf8Reason; + } break;