Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions docs/DEFECT-BACKLOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ This is the current ledger for defects and review findings in EncodingChecker.
It is organised by status, not discovery date, so the open work is visible in
one place. Longer evidence and history follow the ledger.

<!-- backlog-counts total=80 fixed=70 open=6 not-reproduced=1 withdrawn=1 intentional-behavior=1 decision=1 -->
<!-- backlog-counts total=81 fixed=71 open=6 not-reproduced=1 withdrawn=1 intentional-behavior=1 decision=1 -->

**Derived count: 80 findings — 70 fixed, 6 open, 1 not reproduced, 1 withdrawn,
**Derived count: 81 findings — 71 fixed, 6 open, 1 not reproduced, 1 withdrawn,
1 intentional behavior, and 1 design decision.** Recompute and check these
figures with:

Expand Down Expand Up @@ -87,6 +87,7 @@ the 2026-09-08 reformat to findings that previously had only a sentence.
| BL-34 | Repeating a backed-up UTF-8 conversion could replace the original backup once the text was ASCII | Fixed | — | — | [BL-34](#bl-34) |
| BL-35 | Plans, journals, reports and settings replaced a read-only or linked file | Fixed | — | — | [BL-35](#bl-35) |
| BL-36 | A conversion refused for a changed file still replaced its earlier backup | Fixed | — | — | [BL-36](#bl-36) |
| BL-37 | A contradicting source choice that was also the target was not refused | Fixed | — | — | [BL-37](#bl-37) |
| EC-32 | A cancellation timeout could hide repeated refusals to press Cancel | Fixed | — | — | [EC-32](#ec-32) |
| EC-15 | Plans and journals showed fixed safety flags as if they recorded checks | Fixed | — | — | [EC-15](#ec-15) |
| BL-18 | BOM-less UTF-16 could be detected and converted as UTF-32 | Fixed | — | — | [BL-18](#bl-18) |
Expand Down Expand Up @@ -1360,6 +1361,30 @@ This covers a change made before the backup is staged. A change after that point
or another later conversion failure, can still leave the new backup in place of
the old one.

### BL-37

**A source choice that contradicts reliable detection is now refused even when it
names the target encoding.** The policy checked "already in the target" before the
conflict, so when the chosen source was also the target the file skipped the
conflict check. Reproduced on 2026-09-28 with the `master` build:

- `-From windows-1252 -Target windows-1252` on a UTF-8 file with non-ASCII text
reported it `Unchanged`, already windows-1252, and exited 0. The report was
false, and `-FailOnChanges` passed.
- `-From utf-8 -Target utf-8` on UTF-16 with a BOM ended as a validation error,
exit 3, instead of a refusal naming the conflict.

Nothing was written in either case. The conflict check now runs before both
"already in the target" rules, so both files are `Refused` with
`ExplicitSourceConflictsWithDetection` and exit 5, and an existing backup is left
as it was. A choice that agrees with detection is still `Unchanged`. Tests cover
both reproductions, the agreeing control and the rule order; restoring the old
order fails them.

Semantics stay at 8. The change only turns some `Unchanged` decisions into
refusals, which revalidation of an applied plan already permits, and semantics 8
has not been released.

## Decisions and mistakes that must remain visible

### A known defect shipped after being reported closed
Expand Down
65 changes: 65 additions & 0 deletions sources/EncodingChecker.Tests/ExplicitSourceEncodingTests.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System.Text;

using System.Text.Json;
using static EncodingChecker.Tests.CliRunner;
using static EncodingChecker.Tests.ExpectedExitCode;

namespace EncodingChecker.Tests;

Expand Down Expand Up @@ -268,4 +270,67 @@ public void DetectionModeIsRecordedSoTheTwoClaimsStayDistinct()
Assert.False(Assert.Single(Scan(from: null)).SourceEncodingWasSpecified);
Assert.True(Assert.Single(Scan(from: "windows-1252")).SourceEncodingWasSpecified);
}

// The conflict check comes before "already in the target". When the chosen source was
// also the target, the file was reported as already in it: a UTF-8 file as already
// windows-1252 with exit 0, or UTF-16 as a UTF-8 validation error with exit 3.
[Theory]
[InlineData("utf-16-with-bom", "utf-8")]
[InlineData("utf-8", "windows-1252")]
public void AChoiceThatContradictsReliableDetectionIsRefusedEvenWhenItIsTheTarget(
string actual, string chosen)
{
byte[] original = actual == "utf-8"
? new UTF8Encoding(false).GetBytes("Hello 世界, café text\n")
: [.. Encoding.Unicode.GetPreamble(), .. Encoding.Unicode.GetBytes("Hello 世界 text\n")];
string path = Write("reliable.txt", original);
byte[] earlierBackup = Encoding.ASCII.GetBytes("backup from an earlier run\r\n");
File.WriteAllBytes(path + ".bak", earlierBackup);

(int exit, string output, _) = RunCaptured(
"-BasePath", _root, "-From", chosen, "-Target", chosen, "-Backup");

Assert.Equal(ExpectedSafeRefusal, exit);
Assert.Contains(
output.Split(Environment.NewLine),
line => line.StartsWith(path + ",", StringComparison.Ordinal)
&& line.Contains(
$",Refused,{ConversionReasonCodes.ExplicitSourceConflictsWithDetection},",
StringComparison.Ordinal));
Assert.Equal(original, File.ReadAllBytes(path));
Assert.Equal(earlierBackup, File.ReadAllBytes(path + ".bak"));
}

[Fact]
public void AChoiceThatAgreesWithDetectionAndIsTheTargetIsStillUnchanged()
{
byte[] original = new UTF8Encoding(false).GetBytes("Hello 世界, café text\n");
string path = Write("agrees.txt", original);

(int exit, string output, _) = RunCaptured(
"-BasePath", _root, "-From", "utf-8", "-Target", "utf-8", "-Backup");

Assert.Equal(ExpectedClean, exit);
Assert.Contains(
output.Split(Environment.NewLine),
line => line.StartsWith(path + ",", StringComparison.Ordinal)
&& line.Contains(",Unchanged,", StringComparison.Ordinal));
Assert.Equal(original, File.ReadAllBytes(path));
Assert.False(File.Exists(path + ".bak"));
}

[Fact]
public void ThePolicyChecksAConflictBeforeTheSameCodecRule()
{
PlannedAction action = ConversionPolicy.Decide(
"windows-1252", sourceCodePage: 1252, sourceHasBom: false,
"windows-1252", targetCodePage: 1252, targetHasBom: false,
sourceWasSpecified: true, isUnicodeOrAscii: false,
explicitSourceConflictsWithReliableDetection: true,
automaticBomlessUnicodeDoubt: BomlessUnicodeKind.None,
out SourceInterpretation interpretation, out _);

Assert.Equal(PlannedAction.Refuse, action);
Assert.Equal(SourceInterpretation.ExplicitSource, interpretation);
}
}
22 changes: 12 additions & 10 deletions sources/EncodingChecker/ConversionPolicy.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,17 @@ internal static PlannedAction Decide(
return PlannedAction.Skip;
}

// A source choice that contradicts reliable detection is refused before anything
// else is decided, including "already in the target": that claim would rest on the
// contradicted choice, and would report a UTF-8 file as already windows-1252.
if (sourceWasSpecified && explicitSourceConflictsWithReliableDetection)
{
sourceInterpretation = SourceInterpretation.ExplicitSource;
reason = "The selected source encoding conflicts with EC's reliable Unicode "
+ "or ASCII detection and could change the text. No conversion was performed.";
return PlannedAction.Refuse;
}

// An unchanged file is not rewritten, so no source choice is needed. It is still
// read and validated in full after this decision.
//
Expand All @@ -79,19 +90,10 @@ internal static PlannedAction Decide(
return PlannedAction.Unchanged;
}

if (sourceWasSpecified && explicitSourceConflictsWithReliableDetection)
{
sourceInterpretation = SourceInterpretation.ExplicitSource;
reason = "The selected source encoding conflicts with EC's reliable Unicode "
+ "or ASCII detection and could change the text. No conversion was performed.";
return PlannedAction.Refuse;
}

// ASCII bytes are already BOM-less UTF-8. Converting would rewrite identical bytes
// and, with backups on, a repeat run would replace the backup of an earlier original
// with a copy of the current file. The unchanged path validates the whole file as
// ASCII, so a later non-ASCII byte is reported as an error. This follows the conflict
// check so a source choice that contradicts reliable detection is still refused.
// ASCII, so a later non-ASCII byte is reported as an error.
if (sourceCodePage == AsciiCodePage
&& !sourceHasBom
&& targetCodePage == Utf8CodePage
Expand Down
Loading