Skip to content

Commit 793edd4

Browse files
Fix requirements files read as absent when not UTF-8 (#1120, #1119) (#1152)
* Start fix for #1120, #1119 Assisted-by: Claude Code:claude-opus-5-5 * Read requirements files the way pip decodes them A requirements.txt that pip or uv installs from was read as absent whenever it was not plain UTF-8: - A file saved as Latin-1 or cp1252 with a PEP 263 coding line ("# -*- coding: latin-1 -*-") was skipped by lock-only discovery, so a fresh-checkout scan said "No pypi packages found" and exited 0 while pip installed the unpatched release (#1119). - A UTF-16 requirements.txt exported beside uv.lock (what `uv export > requirements.txt` writes in Windows PowerShell 5.1) was ignored by the vendored routing, `vendor --check` and `vex`. The scan gave no pypi_multiple_lockfiles warning, the check passed and VEX attested not_affected while `uv pip install -r requirements.txt` installed the unpatched release (#1120). pip's decoder now lives in one place. utils::requirements::decode follows pip's auto_decode: byte-order marks first, then a PEP 263 coding line in the first two lines (UTF-8, ASCII, Latin-1 and cp1252 and their Python aliases), then UTF-8. A codec it does not model is reported as unreadable, never guessed. Every reader that only observes requirements files now goes through it: the -r include walk, the vendored #612 probe beside a tool lock, and VEX discovery. The probe also counts a requirements.txt it cannot decode as one that may pin the package, so the warning fails closed. The vendored planner still refuses files that are not UTF-8, since it rewrites them byte for byte. Fixes #1119 Fixes #1120 Assisted-by: Claude Code:claude-opus-5-5 * Test vendor --check on a UTF-16 export beside uv Adds the CLI-level regression for #1120: a requirements.txt exported beside uv.lock (UTF-8, UTF-16 LE and UTF-16 BE) makes `vendor` warn pypi_multiple_lockfiles and `vendor --check` fail on the contested wiring, and is left byte-for-byte as the user wrote it. Before the decoder change, both UTF-16 cases passed the check as if the file were absent. Refs #1120 Assisted-by: Claude Code:claude-opus-5-5 * Use the shared BOM helper in the requirements decoder The formats::text guard test requires every UTF-8 BOM check to go through the shared helpers (#905). The PEP 263 step checked for the mark inline; it now asks strip_bom_bytes instead. No behavior change. Refs #1119 Assisted-by: Claude Code:claude-opus-5-5 --------- Co-authored-by: Claude <noreply@anthropic.com>
1 parent 03b9418 commit 793edd4

8 files changed

Lines changed: 517 additions & 17 deletions

File tree

‎crates/socket-patch-cli/tests/mode_migration_pypi.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1416,6 +1416,67 @@ async fn vendor_check_fails_after_hatch_dependency_reset() {
14161416
assert_check_catches_relock(&root, files);
14171417
}
14181418

1419+
/// #1120: a `requirements.txt` exported beside `uv.lock` pins the
1420+
/// package from PyPI, so the uv-only wiring leaves installs from it
1421+
/// unpatched. `uv export > requirements.txt` in Windows PowerShell 5.1
1422+
/// writes it as UTF-16 with a byte-order mark, which pip and uv read like
1423+
/// its UTF-8 twin, so `vendor` must name it (`pypi_multiple_lockfiles`)
1424+
/// and `vendor --check` must fail on the contested wiring in every
1425+
/// encoding, not pass as if the file were absent.
1426+
#[tokio::test]
1427+
async fn uv_requirements_export_contests_the_wiring_in_any_encoding() {
1428+
let text = "six==1.16.0\r\n";
1429+
let le: Vec<u8> = [0xFF, 0xFE]
1430+
.into_iter()
1431+
.chain(text.encode_utf16().flat_map(u16::to_le_bytes))
1432+
.collect();
1433+
let be: Vec<u8> = [0xFE, 0xFF]
1434+
.into_iter()
1435+
.chain(text.encode_utf16().flat_map(u16::to_be_bytes))
1436+
.collect();
1437+
for (case, bytes) in [
1438+
("utf-8", text.as_bytes().to_vec()),
1439+
("utf-16 le", le),
1440+
("utf-16 be", be),
1441+
] {
1442+
let (_tmp, root) = project();
1443+
stage_uv(&root);
1444+
std::fs::write(root.join("requirements.txt"), &bytes).unwrap();
1445+
stage_manifest(&root);
1446+
let (code, env) = run_cli(&root, &["vendor"], &[]);
1447+
assert_eq!(code, 0, "{case}: {env:#}");
1448+
assert!(
1449+
std::fs::read_to_string(root.join("uv.lock"))
1450+
.unwrap()
1451+
.contains(&format!(".socket/vendor/pypi/{UUID}/")),
1452+
"{case}: uv.lock is wired: {env:#}"
1453+
);
1454+
assert!(
1455+
env.to_string().contains("pypi_multiple_lockfiles"),
1456+
"{case}: vendor names requirements.txt as an unpatched install source: {env:#}"
1457+
);
1458+
1459+
let (code, env) = run_cli(&root, &["vendor", "--check"], &[]);
1460+
assert_eq!(
1461+
code, 1,
1462+
"{case}: requirements.txt installs unpatched: {env:#}"
1463+
);
1464+
let event = &env["events"][0];
1465+
assert_eq!(event["errorCode"], "vendor_check_failed", "{case}: {env:#}");
1466+
let reason = event["reason"].as_str().unwrap_or_default();
1467+
assert!(
1468+
reason.contains("wiring contested") && reason.contains("requirements.txt"),
1469+
"{case}: {env:#}"
1470+
);
1471+
// The vendored file is left as the user wrote it.
1472+
assert_eq!(
1473+
std::fs::read(root.join("requirements.txt")).unwrap(),
1474+
bytes,
1475+
"{case}"
1476+
);
1477+
}
1478+
}
1479+
14191480
/// #699: hosted mode rewrites only the ROOT `requirements.txt`, while
14201481
/// vendored mode also wires a pin in a `-r` include or appends a managed
14211482
/// `(transitive)` line. A vendored → hosted takeover of such a pin used to

‎crates/socket-patch-cli/tests/scan_requirements_lock_only.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
//! * #412: pins reached through in-root `-r` includes;
99
//! * #721: a UTF-16 file with a BOM (Windows PowerShell 5.1's
1010
//! `pip freeze >` output), which pip decodes.
11+
//! * #1119: a file pip decodes through a PEP 263 coding line
12+
//! (`# -*- coding: latin-1 -*-`), as the root file or an include.
1113
//! * #994: include targets pip unquotes (`-r "dev reqs.txt"`,
1214
//! `--requirement="dev.txt"`, `-r dev\ reqs.txt`) or expands
1315
//! (`-r ${REQDIR}/dev.txt`);
@@ -209,6 +211,35 @@ async fn lock_only_scan_discovers_utf16_pins() {
209211
}
210212
}
211213

214+
/// #1119: with no BOM, pip decodes a requirements file through a PEP 263
215+
/// coding line, so a Latin-1 file with a non-ASCII comment is discovered,
216+
/// as the root file and as a `-r` include, instead of reading as
217+
/// "No packages found".
218+
#[tokio::test]
219+
async fn lock_only_scan_discovers_pep_263_pins() {
220+
let latin1 =
221+
b"# -*- coding: latin-1 -*-\n# Maintainer: Jos\xe9\nsp-fixture-six==1.16.0\n".to_vec();
222+
assert_lock_only_discovers_bytes(
223+
&[("requirements.txt", &latin1[..])],
224+
&["pkg:pypi/sp-fixture-six@1.16.0"],
225+
)
226+
.await;
227+
assert_lock_only_discovers_bytes(
228+
&[
229+
(
230+
"requirements.txt",
231+
&b"-r dev.txt\nsp-fixture-idna==3.7\n"[..],
232+
),
233+
("dev.txt", &latin1[..]),
234+
],
235+
&[
236+
"pkg:pypi/sp-fixture-idna@3.7",
237+
"pkg:pypi/sp-fixture-six@1.16.0",
238+
],
239+
)
240+
.await;
241+
}
242+
212243
/// #994: pip `shlex`-splits an include line's options, so a quoted or
213244
/// backslash-escaped target names the file without its quotes, and a
214245
/// target with a space is one path, not two words.

‎crates/socket-patch-core/src/utils/requirements.rs‎

Lines changed: 136 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,20 @@
1414
//! `--hash=sha256:ab#cd` are data. Exactly one leading BOM is encoding, not
1515
//! data (pip decodes with utf-8-sig; uv strips it too).
1616
17-
use crate::formats::text::strip_bom;
17+
use crate::formats::text::{strip_bom, strip_bom_bytes};
1818

1919
/// Decode a requirements file the way pip's `auto_decode` does: a UTF-16
20-
/// or UTF-32 byte-order mark selects that encoding and is dropped; anything
20+
/// or UTF-32 byte-order mark selects that encoding and is dropped; a
21+
/// mark-less file with a PEP 263 coding line (`# -*- coding: latin-1 -*-`)
22+
/// in its first two lines is decoded in that encoding (#1119); anything
2123
/// else is UTF-8, its one leading BOM kept for [`logical_lines`] to drop.
2224
/// Windows PowerShell 5.1 writes `pip freeze > requirements.txt` as UTF-16
2325
/// LE with a BOM, and pip installs from it (#721). pip tries the UTF-16
2426
/// marks first, so a UTF-32 LE mark (`FF FE 00 00`) reads as UTF-16 LE, as
25-
/// it does for pip. `None` when the bytes are not valid in that encoding.
26-
/// (pip's last resort, the locale's encoding for a mark-less non-UTF-8
27-
/// file, is machine-dependent and not modelled.)
27+
/// it does for pip. `None` when the bytes are not valid in that encoding,
28+
/// or the coding line names a codec this reader does not model (see
29+
/// [`coding_line_codec`]). (pip's last resort, the locale's encoding for a
30+
/// mark-less non-UTF-8 file, is machine-dependent and not modelled.)
2831
pub(crate) fn decode(bytes: &[u8]) -> Option<String> {
2932
fn utf16(body: &[u8], unit: fn([u8; 2]) -> u16) -> Option<String> {
3033
if !body.len().is_multiple_of(2) {
@@ -49,9 +52,95 @@ pub(crate) fn decode(bytes: &[u8]) -> Option<String> {
4952
.map(|c| char::from_u32(u32::from_be_bytes([c[0], c[1], c[2], c[3]])))
5053
.collect();
5154
}
55+
if strip_bom_bytes(bytes).len() == bytes.len() {
56+
if let Some(name) = coding_line(bytes) {
57+
return match coding_line_codec(&name)? {
58+
Codec::Utf8 => String::from_utf8(bytes.to_vec()).ok(),
59+
Codec::Ascii => bytes
60+
.is_ascii()
61+
.then(|| String::from_utf8_lossy(bytes).into_owned()),
62+
Codec::Latin1 => Some(bytes.iter().map(|&b| char::from(b)).collect()),
63+
Codec::Cp1252 => bytes.iter().map(|&b| cp1252_char(b)).collect(),
64+
};
65+
}
66+
}
5267
String::from_utf8(bytes.to_vec()).ok()
5368
}
5469

70+
/// The codecs a PEP 263 coding line may select that [`decode`] models.
71+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72+
enum Codec {
73+
Utf8,
74+
Ascii,
75+
Latin1,
76+
Cp1252,
77+
}
78+
79+
/// The encoding name of pip's PEP 263 check: the first of the file's first
80+
/// two lines that starts with `#` and matches `coding[:=]\s*([-\w.]+)`.
81+
fn coding_line(bytes: &[u8]) -> Option<String> {
82+
bytes.split(|&b| b == b'\n').take(2).find_map(|line| {
83+
if line.first() != Some(&b'#') {
84+
return None;
85+
}
86+
line.windows(6)
87+
.enumerate()
88+
.filter(|(_, w)| *w == b"coding")
89+
.find_map(|(at, _)| {
90+
let rest = &line[at + 6..];
91+
let rest = rest
92+
.strip_prefix(b":")
93+
.or_else(|| rest.strip_prefix(b"="))?;
94+
let start = rest.iter().position(|b| !b.is_ascii_whitespace())?;
95+
let name: Vec<u8> = rest[start..]
96+
.iter()
97+
.copied()
98+
.take_while(|&b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.'))
99+
.collect();
100+
(!name.is_empty()).then(|| String::from_utf8_lossy(&name).into_owned())
101+
})
102+
})
103+
}
104+
105+
/// Python's codec lookup for the names [`decode`] models: case-folded,
106+
/// `-` and ` ` read as `_`, then Python's alias table for UTF-8, ASCII,
107+
/// Latin-1 and cp1252. Any other codec (pip would use it) is `None`, so
108+
/// the file reads as undecodable rather than being guessed at.
109+
fn coding_line_codec(name: &str) -> Option<Codec> {
110+
let name = name.to_ascii_lowercase().replace(['-', ' '], "_");
111+
Some(match name.as_str() {
112+
"utf_8" | "utf8" | "u8" | "utf" | "utf8_ucs2" | "utf8_ucs4" | "cp65001" | "utf_8_sig" => {
113+
Codec::Utf8
114+
}
115+
"ascii" | "646" | "us_ascii" | "us" | "cp367" | "csascii" | "ibm367" | "iso646_us"
116+
| "iso_ir_6" | "ansi_x3.4_1968" | "ansi_x3_4_1968" | "ansi_x3.4_1986"
117+
| "iso_646.irv_1991" => Codec::Ascii,
118+
"latin_1" | "latin1" | "latin" | "l1" | "8859" | "cp819" | "csisolatin1" | "ibm819"
119+
| "iso8859" | "iso8859_1" | "iso_8859_1" | "iso_8859_1_1987" | "iso_ir_100" => {
120+
Codec::Latin1
121+
}
122+
"cp1252" | "windows_1252" | "1252" => Codec::Cp1252,
123+
_ => return None,
124+
})
125+
}
126+
127+
/// One cp1252 byte as Python decodes it: Latin-1 outside `0x80..=0x9F`,
128+
/// Windows punctuation inside it, and five bytes Python leaves undefined.
129+
fn cp1252_char(b: u8) -> Option<char> {
130+
const HIGH: [u16; 32] = [
131+
0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039,
132+
0x0152, 0, 0x017D, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC,
133+
0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178,
134+
];
135+
match b {
136+
0x80..=0x9F => match HIGH[usize::from(b - 0x80)] {
137+
0 => None,
138+
unit => char::from_u32(u32::from(unit)),
139+
},
140+
_ => Some(char::from(b)),
141+
}
142+
}
143+
55144
/// One logical requirements line.
56145
pub(crate) struct LogicalLine {
57146
/// 0-based index of the first physical line.
@@ -393,6 +482,48 @@ mod tests {
393482
assert_eq!(decode(&[b's', 0xC3, 0x28]), None);
394483
}
395484

485+
/// #1119: with no BOM, pip honours a PEP 263 coding line in the
486+
/// file's first two lines (`auto_decode`), so a Latin-1 file with a
487+
/// non-ASCII comment installs. Codecs this reader does not model stay
488+
/// unreadable instead of guessed.
489+
#[test]
490+
fn decode_follows_pips_pep_263_coding_line() {
491+
let latin1 = b"# -*- coding: latin-1 -*-\n# Maintainer: Jos\xe9\nsix==1.16.0\n";
492+
assert_eq!(
493+
decode(latin1).as_deref(),
494+
Some("# -*- coding: latin-1 -*-\n# Maintainer: Jos\u{e9}\nsix==1.16.0\n")
495+
);
496+
// The second line counts; every alias spelling Python accepts.
497+
for coding in ["ISO-8859-1", "iso8859_1", "latin1", "L1", "cp819"] {
498+
let text = format!("# deps\n# vim: set fileencoding={coding} :\nsix==1.16.0 # Jos");
499+
let mut bytes = text.into_bytes();
500+
bytes.push(0xE9);
501+
assert!(
502+
decode(&bytes).is_some_and(|t| t.ends_with("Jos\u{e9}")),
503+
"{coding}"
504+
);
505+
}
506+
// cp1252's Windows punctuation, and its bytes Python leaves undefined.
507+
assert_eq!(
508+
decode(b"# coding=cp1252\n# \x93six\x94\nsix==1.16.0\n").as_deref(),
509+
Some("# coding=cp1252\n# \u{201c}six\u{201d}\nsix==1.16.0\n")
510+
);
511+
assert_eq!(decode(b"# coding: windows-1252\n# \x81\n"), None);
512+
// A UTF-8 coding line still requires UTF-8.
513+
assert_eq!(decode(b"# coding: utf-8\n# Jos\xe9\n"), None);
514+
assert_eq!(decode(b"# coding: ascii\n# Jos\xe9\n"), None);
515+
// A coding line on the third line, or not in a comment, is data.
516+
assert_eq!(decode(b"# a\n# b\n# coding: latin-1\n# \xe9\n"), None);
517+
assert_eq!(decode(b"six==1.16.0 # coding: latin-1 \xe9\n"), None);
518+
// A codec this reader does not model is unreadable, never guessed.
519+
assert_eq!(decode(b"# coding: koi8-r\n# \xe9\n"), None);
520+
// A BOM wins over a coding line, as in pip.
521+
assert_eq!(
522+
decode(b"\xef\xbb\xbf# coding: latin-1\nsix==1.16.0\n").as_deref(),
523+
Some("\u{feff}# coding: latin-1\nsix==1.16.0\n")
524+
);
525+
}
526+
396527
#[test]
397528
fn requires_hashes_reads_pip_hash_checking_mode() {
398529
for hashed in [

‎crates/socket-patch-core/src/vendor/lock_inventory/tests.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3608,6 +3608,56 @@ async fn requirements_utf16_files_are_inventoried() {
36083608
}
36093609
}
36103610

3611+
/// #1119: with no BOM, pip decodes a requirements file through a PEP 263
3612+
/// coding line, so a Latin-1 root file and a Latin-1 include are
3613+
/// inventoried on a fresh checkout instead of reading as "no requirements",
3614+
/// on disk and in memory.
3615+
#[tokio::test]
3616+
async fn requirements_pep_263_files_are_inventoried() {
3617+
let latin1 = |pins: &str| {
3618+
let mut bytes = b"# -*- coding: latin-1 -*-\n# Maintainer: Jos\xe9\n".to_vec();
3619+
bytes.extend_from_slice(pins.as_bytes());
3620+
bytes
3621+
};
3622+
for (root_bytes, base_bytes) in [
3623+
// The root file itself.
3624+
(
3625+
latin1("-r requirements/base.txt\nidna==3.7\n"),
3626+
b"six==1.16.0\n".to_vec(),
3627+
),
3628+
// Only the include.
3629+
(
3630+
b"-r requirements/base.txt\nidna==3.7\n".to_vec(),
3631+
latin1("six==1.16.0\n"),
3632+
),
3633+
] {
3634+
let tmp = tempfile::tempdir().unwrap();
3635+
std::fs::create_dir_all(tmp.path().join("requirements")).unwrap();
3636+
std::fs::write(tmp.path().join("requirements.txt"), &root_bytes).unwrap();
3637+
std::fs::write(tmp.path().join("requirements/base.txt"), &base_bytes).unwrap();
3638+
let entries = inventory_pypi_locks(tmp.path()).await.unwrap();
3639+
assert_eq!(
3640+
sorted_pairs(&entries),
3641+
vec![
3642+
("idna".to_string(), "3.7".to_string()),
3643+
("six".to_string(), "1.16.0".to_string()),
3644+
],
3645+
"{entries:?}"
3646+
);
3647+
3648+
let mut project = MemoryProject::new();
3649+
project.insert("requirements.txt", MemoryEntry::Binary(root_bytes.into()));
3650+
project.insert(
3651+
"requirements/base.txt",
3652+
MemoryEntry::Binary(base_bytes.into()),
3653+
);
3654+
let in_memory = super::pypi::inventory_pypi_locks_in(&ProjectView::Memory(&project))
3655+
.await
3656+
.unwrap();
3657+
assert_eq!(sorted_pairs(&in_memory), sorted_pairs(&entries));
3658+
}
3659+
}
3660+
36113661
/// pip applies an index option from ANY file of the tree globally, so an
36123662
/// `--index-url` inside an include keeps the root file's hashed pins
36133663
/// unverifiable too (the `public_index` rule spans the whole tree).

0 commit comments

Comments
 (0)