From 6f46cb2ce2da44080a5af9e1852fedaec62027a6 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Mon, 28 Sep 2026 18:59:34 +0200 Subject: [PATCH 1/3] ci: pin TLS-verified patch hosts on macOS compat legs The Bun, vlt and Poetry compatibility workflows drive real package managers against the production patch service. On GitHub's hosted macOS runners the system resolver intermittently answers patch.socket.dev with EAI_NONAME ("[Errno 8] nodename nor servname provided"; bun: FailedToOpenSocket; Bun 1.3.x workspace installs never exit) for minutes at a time, at job start or mid-job, while the service is up: ubuntu and windows legs of the same run pass, and the same macOS cells pass before and after the window. Over the last 60 Bun runs (69 attempts) 29 macOS native jobs failed this way and no other OS did; the CLI's own API calls in those cells succeeded. A pre-flight wait cannot cover a mid-job window, and the failing processes are bun / vlt / poetry / python rather than the CLI, so a product retry cannot help. The runner's resolver is not under test, so take it out of the path: .github/actions/pin-socket-hosts runs scripts/pin-socket-hosts.py on macOS, which resolves patch.socket.dev and patches-api.socket.dev (system resolver, then DNS-over-HTTPS by IP literal, with bounded backoff), keeps only addresses whose TLS handshake verifies the hostname, and pins them in /etc/hosts. Every cell still hits production over TLS verified for the hostname, so the captures depscan imports stay production captures. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/pin-socket-hosts/action.yml | 42 ++++++ .github/workflows/bun-compatibility.yml | 9 ++ .github/workflows/poetry-compatibility.yml | 8 ++ .github/workflows/vlt-compatibility.yml | 8 ++ docs/testing/bun-compatibility.md | 12 ++ docs/testing/vlt-compatibility.md | 5 +- scripts/pin-socket-hosts.py | 140 ++++++++++++++++++++ scripts/tests/test_pin_socket_hosts.py | 76 +++++++++++ 8 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 .github/actions/pin-socket-hosts/action.yml create mode 100644 scripts/pin-socket-hosts.py create mode 100644 scripts/tests/test_pin_socket_hosts.py diff --git a/.github/actions/pin-socket-hosts/action.yml b/.github/actions/pin-socket-hosts/action.yml new file mode 100644 index 000000000..c7807042e --- /dev/null +++ b/.github/actions/pin-socket-hosts/action.yml @@ -0,0 +1,42 @@ +name: Pin Socket patch hosts +description: >- + On macOS runners, resolve the production patch hosts once (system resolver, + then DNS-over-HTTPS by IP literal), TLS-verify every address for its host, + and pin them in /etc/hosts for the rest of the job +inputs: + hosts: + description: Space-separated hostnames to pin + default: patch.socket.dev patches-api.socket.dev +runs: + using: composite + steps: + # GitHub's hosted macOS runners intermittently answer patch.socket.dev + # with EAI_NONAME ("[Errno 8] nodename nor servname provided", bun's + # `FailedToOpenSocket`) for minutes at a time — at job start or mid-job — + # while the service is up: the ubuntu / windows legs of the same run pass + # and the same macOS cells pass before and after the window. A pre-flight + # wait cannot cover a mid-job window and the failing processes are the + # real package managers, not the CLI, so the job takes the runner's + # resolver out of the path instead. Every request still goes to the + # production service over TLS verified for the hostname. + # scripts/pin-socket-hosts.py documents the resolution and verification. + - name: Pin hosts + if: runner.os == 'macOS' + shell: bash + env: + PIN_HOSTS: ${{ inputs.hosts }} + run: | + set -euo pipefail + # shellcheck disable=SC2086 # PIN_HOSTS is a space-separated list + lines=$(python3 "$GITHUB_WORKSPACE/scripts/pin-socket-hosts.py" $PIN_HOSTS) + printf '%s\n' "$lines" + printf '\n# pinned by .github/actions/pin-socket-hosts\n%s\n' "$lines" | sudo tee -a /etc/hosts >/dev/null + sudo dscacheutil -flushcache + sudo killall -HUP mDNSResponder || true + for host in $PIN_HOSTS; do + got=$(python3 -c 'import socket, sys; print(" ".join(sorted({i[4][0] for i in socket.getaddrinfo(sys.argv[1], 443)})))' "$host" || true) + echo "$host now resolves to: ${got:-nothing}" + if [ -z "$got" ] || ! grep -qE "^(${got// /|}) $host\$" <<<"$lines"; then + echo "::warning::$host does not resolve to its pinned address after pinning (got: ${got:-nothing})" + fi + done diff --git a/.github/workflows/bun-compatibility.yml b/.github/workflows/bun-compatibility.yml index 3f6b325f3..9894fe3b4 100644 --- a/.github/workflows/bun-compatibility.yml +++ b/.github/workflows/bun-compatibility.yml @@ -17,6 +17,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/bun-compatibility.yml' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' @@ -57,6 +59,8 @@ on: branches: [main] paths: - '.github/workflows/bun-compatibility.yml' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' - 'scripts/bun-historical-shas.json' @@ -187,6 +191,11 @@ jobs: with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts + - name: Download Bun ${{ matrix.bun }} id: bun # Pre-populate the exact directory layout the script's install_tool() diff --git a/.github/workflows/poetry-compatibility.yml b/.github/workflows/poetry-compatibility.yml index c4b07934d..585e83786 100644 --- a/.github/workflows/poetry-compatibility.yml +++ b/.github/workflows/poetry-compatibility.yml @@ -15,6 +15,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/poetry-compatibility.yml' - 'scripts/backtest-poetry.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' @@ -29,6 +31,8 @@ on: branches: [main] paths: - 'scripts/backtest-poetry.py' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' - 'crates/socket-patch-core/src/patch/redirect/**' - 'crates/socket-patch-core/src/vendor/pypi*.rs' @@ -91,6 +95,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts # uv bootstraps every pinned Poetry release (and its interpreter) # itself; pinning uv keeps the bootstrap reproducible. - run: python -m pip install uv==0.11.19 diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 284164b71..4eb555a91 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -21,6 +21,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -58,6 +60,8 @@ on: branches: [main] paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -390,6 +394,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts - name: Backtest against production # Every hosted cell probes the artifact first; it records # blocked-by-server-encoding only when that probe saw a non-identity diff --git a/docs/testing/bun-compatibility.md b/docs/testing/bun-compatibility.md index 53ad0febd..eea92e67c 100644 --- a/docs/testing/bun-compatibility.md +++ b/docs/testing/bun-compatibility.md @@ -345,6 +345,18 @@ matrix to six concurrent jobs, with three cells per job. Each cell has its own temporary directory so historical Bun processes cannot collide while extracting identically named packages. +On macOS the job first runs `.github/actions/pin-socket-hosts` +(`scripts/pin-socket-hosts.py`): the hosted macOS resolver intermittently +answers `patch.socket.dev` with EAI_NONAME for minutes at a time, at job start +or mid-job, while the service is up, which failed every hosted cell in the +window (`FailedToOpenSocket`, `[Errno 8] nodename nor servname provided`, or a +Bun 1.3.x workspace install that never exits). The action resolves the patch +hosts once (the system resolver, then DNS-over-HTTPS by IP literal), keeps only +addresses whose TLS handshake verifies the hostname, and pins them in +`/etc/hosts`, so cells still reach the production service over verified TLS +without depending on the runner's resolver. The vlt and Poetry workflows run +the same step. + **Pinned versions:** 0.8.1, 1.0.0, 1.0.36, 1.1.0, 1.1.38 (binary lock), 1.1.39 (first text lock, version 0), 1.1.43 (first `--lockfile-only`), 1.1.45 (last version-0 writer), 1.2.0, 1.2.23, 1.3.0 (version 1), 1.3.9 / 1.3.10 diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index 3129c0d32..1a162d629 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -74,7 +74,10 @@ asserted and each named test or row exists. releases, the Node floors 22.22.0 / 22.13.0 / 22.7.0 / 22.0.0 with the collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm` cache root); `native` (the backtest against production, artifacts - `vlt-results--` in depscan's capture `result.json` shape); + `vlt-results--` in depscan's capture `result.json` shape; on macOS + it first pins the TLS-verified patch hosts in `/etc/hosts` through + `.github/actions/pin-socket-hosts`, because the hosted macOS resolver + intermittently loses `patch.socket.dev` for minutes while the service is up); `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux, macOS and Windows); `matrix-coverage` (every era × suite × OS). - **Nightly:** `canary` runs every capstone on `vlt@latest` on 3 OS (only the diff --git a/scripts/pin-socket-hosts.py b/scripts/pin-socket-hosts.py new file mode 100644 index 000000000..df36bb1ea --- /dev/null +++ b/scripts/pin-socket-hosts.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Pin the production Socket patch hosts in the hosts file of a CI runner. + +The compatibility workflows drive REAL package managers (bun, vlt, poetry, +...) against the production patch service. On GitHub's hosted macOS runners +the system resolver intermittently answers `patch.socket.dev` with +EAI_NONAME ("[Errno 8] nodename nor servname provided, or not known"; +bun: `FailedToOpenSocket`) for minutes at a time, starting at job start or +mid-job, while the service itself is up (the ubuntu and windows legs of the +same run pass, and the same macOS cells pass before and after the window). +The runner's resolver is not under test, so the workflow takes it out of the +path: this script resolves each host once, verifies every address, and +prints `hosts(5)` lines the workflow appends to /etc/hosts. + +Resolution tries the system resolver first, then DNS-over-HTTPS to IP-literal +endpoints (no DNS needed to reach them), retrying with backoff inside a +bounded window. An address is only pinned after a TLS handshake to it with +SNI = the host verifies the host's certificate, so a pinned address is one +that really serves that name; the package managers still verify TLS for the +hostname on every request. Both families are resolved; an IPv6 address is +pinned only when it verifies too, so a runner without an IPv6 route never gets +an unreachable entry. + +Exit status is non-zero, with nothing printed, when a host cannot be pinned +within the window: the job then fails at this step, naming the host, rather +than in a hundred cells downstream. +""" + +import argparse +import ipaddress +import json +import socket +import ssl +import sys +import time +import urllib.request + +DEFAULT_HOSTS = ['patch.socket.dev', 'patches-api.socket.dev'] +# DoH JSON endpoints reached by IP literal; both serve certificates with the +# IP in the subjectAltName, so they verify without any name resolution. +DOH_ENDPOINTS = [ + 'https://1.1.1.1/dns-query?name={host}&type={rrtype}', + 'https://8.8.8.8/resolve?name={host}&type={rrtype}', +] +RR_TYPES = {'A': 1, 'AAAA': 28} + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def system_resolve(host): + infos = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + return [info[4][0] for info in infos] + + +def doh_resolve(host, template, timeout): + addresses = [] + for rrtype, code in RR_TYPES.items(): + request = urllib.request.Request(template.format(host=host, rrtype=rrtype), + headers={'accept': 'application/dns-json'}) + with urllib.request.urlopen(request, timeout=timeout) as response: + answer = json.loads(response.read()).get('Answer') or [] + # CNAME answers (type 5) precede the address records and are skipped. + addresses += [record['data'] for record in answer if record.get('type') == code] + return addresses + + +def verified(host, address, timeout): + """A TLS handshake to `address` with SNI `host` verifies `host`'s cert.""" + context = ssl.create_default_context() + try: + with socket.create_connection((address, 443), timeout=timeout) as raw: + with context.wrap_socket(raw, server_hostname=host): + return True + except (OSError, ssl.SSLError) as error: + log(f'{host}: {address} failed verification: {error}') + return False + + +def resolve(host, window, timeout): + """Verified addresses of `host` (IPv4 first), or [] once `window` seconds pass.""" + sources = [('system resolver', lambda: system_resolve(host))] + sources += [(template.split('/')[2], lambda t=template: doh_resolve(host, t, timeout)) + for template in DOH_ENDPOINTS] + deadline = time.monotonic() + window + attempt = 0 + fallback = [] + while True: + attempt += 1 + for name, source in sources: + try: + candidates = source() + except Exception as error: # noqa: BLE001 - every source is best effort + log(f'{host}: {name} attempt {attempt} failed: {error}') + continue + addresses = [] + for candidate in dict.fromkeys(candidates): + try: + ipaddress.ip_address(candidate) + except ValueError: + continue + if verified(host, candidate, timeout): + addresses.append(candidate) + addresses.sort(key=lambda a: ipaddress.ip_address(a).version) + # A source that only verified IPv6 is not enough on its own: try + # the next one for an IPv4 address before settling for it. + if any(ipaddress.ip_address(a).version == 4 for a in addresses): + log(f'{host}: pinned {" ".join(addresses)} (from {name}, attempt {attempt})') + return addresses + log(f'{host}: {name} attempt {attempt} gave no verified IPv4 address') + fallback = fallback or addresses + remaining = deadline - time.monotonic() + if remaining <= 0: + return fallback + time.sleep(min(5 * attempt, 30, remaining)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('hosts', nargs='*', default=DEFAULT_HOSTS) + parser.add_argument('--window', type=float, default=300, + help='seconds to keep retrying a host before failing (default 300)') + parser.add_argument('--timeout', type=float, default=10, + help='per-request timeout in seconds (default 10)') + args = parser.parse_args(argv) + lines = [] + for host in args.hosts: + addresses = resolve(host, args.window, args.timeout) + if not addresses: + log(f'::error::could not resolve and verify {host} within {args.window:.0f} s') + return 1 + lines += [f'{address} {host}' for address in addresses] + print('\n'.join(lines)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/tests/test_pin_socket_hosts.py b/scripts/tests/test_pin_socket_hosts.py new file mode 100644 index 000000000..56094000b --- /dev/null +++ b/scripts/tests/test_pin_socket_hosts.py @@ -0,0 +1,76 @@ +"""Hermetic coverage for scripts/pin-socket-hosts.py's resolution fallbacks.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import socket +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location( + 'pin_socket_hosts', Path(__file__).resolve().parents[1] / 'pin-socket-hosts.py') +pin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pin) + +HOST = 'patch.socket.dev' +EAI_NONAME = socket.gaierror(8, 'nodename nor servname provided, or not known') + + +def run(fn): + with contextlib.redirect_stderr(io.StringIO()): + return fn() + + +class PinSocketHostsTests(unittest.TestCase): + def test_system_resolver_answer_is_pinned_when_it_verifies(self): + with patch.object(pin, 'system_resolve', return_value=['172.66.3.58', '172.66.3.58']), \ + patch.object(pin, 'doh_resolve') as doh, \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['172.66.3.58']) + doh.assert_not_called() + + def test_doh_takes_over_when_the_system_resolver_fails(self): + with patch.object(pin, 'system_resolve', side_effect=EAI_NONAME), \ + patch.object(pin, 'doh_resolve', return_value=['2606:4700:7::32d', '162.159.143.62']), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), + ['162.159.143.62', '2606:4700:7::32d']) + + def test_unverified_addresses_are_never_pinned(self): + with patch.object(pin, 'system_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'doh_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'verified', return_value=False): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), []) + + def test_ipv6_only_is_a_last_resort(self): + with patch.object(pin, 'system_resolve', return_value=['2606:4700:7::32d']), \ + patch.object(pin, 'doh_resolve', return_value=[]), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['2606:4700:7::32d']) + + def test_retries_until_the_resolver_recovers(self): + answers = [EAI_NONAME, ['172.66.3.58']] + with patch.object(pin, 'system_resolve', side_effect=answers), \ + patch.object(pin, 'doh_resolve', side_effect=OSError('no route')), \ + patch.object(pin, 'verified', return_value=True), \ + patch.object(pin.time, 'sleep') as sleep: + self.assertEqual(run(lambda: pin.resolve(HOST, 60, 1)), ['172.66.3.58']) + sleep.assert_called_once() + + def test_main_prints_hosts_lines_and_fails_closed(self): + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=['172.66.3.58']), \ + contextlib.redirect_stdout(out): + self.assertEqual(pin.main([HOST]), 0) + self.assertEqual(out.getvalue(), f'172.66.3.58 {HOST}\n') + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=[]), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(pin.main([HOST]), 1) + self.assertEqual(out.getvalue(), '') + + +if __name__ == '__main__': + unittest.main() From b06c36c15dc2f23e774f5f89d045c896067b03ff Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 30 Sep 2026 09:36:11 +0200 Subject: [PATCH 2/3] ci: fix historical Yarn and vlt upstream restore expectations --- .../tests/e2e_redirect_vlt_build.rs | 44 ++++++---- .../tests/mode_migration_npm.rs | 81 ++++++++++++++----- docs/testing/vlt-compatibility.md | 7 ++ scripts/yarn-classic-vex-matrix.sh | 2 + 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs b/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs index 55a4708ac..3807f9e8f 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs @@ -29,6 +29,24 @@ fn hosted_leg(name: &'static str) -> Option { Leg::start(SUITE, name) } +/// v5 restores upstream pins without a saved lock fragment. The earliest +/// vlt releases record npmjs URLs even with the harness registry configured; +/// restore may omit that redundant slot or point it at the harness registry. +/// All other bytes, including bystanders and line endings, must still match. +fn assert_restored_lock(fx: &Fixture, before: &[u8]) { + let mut expected = String::from_utf8(before.to_vec()).unwrap(); + let mut actual = String::from_utf8(lock_bytes(&fx.proj)).unwrap(); + if fx.leg.version() <= VltVersion::zero(11) { + for target in &fx.svc.targets { + let bare = target.name.rsplit('/').next().unwrap(); + let path = Registry::tarball_path(&target.name, bare, &target.version); + expected = expected.replace(&format!(",\"https://registry.npmjs.org{path}\""), ""); + actual = actual.replace(&format!(",\"{}{path}\"", fx.reg.server.uri()), ""); + } + } + assert_eq!(actual, expected, "rollback restores the upstream lock"); +} + // ── drivers and fresh checkouts ─────────────────────────────────────────── /// `scan --mode hosted --vex`: the lock pins the artifact (one preflight @@ -167,7 +185,7 @@ async fn vlt_pinned_matrix_hosted_tamper_cold_eintegrity() { // ── rollback, rerun, heal ───────────────────────────────────────────────── -/// Rollback restores the lock byte-for-byte, heals the patched store copy +/// Rollback restores the upstream lock, heals the patched store copy /// (and nothing else), and the next `vlt install` is pristine. #[tokio::test(flavor = "multi_thread")] #[ignore = "real vlt: SOCKET_PATCH_VLT_E2E_JS"] @@ -184,11 +202,7 @@ async fn vlt_pinned_matrix_hosted_rollback_byte_exact() { let out = fx.rollback(&[]); assert_eq!(out.code, 0, "{out}"); let doc = out.json(); - assert_eq!( - String::from_utf8_lossy(&lock_bytes(&fx.proj)), - String::from_utf8_lossy(&fx.lock_before), - "rollback restores vlt-lock.json byte-for-byte" - ); + assert_restored_lock(&fx, &fx.lock_before); assert!( fx.ledger().is_none(), "no hosted ledger is ever written (v5)" @@ -762,11 +776,10 @@ async fn vlt_pinned_matrix_hosted_crlf_lock() { assert_eq!(state(&co, fx.t()), State::Patched); let out = fx.rollback(&[]); assert_eq!(out.code, 0, "{out}"); - assert_eq!( - lock_bytes(&fx.proj), - crlf, - "rollback restores the CRLF lock" - ); + assert_restored_lock(&fx, &crlf); + let co = fx.checkout("restored-crlf"); + fx.vlt_ok_profile(&co, &fx.leg.locked_install_args(), "restored-crlf"); + assert_eq!(state(&co, fx.t()), State::Pristine); fx.leg.ran(); } @@ -1185,11 +1198,7 @@ async fn vlt_pinned_matrix_hosted_idempotence() { assert!(fx.ledger().is_none()); let out = fx.rollback(&[]); assert_eq!(out.code, 0, "{out}"); - assert_eq!( - String::from_utf8_lossy(&lock_bytes(&fx.proj)), - String::from_utf8_lossy(&fx.lock_before), - "rollback restores the registry lock byte-for-byte: {out}" - ); + assert_restored_lock(&fx, &fx.lock_before); assert!(fx.ledger().is_none()); let files = package_files(&fx.proj); let out = fx.rollback(&[]); @@ -1199,6 +1208,9 @@ async fn vlt_pinned_matrix_hosted_idempotence() { "a second rollback finds no state: {out}" ); assert_eq!(package_files(&fx.proj), files, "and writes nothing"); + let co = fx.checkout("restored-idempotence"); + fx.vlt_ok_profile(&co, &fx.leg.locked_install_args(), "restored-idempotence"); + assert_eq!(state(&co, fx.t()), State::Pristine); fx.leg.ran(); } diff --git a/crates/socket-patch-cli/tests/mode_migration_npm.rs b/crates/socket-patch-cli/tests/mode_migration_npm.rs index f734f77af..7e7a3b718 100644 --- a/crates/socket-patch-cli/tests/mode_migration_npm.rs +++ b/crates/socket-patch-cli/tests/mode_migration_npm.rs @@ -388,11 +388,15 @@ async fn mount_hosted_mocks( /// Serve, from `server` (as `SOCKET_NPM_REGISTRY`), the npm registry version /// document the v5 upstream restore reads for DEP — mirrored from what the /// PRISTINE classic lock recorded (`resolved "#"`, -/// `integrity`). The restore of a hosted classic entry must reproduce the -/// registry entry yarn wrote from exactly that document; mirroring it keeps +/// `integrity`, or the SHA-1 fragment on pre-1.10 releases). The restore must +/// reproduce the registry entry yarn wrote from that document; mirroring it keeps /// the unwind hermetic (the binary's TLS stack need not reach the real -/// registry). Returns the registry base to hand the binary. -async fn mount_registry_from_classic_lock(server: &MockServer, lock: &str) -> String { +/// registry). Returns the registry base and expected upstream lock. Hosted +/// mode adds an integrity line even on pre-1.10 yarn, and v5 restores that +/// line's registry hash without a saved fragment to recover its absence. +async fn mount_registry_from_classic_lock(server: &MockServer, lock: &str) -> (String, String) { + use base64::Engine as _; + let block = lock .split("\n\n") .find(|b| { @@ -404,23 +408,43 @@ async fn mount_registry_from_classic_lock(server: &MockServer, lock: &str) -> St .lines() .find_map(|l| l.trim().strip_prefix(&format!("{name} "))) .map(|v| v.trim_matches('"').to_string()) - .unwrap_or_else(|| panic!("no `{name}` in {block}")) }; - let resolved = field("resolved"); + let resolved = field("resolved").unwrap_or_else(|| panic!("no `resolved` in {block}")); let (tarball, shasum) = resolved .split_once('#') .map(|(t, s)| (t.to_string(), Some(s.to_string()))) .unwrap_or((resolved.clone(), None)); + let integrity = field("integrity").unwrap_or_else(|| { + let sha1 = hex::decode( + shasum + .as_ref() + .expect("pre-1.10 yarn pins a SHA-1 fragment"), + ) + .expect("the resolved fragment is hex SHA-1"); + format!( + "sha1-{}", + base64::engine::general_purpose::STANDARD.encode(sha1) + ) + }); + let upstream_lock = if field("integrity").is_some() { + lock.to_string() + } else { + lock.replacen( + &format!(" resolved \"{resolved}\""), + &format!(" resolved \"{resolved}\"\n integrity {integrity}"), + 1, + ) + }; Mock::given(method("GET")) .and(path(format!("/registry/{DEP}/{DEP_VERSION}"))) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "name": DEP, "version": DEP_VERSION, - "dist": { "tarball": tarball, "integrity": field("integrity"), "shasum": shasum } + "dist": { "tarball": tarball, "integrity": integrity, "shasum": shasum } }))) .mount(server) .await; - format!("{}/registry", server.uri()) + (format!("{}/registry", server.uri()), upstream_lock) } fn run_hosted_scan(proj: &Path, server_uri: &str) -> (i32, String, String) { @@ -642,7 +666,7 @@ fn assert_pure_vendored_and_round_trip( "fresh vendored install must carry the PATCHED bytes ({tag})" ); - // (b) Round trip: `vendor --revert` restores the REGISTRY lock + // (b) Round trip: `vendor --revert` restores the expected REGISTRY lock // byte-identically (pre-fix it restored the hosted fragment, with no CLI // path back to registry state). let (code, stdout, stderr) = run_socket( @@ -659,8 +683,8 @@ fn assert_pure_vendored_and_round_trip( assert_eq!( std::fs::read(proj.join("yarn.lock")).unwrap(), lock_pristine, - "yarn.lock must be restored byte-identical to the pre-hosted \ - REGISTRY pristine ({tag}); got:\n{}", + "yarn.lock must be restored byte-identical to the expected \ + upstream REGISTRY lock ({tag}); got:\n{}", read(proj, "yarn.lock") ); assert_eq!( @@ -809,7 +833,7 @@ async fn classic_hosted_then_vendored_takeover_round_trips_to_registry() { // upstream restore re-resolves the registry entry (mirrored from the // pristine lock), and the mock origin is named hosted via // --patch-server-url. - let registry = + let (registry, lock_upstream) = mount_registry_from_classic_lock(&server, &String::from_utf8_lossy(&lock_pristine)).await; stage_patch(&proj, &fx.orig, &fx.patched); let (code, stdout, stderr) = run_socket_env( @@ -847,7 +871,7 @@ async fn classic_hosted_then_vendored_takeover_round_trips_to_registry() { "classic", false, &hosted_url, - &lock_pristine, + lock_upstream.as_bytes(), &pkg_json_pristine, &stdout, ); @@ -1055,10 +1079,10 @@ async fn classic_vendored_then_hosted_takeover_leaves_pure_hosted() { ) }); - // The originals chain across migrations: `rollback` restores the hosted - // pin's upstream registry entry, which is the pristine lock byte for - // byte (online: the entry is re-resolved from the registry document). - let registry = + // Rollback re-resolves the upstream registry entry. Hosted mode added an + // integrity line even on pre-1.10 yarn; v5 has no saved fragment to tell + // whether it was originally absent, so it restores the registry hash. + let (registry, lock_upstream) = mount_registry_from_classic_lock(&server, &String::from_utf8_lossy(&lock_pristine)).await; let (code, stdout, stderr) = run_socket_env( &proj, @@ -1076,8 +1100,27 @@ async fn classic_vendored_then_hosted_takeover_leaves_pure_hosted() { assert_eq!(code, 0, "rollback failed: {stdout}\n{stderr}"); assert_eq!( read(&proj, "yarn.lock"), - String::from_utf8_lossy(&lock_pristine), - "rollback lands on the pristine registry lock" + lock_upstream, + "rollback restores the registry lock, allowing the added upstream integrity" + ); + let fresh = fresh_checkout(&proj, fx.tmp.path(), "classic-rollback", false); + let fresh_cache = fx.tmp.path().join("fresh-cache-classic-rollback"); + let ci = corepack( + &fresh, + &yarn_classic_vex::yarn_classic(), + &["install", "--frozen-lockfile", "--no-progress"], + &[("YARN_CACHE_FOLDER", fresh_cache.to_str().unwrap())], + ); + assert!( + ci.status.success(), + "fresh-checkout rollback install must succeed.\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&ci.stdout), + String::from_utf8_lossy(&ci.stderr), + ); + assert_eq!( + std::fs::read(fresh.join("node_modules").join(DEP).join("index.js")).unwrap(), + fx.orig, + "rollback installs the pristine registry bytes" ); } diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index 4bb2dbe91..4fd506694 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -134,6 +134,13 @@ The capstones serve npmjs bytes from a local wiremock registry `R` and write | rc.30 … rc.32 | `{"config":{"registry": R}}` | yes | URL-segment DepIDs | | ≥ rc.33 | `{"config":{"registries":{"npm": R}}}` (+ `config.registry = R` for rc.33 … 1.0.4) | yes | | +The 0.0.0-1 and 0.0.0-11 writers can record an explicit npmjs tarball URL +despite the configured harness registry. v5 rollback reconstructs the upstream +pin without a saved lock fragment, so the rollback assertions allow that target +URL to be omitted or restored on the harness registry. They still compare every +other byte, including bystanders and CRLF line endings, and verify pristine +package contents after a real install from the restored lock. + `scripts/backtest-vlt.py`'s `write_vlt_json` follows the same table against public npm (a `registry` equal to vlt's npmjs default is left out: vlt strips it from the lock anyway). diff --git a/scripts/yarn-classic-vex-matrix.sh b/scripts/yarn-classic-vex-matrix.sh index 5b0ffc9d2..ea509ba05 100755 --- a/scripts/yarn-classic-vex-matrix.sh +++ b/scripts/yarn-classic-vex-matrix.sh @@ -41,11 +41,13 @@ for release in "${releases[@]}"; do for entry in "${suites[@]}"; do IFS=: read -r suite filter ignored <<<"$entry" echo "== yarn@$release $suite ${filter:-}" >&2 + start_line=$(wc -l < "$log") if ! (cd "$root" && SOCKET_PATCH_YARN_CLASSIC_E2E_VERSION="$release" \ SOCKET_PATCH_YARN_E2E_REQUIRED=1 \ cargo test -q -p socket-patch-cli --test "$suite" -- ${filter:+"$filter"} \ ${ignored:+"$ignored"} --nocapture --test-threads=1 >>"$log" 2>&1); then echo "FAIL yarn@$release $suite (log: $log)" >&2 + tail -n "+$((start_line + 1))" "$log" >&2 echo "VEXCELL leg=$suite yarn=$release mode=- cell=SUITE FAIL" >>"$log" status=1 fi From 9332f3bcccd749a5ae6c8b28cd221318fbf32648 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 30 Sep 2026 09:46:10 +0200 Subject: [PATCH 3/3] ci: retain JSON CLI failures for vlt transport retries --- scripts/backtest-vlt.py | 11 ++++++++++- scripts/tests/test_backtest_harnesses.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/backtest-vlt.py b/scripts/backtest-vlt.py index ed8b54367..f9e565ae9 100644 --- a/scripts/backtest-vlt.py +++ b/scripts/backtest-vlt.py @@ -992,13 +992,20 @@ def __init__(self, ctx, version, mode, shape_name): self.record = ctx['record'] self.envelopes = [] self.fresh_patched = {} + self.cli_failures = [] # CLI ------------------------------------------------------------------- def cli(self, args, cwd=None): command = [self.ctx['cli'], *args, '--json', '--no-telemetry'] if self.ctx.get('patch_server_url'): command += ['--patch-server-url', self.ctx['patch_server_url']] - return run(command, cwd or self.project, self.ctx['cli_env'], self.log) + code, out, err = run(command, cwd or self.project, self.ctx['cli_env'], self.log) + if code: + # --json errors go to stdout, including on repeat/revert calls. + # Keep them on the result row so the transport retry sees them. + self.cli_failures.append(dict(command=str(args[0]), exitCode=code, + envelope=parse_envelope(out), stderr=tail(err, 3000))) + return code, out, err def patch_run(self, mode, cwd=None): cwd = cwd or self.project @@ -1195,6 +1202,8 @@ def blocked_refusal(self, row, envelopes, reference): return clean def finish(self, row, checks, started): + if self.cli_failures: + row['cliFailures'] = self.cli_failures row['failingChecks'] = [k for k, v in checks.items() if v is False] row['notEvaluated'] = [k for k, v in checks.items() if v is None] row.setdefault('codes', []) diff --git a/scripts/tests/test_backtest_harnesses.py b/scripts/tests/test_backtest_harnesses.py index b5b7dc07a..5b393228d 100644 --- a/scripts/tests/test_backtest_harnesses.py +++ b/scripts/tests/test_backtest_harnesses.py @@ -608,6 +608,29 @@ def test_snapshot_never_follows_a_link(self): class VltRetryTests(unittest.TestCase): + def test_json_cli_failures_reach_the_retry_classifier(self): + for phase in ('initial', 'repeat', 'rollback'): + for status in (504, 404): + with self.subTest(phase=phase, status=status), tempfile.TemporaryDirectory() as temp: + cell = vlt.Cell({'out': Path(temp), 'record': {}, 'cli': 'socket-patch', + 'cli_env': {}}, '1.2.0', 'hosted', 'direct') + cell.project.mkdir(parents=True) + row = dict(cell=cell.name, expectedVerdict='patched', passed=False, checks={}) + envelope = {'status': 'error', + 'error': f'API request failed with status {status}: error code: {status}'} + with patch.object(vlt, 'run', return_value=(1, json.dumps(envelope), '')): + if phase == 'initial': + cell.patch_run('hosted') + elif phase == 'repeat': + cell.repeat(row, row['checks'], b'') + else: + cell.revert() + with patch('sys.stdout'): + cell.finish(row, row['checks'], vlt.time.time()) + captured = json.loads((cell.case / 'result.json').read_text()) + self.assertEqual(captured['cliFailures'][0]['envelope'], envelope) + self.assertEqual(vlt.transient(captured), status == 504) + def test_only_transport_failures_retry(self): self.assertFalse(vlt.transient({'matchesExpectation': True, 'serveProbe': {'curlExit': 7}})) self.assertTrue(vlt.transient({'serveProbe': {'curlExit': 7}}))