fix(installer-tests): bound apt's own sockets so a stalled mirror fails in seconds, not 12 minutes

scripts/tests/_pm.sh

@@ -0,0 +1,88 @@
+# shellcheck shell=bash
+# =============================================================================
+#  _pm.sh — the bounded package-manager runner, shared by the container harnesses
+# -----------------------------------------------------------------------------
+#  Sourced by scripts/tests/distro-prereqs.sh and scripts/tests/path-persist.sh.
+#  Both run INSIDE a plain distro container with the repo mounted at /src, so a
+#  sibling source is just a local file read — it needs no network and no tools
+#  beyond the shell that is already running.
+#
+#  WHY THIS FILE EXISTS. The two harnesses each carried their own copy of
+#  `_pm_run`, identical in logic and already drifted in prose. When the apt
+#  socket bounds below were added to fix a 12-minute silent hang, they landed on
+#  ONE copy — so `Prereqs` was fixed and `PATH persist`, which fails the same
+#  way, was not. Bugbot caught that inside the very PR that introduced it
+#  (client#1051). One copy, sourced twice, is the only version of this that
+#  cannot half-ship.
+#
+#  What is NOT here: each harness's own `_pm_install_one`. Those genuinely
+#  differ — path-persist supports apk, distro-prereqs does not — and folding
+#  them together would invent a package-manager matrix neither one tests.
+# =============================================================================
+
+# APT NEEDS ITS OWN SOCKET BOUND, not just the external one _pm_run applies.
+#
+# `timeout 60` around apt kills a stalled fetch, but apt never learns anything:
+# it emits NO output, retries nothing, and the next attempt stalls identically.
+# The observed failure is three attempts of pure silence — no `Err:`, no `W:`,
+# no `E:` — then the job's outer bound killing the container with exit 137. A
+# refused connection errors instantly; only a stalled one hangs like that, and
+# apt's default socket timeout outlasts the external kill every time. So the
+# bound was in the wrong place, not missing.
+#
+# WHY NOT THE FIX tracebloc-engine#1029 USED. That change found the specific
+# cause on the RUNNER — the mirrorlist lists the archive over http first and
+# https second, http stopped answering on 2026-09-11, and apt walked 52 index
+# URLs before falling back — and fixed it by rewriting the mirrorlist to https.
+# THAT MUST NOT BE COPIED HERE. These harnesses run inside a BARE distro
+# container, and ubuntu:24.04 ships no ca-certificates: rewriting its sources to
+# https makes every index fetch fail certificate verification. Measured, in the
+# image the job actually uses:
+#
+#     W: Failed to fetch https://…/InRelease  Certificate verification failed:
+#        The certificate is NOT trusted. The certificate issuer is unknown.
+#     apt-get update  -> exit 0   (warnings only — it "succeeds" fetching nothing)
+#     apt-get install -> exit 100, the package is not installed
+#
+# An update that exits 0 having fetched nothing is worse than the hang, because
+# the failure moves to whatever needed the package. So https is not available to
+# us until something installs ca-certificates, which needs apt, which is the
+# circle. The bounds below are what IS available.
+#
+# They are adequate here in a way they were not on the runner. #1029 measured
+# Timeout/Retries alone still stalling 29 minutes, but that was ~52 index URLs;
+# a bare container lists four suites, so a total stall costs minutes and ends in
+# an honest error rather than a 12-minute silent kill. Bounded to something
+# useful, at this scale.
+#
+# apt-only, deliberately: dnf/yum/zypper/apk/pacman take none of these flags and
+# would fail on an unknown option — turning a mirror stall on one distro into a
+# hard argument error on five.
+#
+# NOTE `Acquire::http::Timeout` does not govern https connections (#1029), hence
+# both. No ForceIPv4: an earlier version of this carried it on the theory that a
+# blackholed AAAA caused the stall. #1029 then measured the real cause on the
+# runner to be the SCHEME, not the address family. The flag was harmless but its
+# stated reason was wrong, and a flag shipped on a contradicted hypothesis is
+# the kind of thing that gets copied forward as fact.
+_APT_BOUND='-o Acquire::http::Timeout=10 -o Acquire::https::Timeout=10 -o Acquire::Retries=3'
+
+# Bounded + retried package-manager invocation; "$@" = the PM argv.
+#
+# `command -v` (not has()) because this runs BEFORE common.sh is sourced;
+# notices go to stderr.
+_pm_run() {
+  local i
+  for i in 1 2 3; do
+    if   command -v timeout  >/dev/null 2>&1; then timeout  "${TB_PM_TIMEOUT:-60}" "$@" && return 0
+    elif command -v gtimeout >/dev/null 2>&1; then gtimeout "${TB_PM_TIMEOUT:-60}" "$@" && return 0
+    else "$@" && return 0; fi
+    echo "::warning::package-manager step stalled or failed (attempt $i/3): $*" >&2
+    # No backoff after the LAST attempt, so a dead mirror fails RED here well
+    # under the job's timeout-minutes rather than running the clock out into a
+    # silent `cancelled` (the failure class of backend#2859).
+    [ "$i" -lt 3 ] && sleep $((i * 5))
+  done
+  echo "::error::package-manager step failed after 3 bounded attempts: $* — the package manager could not reach its mirrors from inside the CI container. This is NOT this PR's diff. Re-running often does NOT help: two consecutive attempts failed identically on 2026-09-11. If apt, the attempts above should carry Acquire::* bounds; if they did not, that is the bug." >&2
+  return 1
+}

scripts/tests/distro-prereqs.sh

@@ -29,37 +29,15 @@ set -uo pipefail
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 LIB="$HERE/../lib"
+# The bounded package-manager runner and apt's socket bounds live in ONE place;
+# both container harnesses source it. See scripts/tests/_pm.sh for why.
+# shellcheck source=scripts/tests/_pm.sh
+. "${BASH_SOURCE[0]%/*}/_pm.sh"
 
-# ── Make the container resemble a real host ──────────────────────────────────
-# Anyone running the real installer reached it via `curl | bash`, so curl always
-# exists and the box has sudo. Minimal base images ship neither — install them
-# up front (we are root here) so the rest of the run mirrors a real machine.
-# These bootstrap installs run in an EPHEMERAL CI container against distro mirrors
-# that occasionally STALL rather than fail. An unbounded package-manager call then
-# hangs until the job's `timeout-minutes`, which GitHub reports as `cancelled` — NOT
-# a red check — so the failure is silent (this is the exact class that bit the
-# sibling path-persist job's opensuse leg on 2026-08-31; backend#2859). Bound each
-# attempt and retry: a transient stall recovers, a dead mirror fails FAST with an
-# honest error. `command -v` (not has()) because this runs BEFORE common.sh is
-# sourced; notices go to stderr. NB: this bounds the harness's own bootstrap only —
-# the real installer functions invoked below use common.sh's bounded probes.
-_pm_run() { # bounded + retried package-manager invocation; $@ = the PM argv
-  local i
-  for i in 1 2 3; do
-    if   command -v timeout  >/dev/null 2>&1; then timeout  "${TB_PM_TIMEOUT:-60}" "$@" && return 0
-    elif command -v gtimeout >/dev/null 2>&1; then gtimeout "${TB_PM_TIMEOUT:-60}" "$@" && return 0
-    else "$@" && return 0; fi
-    echo "::warning::package-manager step stalled or failed (attempt $i/3): $*" >&2
-    # No backoff after the LAST attempt, so a dead mirror fails RED here well under
-    # the job's timeout-minutes rather than running the clock out into a silent
-    # `cancelled` (the failure class of #2859).
-    [ "$i" -lt 3 ] && sleep $((i * 5))
-  done
-  echo "::error::package-manager step failed after 3 bounded attempts: $* — distro-mirror connectivity inside the CI container, not this PR. Re-run this job." >&2
-  return 1
-}
 _pm_install_one() { # install a single package with whatever PM exists
-  if   command -v apt-get >/dev/null 2>&1; then _pm_run apt-get update -qq && _pm_run apt-get install -y -qq "$1"
+  if   command -v apt-get >/dev/null 2>&1; then
+    # shellcheck disable=SC2086  # _APT_BOUND is a deliberate word-split argv
+    _pm_run apt-get update -qq $_APT_BOUND && _pm_run apt-get install -y -qq $_APT_BOUND "$1"
   elif command -v dnf     >/dev/null 2>&1; then _pm_run dnf install -y -q "$1"
   elif command -v yum     >/dev/null 2>&1; then _pm_run yum install -y -q "$1"
   elif command -v zypper  >/dev/null 2>&1; then _pm_run zypper --non-interactive install "$1"

scripts/tests/path-persist.sh

@@ -78,42 +78,14 @@ umask 022
 DEFAULT_CLI_REF="https://github.com/tracebloc/cli/releases/latest/download/install.sh"
 CLI_REF="${TRACEBLOC_CLI_REF:-$DEFAULT_CLI_REF}"
 CLI_VERSION="${TRACEBLOC_CLI_VERSION:-}"
-
-# ── Make the container resemble a real host ──────────────────────────────────
-# A customer reached install.sh via `curl | sh`, so curl always exists. Minimal
-# base images may ship neither curl nor a shell beyond /bin/sh — install what we
-# need so the run mirrors a real machine. We are root in the container.
-# Package installs run in an EPHEMERAL CI container against distro mirrors that
-# occasionally STALL rather than fail. An unbounded `zypper`/`apt`/`dnf` then hangs
-# until the job's `timeout-minutes`, and GitHub reports that as `cancelled` — NOT a
-# red check — so a scheduled run can rot unnoticed for a week (backend#2859:
-# opensuse/leap:15.6 hung the full 20 min on 2026-08-31 while every other leg, and
-# this same script run by hand, went green in ~80s). Same infra-stall class the
-# workflow already bounds one layer out for `docker pull` (#525/#592) — the
-# container's own package installs were the layer left unbounded. Bound each attempt
-# and retry: a transient stall recovers (→ green); a genuinely dead mirror fails
-# FAST with an honest error (→ red), never a silent 20-minute cancel. `command -v`
-# (not has()) so this is safe even before common.sh is sourced; notices go to stderr
-# so a caller's `>/dev/null` can't swallow them.
-_pm_run() { # bounded + retried package-manager invocation; $@ = the PM argv
-  local i
-  for i in 1 2 3; do
-    if   command -v timeout  >/dev/null 2>&1; then timeout  "${TB_PM_TIMEOUT:-60}" "$@" && return 0
-    elif command -v gtimeout >/dev/null 2>&1; then gtimeout "${TB_PM_TIMEOUT:-60}" "$@" && return 0
-    else "$@" && return 0; fi
-    echo "::warning::package-manager step stalled or failed (attempt $i/3): $*" >&2
-    # No backoff after the LAST attempt. Sized so the pathological all-installs-
-    # stall case (a few best-effort installs × 3 bounded attempts) stays well under
-    # the job's 20-min timeout-minutes, so a dead mirror fails RED here — it does
-    # not run the clock out into a silent `cancelled` (the very failure of #2859).
-    [ "$i" -lt 3 ] && sleep $((i * 5))
-  done
-  echo "::error::package-manager step failed after 3 bounded attempts: $* — distro-mirror connectivity inside the CI container, not this PR. Re-run this job." >&2
-  return 1
-}
+# The bounded package-manager runner and apt's socket bounds live in ONE place;
+# both container harnesses source it. See scripts/tests/_pm.sh for why.
+# shellcheck source=scripts/tests/_pm.sh
+. "${BASH_SOURCE[0]%/*}/_pm.sh"
 
 _pm_install() { # install one or more packages with whatever PM exists; best-effort
-  if   command -v apt-get >/dev/null 2>&1; then _pm_run apt-get update -qq && _pm_run apt-get install -y -qq "$@"
+  # shellcheck disable=SC2086  # _APT_BOUND is a deliberate word-split argv
+  if   command -v apt-get >/dev/null 2>&1; then _pm_run apt-get update -qq $_APT_BOUND && _pm_run apt-get install -y -qq $_APT_BOUND "$@"
   elif command -v dnf     >/dev/null 2>&1; then _pm_run dnf install -y -q "$@"
   elif command -v yum     >/dev/null 2>&1; then _pm_run yum install -y -q "$@"
   elif command -v zypper  >/dev/null 2>&1; then _pm_run zypper --non-interactive --quiet install "$@"