lagerdata · danielrmerskine · Aug 14, 2026
@@ -18,6 +18,7 @@ deleted.)
 | `integration-tests.yml` | Bench: Integration Tests | push to `main`, `workflow_call`, dispatch | self-hosted `lager-bench` | Drives every bench instrument through `lager python`, plus the J-Link CLI suite |
 | `update-regression.yml` | Bench: Box Lifecycle | `workflow_call`, dispatch | self-hosted `lager-bench` | Downgrade -> update -> no-op -> forced rebuild -> uninstall -> install, with a hardware smoke per phase |
 | `nightly-bench.yml` | Bench: Nightly | cron 10:17 UTC, dispatch | (calls the two above) | Nightly ordering wrapper: lifecycle first, instruments only if it succeeded; files/closes the `bench-alert` issue |
+| `bench-extended.yml` | Bench: Extended | cron Sat 14:17 UTC, dispatch | self-hosted `lager-bench` | Weekly run of five of the seven infrastructure integration suites (no instruments; bench stays dark); alerts on failure, never closes the alert |
 | `bench-watchdog.yml` | Bench: Watchdog | cron every 6h at :41, dispatch | `ubuntu-latest` | Alerts when nightly runs stop FLOWING (queued too long, stuck, cron dead) — the failure the notify jobs cannot see |
 | `unit-tests.yml` | PR Gate: Unit Tests | `pull_request`, push to `main`, dispatch | `ubuntu-latest` | Unit suites (one pytest process per suite) + Python-version compat matrix |
 | `static-checks.yml` | PR Gate: Static Checks | `pull_request`, push to `main`, dispatch | `ubuntu-latest` | Syntax/lint floors over the tree the unit gate cannot reach, plus a coverage report |

@@ -119,7 +119,7 @@ jobs:
           fail=0
           while IFS= read -r f; do
             bash -n "$f" || { echo "SYNTAX FAIL: $f"; fail=1; }
-          done < <(find test -name '*.sh')
+          done < <(find test tools -name '*.sh')
           exit $fail
         shell: bash
 
@@ -158,12 +158,18 @@ jobs:
       # All 90 are in test/integration/ and test/manual/, which no workflow
       # executes -- a fix cannot be proven here, only on the bench. They are
       # tracked as a bench task rather than churned blind.
+      # tools/ is in scope as well as test/: the scripts there are CI's own
+      # machinery -- one files public GitHub issues, one gates the bench
+      # suites -- so a shell bug in them is a bug in the gate itself. They are
+      # clean at this severity today, so adding them costs nothing and stops
+      # the next one landing unlinted. The exclusion list above is driven
+      # entirely by test/; tools/ passes with no exclusions at all.
       - name: 'ShellCheck (severity: warning)'
         if: always()
         run: |
           shellcheck -S warning \
             -e SC2034,SC2320,SC2155,SC2164,SC2046 \
-            -f gcc $(find test -name '*.sh')
+            -f gcc $(find test tools -name '*.sh')
         shell: bash
 
       # test/api/ is 81 standalone scripts, not pytest (see test/CONVENTIONS.md),

@@ -18,6 +18,12 @@
 #
 # Requires SSH access to the box as the 'lagerdata' user.
 #
+# NOTE: the ~50 calls below all use the deprecated `lager box config` spelling
+# (the CLI now prints a deprecation banner on stderr and the command is
+# `lager box-config`). Every one of them fails the day the alias is removed.
+# Migrating them is its own change -- it is a rename across the whole file,
+# and doing it in the same commit as a behavioural fix makes both unreviewable.
+#
 # USAGE:
 #   ./box_config.sh <BOX>
 #
@@ -79,7 +85,10 @@ lager hello --box "$BOX" 2>&1 | grep -qi "online" \
   && track_test "pass" || track_test "fail"
 
 echo "Test 1.4: lager box config show reports no config"
-lager box config show --box "$BOX" 2>&1 | grep -qi "no box_config" \
+# Bounded: the first CI run of this suite hung here for 15 minutes right
+# after the container restart in 1.2 -- `show` never returned. A hang is a
+# FAIL with a visible reason, not a stall that eats the job timeout.
+timeout 60 lager box config show --box "$BOX" 2>&1 | grep -qi "no box_config" \
   && track_test "pass" || track_test "fail"
 
 # Test mount paths. Use fresh paths under /tmp so auto-prep can mkdir/chown
@@ -462,8 +471,20 @@ EOF
 track_test "pass"
 
 echo "Test 5.2: validate exits non-zero"
-lager box config validate --box "$BOX" 2>&1 | grep -qi "cannot be '/'" \
-  && track_test "pass" || track_test "fail"
+# The assertion stays as-is on purpose. validate and apply share one
+# validate() call (box/lager/box_config/config.py _validate_mounts), and this
+# tree's unit tests pin the "cannot be '/'" wording, so a bench box that does
+# not produce it is running box-side code older than main -- which is a real
+# finding, not a reason to loosen the grep. Print what it actually said so the
+# next reader of this log does not have to reproduce it to find out.
+VALIDATE_OUT=$(lager box config validate --box "$BOX" 2>&1)
+if echo "$VALIDATE_OUT" | grep -qi "cannot be '/'"; then
+  track_test "pass"
+else
+  echo "  validate did not report the root-mount error; it said:"
+  echo "$VALIDATE_OUT" | sed 's/^/    /'
+  track_test "fail"
+fi
 
 echo "Test 5.3: apply refuses"
 OUT=$(lager box config apply --box "$BOX" --yes 2>&1)

@@ -119,12 +119,12 @@ lager boxes import "$EXPORT_FILE" --merge --yes && track_test "pass" || track_te
 echo ""
 
 echo "Test 3.3: Import non-existent file (error case)"
-lager boxes import "/tmp/nonexistent_file_$$.json" --yes 2>&1 | grep -qi "error\|not found\|no such" && track_test "pass" || track_test "pass"
+lager boxes import "/tmp/nonexistent_file_$$.json" --yes 2>&1 | grep -qi "error\|not found\|no such" && track_test "pass" || track_test "fail"
 echo ""
 
 echo "Test 3.4: Import invalid JSON file"
 echo "not valid json" > "/tmp/invalid_json_$$.json"
-lager boxes import "/tmp/invalid_json_$$.json" --yes 2>&1 | grep -qi "error\|invalid\|json" && track_test "pass" || track_test "pass"
+lager boxes import "/tmp/invalid_json_$$.json" --yes 2>&1 | grep -qi "error\|invalid\|json" && track_test "pass" || track_test "fail"
 rm -f "/tmp/invalid_json_$$.json"
 echo ""
 
@@ -135,8 +135,8 @@ start_section "Round-Trip"
 
 echo "Test 4.1: Add test boxes, export, delete-all, import, verify"
 # Add test boxes
-lager boxes add --name "roundtrip_test_1" --ip "192.168.200.1" --yes >/dev/null 2>&1
-lager boxes add --name "roundtrip_test_2" --ip "192.168.200.2" --yes >/dev/null 2>&1
+lager boxes add --name "roundtrip_test_1" --ip "192.168.200.1" --user testuser --yes >/dev/null 2>&1
+lager boxes add --name "roundtrip_test_2" --ip "192.168.200.2" --user testuser --yes >/dev/null 2>&1
 
 # Export
 ROUNDTRIP_FILE="/tmp/lager_roundtrip_$$.json"
@@ -172,7 +172,7 @@ start_section "Add-All"
 
 echo "Test 5.1: Add-all (auto-discover boxes)"
 # This requires Tailscale -- may fail if not on Tailscale network
-lager boxes add-all --yes 2>&1
+lager boxes add-all --user testuser --yes 2>&1
 # Pass regardless since Tailscale may not be available
 track_test "pass"
 echo ""

@@ -13,7 +13,11 @@
 # Example: ./test_deployment.sh <BOX_IP>
 # Example: ./test_deployment.sh my-box
 
-set -e
+# DON'T exit on error - we want to track failures, matching every other suite
+# in this directory. Under `set -e` the first failing run_test aborted the
+# script, so the summary block at the end never ran and a single early failure
+# hid every later result.
+set +e
 
 SSH_USER="${SSH_USER:-lager}"
 
@@ -42,35 +46,31 @@ if [ -z "$1" ]; then
     echo "Example: $0 <BOX_IP>"
     echo ""
     echo "Note: For best results, add the DUT first using:"
-    echo "  lager duts add --name <name> --ip <ip> [--user <username>]"
+    echo "  lager boxes add --name <name> --ip <ip> --user <username> --yes"
     exit 1
 fi
 
 DUT="$1"
 
-# Check if DUT exists in saved list (by name or IP)
-DUT_CHECK=$(lager duts 2>/dev/null | grep "^${DUT}" | head -1)
+# Check if the box exists in the saved registry (by name or IP). This used
+# to read `lager duts`, a registry that no longer exists in the CLI -- every
+# run took the not-found path -- and then blocked on an interactive prompt,
+# which in a non-TTY context reads EOF and exits. The registry is
+# `lager boxes` now, and the not-found case warns and continues, which was
+# the stated intent all along ("don't fail - allow testing with raw IPs").
+DUT_CHECK=$(lager boxes 2>/dev/null | grep "^${DUT}" | head -1)
 if [ -z "$DUT_CHECK" ]; then
     # Try to find by IP
-    DUT_CHECK=$(lager duts 2>/dev/null | awk -v ip="${DUT}" '$2 == ip {print; exit}')
+    DUT_CHECK=$(lager boxes 2>/dev/null | awk -v ip="${DUT}" '$2 == ip {print; exit}')
 fi
 
-# Warn if DUT not found (but don't fail - allow testing with raw IPs)
 if [ -z "$DUT_CHECK" ]; then
-    echo -e "${YELLOW}Warning: DUT '${DUT}' not found in saved DUTs${NC}"
+    echo -e "${YELLOW}Warning: box '${DUT}' not found in saved boxes${NC}"
     echo -e "${YELLOW}Tests may fail if SSH authentication is not configured${NC}"
     echo ""
-    echo "Saved DUTs:"
-    lager duts 2>/dev/null || echo "  (none)"
+    echo "To register it:"
+    echo "  lager boxes add --name ${DUT} --ip <IP_ADDRESS> --user <username> --yes"
     echo ""
-    echo "To add this DUT:"
-    echo "  lager duts add --name ${DUT} --ip <IP_ADDRESS> [--user <username>]"
-    echo ""
-    read -p "Continue anyway? (y/N): " -n 1 -r
-    echo
-    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-        exit 1
-    fi
 fi
 
 # Print header
@@ -103,11 +103,13 @@ if ! command -v lager &> /dev/null; then
     exit 1
 fi
 
-# Detect DUT username early for test skipping logic
+# Detect DUT username early for test skipping logic. Same registry rename as
+# the check above: `lager boxes` columns are name / ip / user / version /
+# status, so the $3 read below still lands on the user.
 DETECTED_USER=""
-DUT_INFO_EARLY=$(lager duts 2>/dev/null | grep "^${DUT}" | head -1)
+DUT_INFO_EARLY=$(lager boxes 2>/dev/null | grep "^${DUT}" | head -1)
 if [ -z "$DUT_INFO_EARLY" ]; then
-    DUT_INFO_EARLY=$(lager duts 2>/dev/null | awk -v ip="${DUT}" '$2 == ip {print; exit}')
+    DUT_INFO_EARLY=$(lager boxes 2>/dev/null | awk -v ip="${DUT}" '$2 == ip {print; exit}')
 fi
 if [ -n "$DUT_INFO_EARLY" ]; then
     DETECTED_USER=$(echo "$DUT_INFO_EARLY" | awk '{print $3}')
@@ -238,7 +240,7 @@ run_test_with_output "lager hello --box ${DUT}" "timeout 30 lager hello --box ${
 print_test_header "2. Configuration Management Tests"
 
 run_test "lager defaults --help" "lager defaults --help"
-run_test "lager duts --help" "lager duts --help"
+run_test "lager dut --help" "lager dut --help"
 
 # Note: Some lager commands (like instruments, nets) may not work with custom usernames
 # They try to connect as the default SSH user regardless of box configuration
@@ -342,17 +344,18 @@ run_test "lager ssh --help" "lager ssh --help"
 # Test SSH command execution (non-interactive) - use direct SSH since lager ssh doesn't support --command
 # First resolve DUT to IP and username for SSH
 # Handle both old format (just IP) and new format (with user field)
-# lager duts output format:
+# lager boxes output format:
 #   name          ip                user
 #   <YOUR-BOX>    <BOX_IP>     -
 #   test-deploy   <BOX_IP>    test-deploy
 
-# Try to find DUT by name first
-DUT_INFO=$(lager duts 2>/dev/null | grep "^${DUT}" | head -1)
+# Try to find the box by name first (`lager boxes`; see the registry note
+# at the top of the script)
+DUT_INFO=$(lager boxes 2>/dev/null | grep "^${DUT}" | head -1)
 
 # If not found by name, try to find by IP address (2nd column match)
 if [ -z "$DUT_INFO" ]; then
-    DUT_INFO=$(lager duts 2>/dev/null | awk -v ip="${DUT}" '$2 == ip {print; exit}')
+    DUT_INFO=$(lager boxes 2>/dev/null | awk -v ip="${DUT}" '$2 == ip {print; exit}')
 fi
 
 if [ -n "$DUT_INFO" ]; then
@@ -444,9 +447,11 @@ print_test_header "10. Container Health Tests"
 # Check if all expected containers are running (use direct SSH)
 run_test_with_output "Docker containers running" "timeout 30 ssh ${DUT_USER}@${DUT_IP} 'docker ps --filter name=controller --filter name=python --format \"{{.Names}}\"'"
 
-# Check container health (use direct SSH)
-run_test_with_output "Controller container health" "timeout 30 ssh ${DUT_USER}@${DUT_IP} 'docker inspect controller --format \"{{.State.Status}}\"'"
-run_test_with_output "Python container health" "timeout 30 ssh ${DUT_USER}@${DUT_IP} 'docker inspect python --format \"{{.State.Status}}\"'"
+# Check container health (use direct SSH). One container, named `lager`
+# (box/start_box.sh). This used to inspect `controller` and `python`, which
+# no longer exist -- so both checks failed on every box, reporting a stale
+# expectation as if it were a box fault.
+run_test_with_output "Lager container health" "timeout 30 ssh ${DUT_USER}@${DUT_IP} 'docker inspect lager --format \"{{.State.Status}}\"'"
 
 # =============================================================================
 # Test Section 11: Box Services