diff --git a/build/usb/includes.chroot/etc/systemd/system/rigos-state.service b/build/usb/includes.chroot/etc/systemd/system/rigos-state.service index 97d891ba..aedfef6c 100644 --- a/build/usb/includes.chroot/etc/systemd/system/rigos-state.service +++ b/build/usb/includes.chroot/etc/systemd/system/rigos-state.service @@ -8,6 +8,7 @@ Before=local-fs.target rigos-state-ready.service Type=oneshot ExecStartPre=/usr/bin/systemd-tmpfiles --create /usr/lib/tmpfiles.d/rigos.conf ExecStart=/usr/local/sbin/rigos-state-orchestrate +TimeoutStartSec=12min RemainAfterExit=yes [Install] diff --git a/build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify b/build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify index ac3dd072..cd1b2057 100644 --- a/build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify +++ b/build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify @@ -42,11 +42,25 @@ def main() -> int: return deny("recovery_status_stale") if status.get("local_console_access") is not True: return deny("local_credential_unavailable") - if status.get("credential_persisted") is not True: - return deny("credential_not_persisted") if status.get("credential_action") not in ("existing", "created", "restored"): return deny("credential_action_invalid") + scope = status.get("credential_scope") + persisted = status.get("credential_persisted") + state_outcome = status.get("state_outcome") + if scope == "persistent": + if persisted is not True: + return deny("persistent_credential_missing") + if state_outcome != "ready": + return deny("persistent_credential_without_ready_state") + elif scope == "boot": + if persisted is not False: + return deny("boot_credential_claims_persistence") + if state_outcome == "ready": + return deny("ready_state_without_persistent_credential") + else: + return deny("credential_scope_invalid") + emit("allowed") return 0 diff --git a/build/usb/includes.chroot/usr/local/sbin/rigos-recovery-access b/build/usb/includes.chroot/usr/local/sbin/rigos-recovery-access index 92a48545..291efe81 100644 --- a/build/usb/includes.chroot/usr/local/sbin/rigos-recovery-access +++ b/build/usb/includes.chroot/usr/local/sbin/rigos-recovery-access @@ -12,6 +12,7 @@ STATE = Path("/var/lib/rigos") CREDENTIAL_DIRECTORY = STATE / "recovery" CREDENTIAL_FILE = CREDENTIAL_DIRECTORY / "rigosadmin-password.hash" BOOT_ID = Path("/proc/sys/kernel/random/boot_id") +STATE_STATUS = RUNTIME / "state-status.json" MAX_HASH_BYTES = 512 MODULAR_CRYPT = re.compile(r"^\$[A-Za-z0-9][A-Za-z0-9./,_=-]*(?:\$[A-Za-z0-9./,_=-]+){2,}$") @@ -52,6 +53,60 @@ def valid_password_hash(value: str) -> bool: return MODULAR_CRYPT.fullmatch(value) is not None +def read_state_status() -> dict: + try: + raw = STATE_STATUS.read_bytes() + if not raw or len(raw) > 64 * 1024: + return {} + value = json.loads(raw) + return value if isinstance(value, dict) else {} + except (OSError, UnicodeError, json.JSONDecodeError): + return {} + + +def persistent_store_ready(status: dict) -> bool: + try: + boot_id = BOOT_ID.read_text(encoding="ascii").strip() + except OSError: + return False + if ( + status.get("schema") != "rigos.state-status/v1" + or status.get("boot_id") != boot_id + or status.get("outcome") != "ready" + or status.get("mountpoint") != str(STATE) + ): + return False + result = subprocess.run( + [ + "/usr/bin/findmnt", + "--json", + "--target", + str(STATE), + "--output", + "TARGET,FSTYPE,OPTIONS", + ], + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + return False + try: + document = json.loads(result.stdout) + filesystems = document.get("filesystems", []) + mount = filesystems[0] if isinstance(filesystems, list) and filesystems else {} + except (TypeError, json.JSONDecodeError): + return False + options = set(str(mount.get("options", "")).split(",")) + required = {"rw", "nosuid", "nodev", "noexec", "noatime"} + return ( + mount.get("target") == str(STATE) + and mount.get("fstype") == "ext4" + and required.issubset(options) + ) + + def ensure_credential_directory() -> None: CREDENTIAL_DIRECTORY.mkdir(mode=0o700, parents=True, exist_ok=True) info = CREDENTIAL_DIRECTORY.lstat() @@ -116,15 +171,25 @@ def restore_password_hash(value: str) -> bool: return result.returncode == 0 and password_ready() -def prompt_for_password(invalid_stored_hash: bool) -> None: - message = ( - "The saved local credential is invalid and will be replaced. Set the local rig " - "administrator password next." - if invalid_stored_hash - else "Set the local rig administrator password next. This enables tty1 diagnostics only." - ) +def prompt_for_password(invalid_stored_hash: bool, persistent: bool) -> None: + if invalid_stored_hash: + message = ( + "The saved local credential is invalid and will be replaced. " + "Set the local rig administrator password next." + ) + elif persistent: + message = ( + "Set the local rig administrator password next. The encrypted password hash " + "will be stored in verified persistent state." + ) + else: + message = ( + "Persistent state is unavailable. Set a temporary rig administrator password " + "for this boot so local and SSH diagnostics remain available. " + "This password is not persistent and may be requested again after reboot." + ) subprocess.run( - ["/usr/bin/whiptail", "--title", "RIGOS RECOVERY ACCESS", "--msgbox", message, "11", "72"], + ["/usr/bin/whiptail", "--title", "RIGOS RECOVERY ACCESS", "--msgbox", message, "12", "76"], check=False, ) subprocess.run(["/usr/bin/passwd", "rigosadmin"], check=True) @@ -162,49 +227,56 @@ def write_status(value: dict) -> None: def main() -> int: - ensure_credential_directory() - stored_hash, stored_hash_invalid = load_persistent_hash() - credential_action = "existing" + state_status = read_state_status() + state_outcome = str(state_status.get("outcome") or "unavailable") + persistent = persistent_store_ready(state_status) + stored_hash = None + stored_hash_invalid = False + if persistent: + ensure_credential_directory() + stored_hash, stored_hash_invalid = load_persistent_hash() + credential_action = "existing" if password_ready(): live_hash = current_password_hash() if live_hash is None: raise RuntimeError("usable local credential hash is unavailable") - if stored_hash != live_hash: + if persistent and stored_hash != live_hash: persist_password_hash(live_hash) - elif stored_hash is not None and restore_password_hash(stored_hash): + elif persistent and stored_hash is not None and restore_password_hash(stored_hash): credential_action = "restored" else: - prompt_for_password(stored_hash_invalid) + prompt_for_password(stored_hash_invalid, persistent) if not password_ready(): raise RuntimeError("local credential setup did not produce a usable password") live_hash = current_password_hash() if live_hash is None: raise RuntimeError("new local credential hash is unavailable") - persist_password_hash(live_hash) + if persistent: + persist_password_hash(live_hash) credential_action = "created" local_access = password_ready() - try: - state_outcome = json.loads((RUNTIME / "state-status.json").read_text(encoding="utf-8")).get("outcome", "unavailable") - except (OSError, json.JSONDecodeError): - state_outcome = "unavailable" remote_active = any(unit_active(name) for name in ("ssh.service", "sshd.service")) remote_enabled = any(unit_enabled(name) for name in ("ssh.service", "sshd.service")) remote_inactive = not remote_active and not remote_enabled - configured = (STATE / "current").is_symlink() + configured = persistent and (STATE / "current").is_symlink() mode = "operational" if state_outcome == "ready" and remote_inactive else "recovery" - write_status({ - "schema": "rigos.recovery-access-status/v1", - "boot_id": BOOT_ID.read_text(encoding="ascii").strip(), - "mode": mode, - "local_console_access": local_access, - "state_outcome": state_outcome, - "remote_access": "inactive" if remote_inactive else "unexpected_active", - "configuration_commit": "present" if configured else "absent", - "credential_action": credential_action, - "credential_persisted": CREDENTIAL_FILE.is_file(), - }) + credential_persisted = persistent and CREDENTIAL_FILE.is_file() + write_status( + { + "schema": "rigos.recovery-access-status/v1", + "boot_id": BOOT_ID.read_text(encoding="ascii").strip(), + "mode": mode, + "local_console_access": local_access, + "state_outcome": state_outcome, + "remote_access": "inactive" if remote_inactive else "unexpected_active", + "configuration_commit": "present" if configured else "absent", + "credential_action": credential_action, + "credential_scope": "persistent" if persistent else "boot", + "credential_persisted": credential_persisted, + } + ) return 0 if local_access and remote_inactive else 1 diff --git a/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate b/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate index ec0bf416..2963114b 100644 --- a/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate +++ b/build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate @@ -14,6 +14,8 @@ ATTESTATION = RUNTIME / "boot-device.json" BOOT_ID = Path("/proc/sys/kernel/random/boot_id") PARTUUID_ROOT = Path("/dev/disk/by-partuuid") PARTUUID_RE = re.compile(r"^[0-9A-Fa-f-]{1,128}$") +FILESYSTEM_TIMEOUT_SECONDS = 300 +MAX_COMMAND_OUTPUT = 64 * 1024 def read_json(path: Path) -> dict: @@ -128,26 +130,78 @@ def mark_repair_required(message: str) -> None: write_status(status) +def bounded_output(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + encoded = value.encode("utf-8", errors="replace") + if len(encoded) > MAX_COMMAND_OUTPUT: + encoded = encoded[:MAX_COMMAND_OUTPUT] + value = encoded.decode("utf-8", errors="replace") + " [truncated]" + return value.strip() + + +def run_repair_command(argv: list[str], accepted: tuple[int, ...]) -> tuple[bool, str | None]: + program = Path(argv[0]).name + try: + result = subprocess.run( + argv, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=FILESYSTEM_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as error: + detail = bounded_output(error.stderr) or bounded_output(error.stdout) + suffix = f": {detail}" if detail else "" + return False, f"{program}: timeout after {FILESYSTEM_TIMEOUT_SECONDS}s{suffix}" + if result.returncode not in accepted: + detail = bounded_output(result.stderr) or bounded_output(result.stdout) + suffix = f": {detail}" if detail else "" + return False, f"{program}: exit {result.returncode}{suffix}" + return True, None + + def forced_check() -> bool: device, error = verified_state_device() if error is not None or device is None: mark_repair_required(error or "verified state device is unavailable") return False - result = subprocess.run( - ["/usr/sbin/e2fsck", "-f", "-p", str(device)], - check=False, + ok, failure = run_repair_command( + ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1) ) - if result.returncode in (0, 1): + if ok: return True - mark_repair_required( - f"forced ext4 check requires owner intervention with exit code {result.returncode}" - ) + mark_repair_required(f"forced ext4 check failed: {failure}") return False +def complete_resize_after_timeout() -> bool: + device, error = verified_state_device() + if error is not None or device is None: + mark_repair_required(error or "verified state device is unavailable") + return False + checked, check_failure = run_repair_command( + ["/usr/sbin/e2fsck", "-f", "-p", str(device)], (0, 1) + ) + if not checked: + mark_repair_required(f"post-timeout ext4 check failed: {check_failure}") + return False + resized, resize_failure = run_repair_command( + ["/usr/sbin/resize2fs", str(device)], (0,) + ) + if not resized: + mark_repair_required(f"state filesystem resize failed: {resize_failure}") + return False + return True + + def main() -> int: retried_missing_device = False repaired_filesystem = False + completed_resize_timeout = False for _attempt in range(4): result = run_core() if result != 0: @@ -156,22 +210,36 @@ def main() -> int: if status.get("outcome") == "ready": return 0 message = str(status.get("message") or "") + outcome = status.get("outcome") + recoverable = outcome in ("limited_capacity", "repair_required") if ( - status.get("outcome") == "limited_capacity" + outcome == "limited_capacity" and not retried_missing_device and "No such file or directory" in message ): retried_missing_device = True time.sleep(1) continue - if ( - status.get("outcome") == "limited_capacity" - and not repaired_filesystem - and "e2fsck -f" in message - ): + if recoverable and ("e2fsck -f" in message or "e2fsck: timeout" in message): + if repaired_filesystem: + mark_repair_required( + "state core ext4 check remained incomplete after bounded repair" + ) + return 0 repaired_filesystem = True if forced_check(): continue + return 0 + if recoverable and "resize2fs: timeout" in message: + if completed_resize_timeout: + mark_repair_required( + "state core resize remained incomplete after bounded repair" + ) + return 0 + completed_resize_timeout = True + if complete_resize_after_timeout(): + continue + return 0 return 0 return 0 diff --git a/build/usb/version.env b/build/usb/version.env index 09f806af..aa1456cb 100644 --- a/build/usb/version.env +++ b/build/usb/version.env @@ -1,8 +1,8 @@ -RIGOS_PRODUCT_VERSION=0.0.4-alpha.10 -RIGOS_IMAGE_VERSION=0.0.4-alpha.10 +RIGOS_PRODUCT_VERSION=0.0.4-alpha.11 +RIGOS_IMAGE_VERSION=0.0.4-alpha.11 RIGOS_IMAGE_ID=rigos-usb-amd64 RIGOS_IMAGE_CHANNEL=alpha -RIGOS_BUILD_ORDINAL=10 +RIGOS_BUILD_ORDINAL=11 RIGOS_XMRIG_VERSION=6.26.0 RIGOS_XMRIG_ARCHIVE_SHA256=fc6f8ae5f64e4f17481f7e3be29a1c56949f216a998414188003eae1db20c9e5 RIGOS_XMRIG_BINARY_SHA256=b20f39fc00d242e706b6c30367ad811c676e0575050a4ec2f30104b696944b49 diff --git a/crates/rigos-config/tests/alpha8_recovery_gate.rs b/crates/rigos-config/tests/alpha8_recovery_gate.rs index dce3b59b..4f4ca0b7 100644 --- a/crates/rigos-config/tests/alpha8_recovery_gate.rs +++ b/crates/rigos-config/tests/alpha8_recovery_gate.rs @@ -9,8 +9,18 @@ fn repo_path(path: &str) -> PathBuf { .join(path) } +fn run_gate(runtime: &PathBuf, boot: &PathBuf) -> std::process::ExitStatus { + let gate = repo_path("build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify"); + Command::new("python3") + .arg(gate) + .env("RIGOS_RUNTIME_PATH", runtime) + .env("RIGOS_BOOT_ID_PATH", boot) + .status() + .unwrap() +} + #[test] -fn recovery_gate_accepts_only_current_persisted_credential_truth() { +fn recovery_gate_accepts_persistent_or_explicit_boot_scoped_credential_truth() { let root = std::env::temp_dir().join(format!("rigos-recovery-gate-{}", Uuid::new_v4())); let runtime = root.join("run"); let boot_id = root.join("boot-id"); @@ -18,46 +28,69 @@ fn recovery_gate_accepts_only_current_persisted_credential_truth() { fs::write(&boot_id, "boot-test\n").unwrap(); let status = runtime.join("recovery-access-status.json"); - let valid = serde_json::json!({ + let persistent = serde_json::json!({ "schema": "rigos.recovery-access-status/v1", "boot_id": "boot-test", "local_console_access": true, "credential_action": "created", - "credential_persisted": true + "credential_scope": "persistent", + "credential_persisted": true, + "state_outcome": "ready" }); - fs::write(&status, serde_json::to_vec(&valid).unwrap()).unwrap(); - - let gate = repo_path("build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify"); - let run = |runtime_path: &PathBuf, boot_path: &PathBuf| { - Command::new("python3") - .arg(&gate) - .env("RIGOS_RUNTIME_PATH", runtime_path) - .env("RIGOS_BOOT_ID_PATH", boot_path) - .status() - .unwrap() - }; + fs::write(&status, serde_json::to_vec(&persistent).unwrap()).unwrap(); + assert!(run_gate(&runtime, &boot_id).success()); - assert!(run(&runtime, &boot_id).success()); + let boot_scoped = serde_json::json!({ + "schema": "rigos.recovery-access-status/v1", + "boot_id": "boot-test", + "local_console_access": true, + "credential_action": "created", + "credential_scope": "boot", + "credential_persisted": false, + "state_outcome": "repair_required" + }); + fs::write(&status, serde_json::to_vec(&boot_scoped).unwrap()).unwrap(); + assert!(run_gate(&runtime, &boot_id).success()); let stale = serde_json::json!({ "schema": "rigos.recovery-access-status/v1", "boot_id": "old-boot", "local_console_access": true, "credential_action": "created", - "credential_persisted": true + "credential_scope": "persistent", + "credential_persisted": true, + "state_outcome": "ready" }); fs::write(&status, serde_json::to_vec(&stale).unwrap()).unwrap(); - assert_eq!(run(&runtime, &boot_id).code(), Some(2)); + assert_eq!(run_gate(&runtime, &boot_id).code(), Some(2)); + + let missing_persistent_store = serde_json::json!({ + "schema": "rigos.recovery-access-status/v1", + "boot_id": "boot-test", + "local_console_access": true, + "credential_action": "created", + "credential_scope": "persistent", + "credential_persisted": false, + "state_outcome": "ready" + }); + fs::write( + &status, + serde_json::to_vec(&missing_persistent_store).unwrap(), + ) + .unwrap(); + assert_eq!(run_gate(&runtime, &boot_id).code(), Some(2)); - let not_persisted = serde_json::json!({ + let false_boot_claim = serde_json::json!({ "schema": "rigos.recovery-access-status/v1", "boot_id": "boot-test", "local_console_access": true, "credential_action": "created", - "credential_persisted": false + "credential_scope": "boot", + "credential_persisted": true, + "state_outcome": "limited_capacity" }); - fs::write(&status, serde_json::to_vec(¬_persisted).unwrap()).unwrap(); - assert_eq!(run(&runtime, &boot_id).code(), Some(2)); + fs::write(&status, serde_json::to_vec(&false_boot_claim).unwrap()).unwrap(); + assert_eq!(run_gate(&runtime, &boot_id).code(), Some(2)); let _ = fs::remove_dir_all(root); } diff --git a/crates/rigos-config/tests/diagnostic_ssh.rs b/crates/rigos-config/tests/diagnostic_ssh.rs index 9251e5d2..b0c9fb0f 100644 --- a/crates/rigos-config/tests/diagnostic_ssh.rs +++ b/crates/rigos-config/tests/diagnostic_ssh.rs @@ -55,8 +55,8 @@ fn diagnostic_ssh_keeps_host_identity_mandatory_without_requiring_ready_state() ); } - assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.10")); - assert!(version.contains("RIGOS_BUILD_ORDINAL=10")); + assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.11")); + assert!(version.contains("RIGOS_BUILD_ORDINAL=11")); } #[test] diff --git a/crates/rigos-config/tests/firstboot_image_gate.rs b/crates/rigos-config/tests/firstboot_image_gate.rs index 8b623ffb..a4c4b7d6 100644 --- a/crates/rigos-config/tests/firstboot_image_gate.rs +++ b/crates/rigos-config/tests/firstboot_image_gate.rs @@ -10,17 +10,18 @@ fn repo_file(path: &str) -> String { } #[test] -fn alpha10_build_runs_source_and_exact_image_firstboot_gates() { +fn alpha11_build_runs_source_and_exact_image_firstboot_gates() { let version = repo_file("build/usb/version.env"); let entrypoint = repo_file("scripts/build-usb-image-entrypoint.sh"); let verifier = repo_file("scripts/verify-firstboot-image.sh"); let hook = repo_file("build/usb/hooks/010-rigos.chroot"); - assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.10")); - assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.10")); - assert!(version.contains("RIGOS_BUILD_ORDINAL=10")); + assert!(version.contains("RIGOS_PRODUCT_VERSION=0.0.4-alpha.11")); + assert!(version.contains("RIGOS_IMAGE_VERSION=0.0.4-alpha.11")); + assert!(version.contains("RIGOS_BUILD_ORDINAL=11")); assert!(entrypoint.contains("--test firstboot_tty")); + assert!(entrypoint.contains("--test state_resize_recovery")); assert!(entrypoint.contains("bash ./scripts/verify-firstboot-image.sh \"$image\"")); for required in [ diff --git a/crates/rigos-config/tests/recovery_access.rs b/crates/rigos-config/tests/recovery_access.rs index fe6ff850..db144b23 100644 --- a/crates/rigos-config/tests/recovery_access.rs +++ b/crates/rigos-config/tests/recovery_access.rs @@ -39,10 +39,17 @@ g['STATE'] = root / 'state' g['CREDENTIAL_DIRECTORY'] = root / 'state' / 'recovery' g['CREDENTIAL_FILE'] = g['CREDENTIAL_DIRECTORY'] / 'rigosadmin-password.hash' g['BOOT_ID'] = root / 'boot-id' +g['STATE_STATUS'] = g['RUNTIME'] / 'state-status.json' g['BOOT_ID'].write_text('boot-test\n', encoding='ascii') g['RUNTIME'].mkdir() g['STATE'].mkdir() -(g['RUNTIME'] / 'state-status.json').write_text('{"outcome":"ready"}', encoding='utf-8') +g['STATE_STATUS'].write_text(json.dumps({ + 'schema': 'rigos.state-status/v1', + 'boot_id': 'boot-test', + 'outcome': 'ready', + 'mountpoint': str(g['STATE']), +}), encoding='utf-8') +g['persistent_store_ready'] = lambda _status: True valid_hash = '$y$j9T$syntheticSalt$syntheticHashValue' assert namespace['valid_password_hash'](valid_hash) @@ -54,7 +61,8 @@ live_ready = {'value': False} prompts = [] persisted = [] g['password_ready'] = lambda: live_ready['value'] -def prompt(_invalid): +def prompt(_invalid, persistent): + assert persistent is True prompts.append(True) live_ready['value'] = True g['prompt_for_password'] = prompt @@ -72,6 +80,7 @@ assert stat.S_IMODE(g['CREDENTIAL_DIRECTORY'].stat().st_mode) == 0o700 assert stat.S_IMODE(g['CREDENTIAL_FILE'].stat().st_mode) == 0o600 status = json.loads((g['RUNTIME'] / 'recovery-access-status.json').read_text()) assert status['credential_action'] == 'created' +assert status['credential_scope'] == 'persistent' assert status['credential_persisted'] is True assert valid_hash not in json.dumps(status) @@ -95,6 +104,8 @@ assert not prompts assert not any('/usr/bin/passwd' in argv for argv, _kwargs in calls) status = json.loads((g['RUNTIME'] / 'recovery-access-status.json').read_text()) assert status['credential_action'] == 'restored' +assert status['credential_scope'] == 'persistent' +assert status['credential_persisted'] is True assert valid_hash not in json.dumps(status) # Existing live credential migrates without a prompt. @@ -109,15 +120,43 @@ g['CREDENTIAL_FILE'].write_text('!unsafe\n', encoding='ascii') g['CREDENTIAL_FILE'].chmod(0o600) live_ready['value'] = False invalid_flags = [] -def replacement(invalid): - invalid_flags.append(invalid) +def replacement(invalid, persistent): + invalid_flags.append((invalid, persistent)) live_ready['value'] = True g['prompt_for_password'] = replacement calls.clear() assert namespace['main']() == 0 -assert invalid_flags == [True] +assert invalid_flags == [(True, True)] assert not any(argv == ['/usr/sbin/chpasswd', '--encrypted'] for argv, _kwargs in calls) assert valid_hash not in json.dumps(json.loads((g['RUNTIME'] / 'recovery-access-status.json').read_text())) + +# Unready state uses a truthful boot-scoped credential and never touches the store. +g['CREDENTIAL_FILE'].unlink(missing_ok=True) +g['STATE_STATUS'].write_text(json.dumps({ + 'schema': 'rigos.state-status/v1', + 'boot_id': 'boot-test', + 'outcome': 'repair_required', + 'mountpoint': None, +}), encoding='utf-8') +g['persistent_store_ready'] = lambda _status: False +live_ready['value'] = False +boot_prompts = [] +def boot_prompt(invalid, persistent): + assert invalid is False and persistent is False + boot_prompts.append(True) + live_ready['value'] = True +g['prompt_for_password'] = boot_prompt +g['persist_password_hash'] = lambda _value: (_ for _ in ()).throw( + AssertionError('boot credential touched persistent state') +) +assert namespace['main']() == 0 +assert boot_prompts == [True] +assert not g['CREDENTIAL_FILE'].exists() +status = json.loads((g['RUNTIME'] / 'recovery-access-status.json').read_text()) +assert status['credential_scope'] == 'boot' +assert status['credential_persisted'] is False +assert status['state_outcome'] == 'repair_required' +assert valid_hash not in json.dumps(status) "#; let result = Command::new("python3") .arg("-c") diff --git a/crates/rigos-config/tests/state_resize_recovery.rs b/crates/rigos-config/tests/state_resize_recovery.rs new file mode 100644 index 00000000..13473f74 --- /dev/null +++ b/crates/rigos-config/tests/state_resize_recovery.rs @@ -0,0 +1,116 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn repo_path(path: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(path) +} + +#[test] +fn state_resize_timeout_has_a_bounded_verified_recovery_path() { + let orchestrator = fs::read_to_string(repo_path( + "build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate", + )) + .unwrap(); + let service = fs::read_to_string(repo_path( + "build/usb/includes.chroot/etc/systemd/system/rigos-state.service", + )) + .unwrap(); + let entrypoint = + fs::read_to_string(repo_path("scripts/build-usb-image-entrypoint.sh")).unwrap(); + let image_verifier = + fs::read_to_string(repo_path("scripts/verify-state-recovery-image.sh")).unwrap(); + + for required in [ + "FILESYSTEM_TIMEOUT_SECONDS = 300", + "def complete_resize_after_timeout() -> bool:", + "timeout=FILESYSTEM_TIMEOUT_SECONDS", + "post-timeout ext4 check failed", + "state filesystem resize failed", + "resize2fs: timeout", + ] { + assert!( + orchestrator.contains(required), + "state recovery contract is missing: {required}" + ); + assert!( + image_verifier.contains(required), + "exact-image verifier is missing: {required}" + ); + } + assert!(service.contains("TimeoutStartSec=12min")); + assert!(image_verifier.contains("losetup --find --show --read-only")); + assert!(image_verifier.contains("mount -o ro")); + assert!(!image_verifier.contains("mount -o rw")); + assert!(entrypoint.contains("bash ./scripts/verify-state-recovery-image.sh \"$image\"")); +} + +#[test] +fn resize_timeout_recovery_retries_core_and_reports_failed_repair_truthfully() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("rigos-state-resize-{unique}")); + fs::create_dir_all(&root).unwrap(); + + let fixture = r#" +import runpy +import sys +from pathlib import Path + +source = Path(sys.argv[1]) +root = Path(sys.argv[2]) +namespace = runpy.run_path(str(source), run_name='rigos_state_resize_test') +g = namespace['main'].__globals__ +g['STATUS'] = root / 'state-status.json' +g['ATTESTATION'] = root / 'boot-device.json' +g['BOOT_ID'] = root / 'boot-id' +g['BOOT_ID'].write_text('boot-test\n', encoding='ascii') + +# A core resize timeout is completed under the longer verified repair budget, +# then core is rerun to perform the normal mount and initialization path. +statuses = [ + {'outcome': 'limited_capacity', 'message': 'bounded command failed: resize2fs: timeout'}, + {'outcome': 'ready', 'message': None}, +] +core_calls = [] +repair_calls = [] +g['run_core'] = lambda: core_calls.append(True) or 0 +def read_json(path): + if path == g['STATUS']: + return statuses.pop(0) + return {} +g['read_json'] = read_json +g['complete_resize_after_timeout'] = lambda: repair_calls.append(True) or True +assert namespace['main']() == 0 +assert len(core_calls) == 2 +assert repair_calls == [True] +assert statuses == [] + +# A failed long repair is not reclassified as limited capacity. +recorded = [] +g['verified_state_device'] = lambda: (Path('/dev/test-state'), None) +results = iter([(True, None), (False, 'resize2fs: timeout after 300s')]) +g['run_repair_command'] = lambda _argv, _accepted: next(results) +g['mark_repair_required'] = lambda message: recorded.append(message) +assert namespace['complete_resize_after_timeout']() is False +assert recorded == ['state filesystem resize failed: resize2fs: timeout after 300s'] +"#; + + let result = Command::new("python3") + .arg("-c") + .arg(fixture) + .arg(repo_path( + "build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate", + )) + .arg(&root) + .status() + .expect("run state resize recovery fixture"); + + let _ = fs::remove_dir_all(root); + assert!(result.success(), "state resize recovery fixture failed"); +} diff --git a/scripts/build-usb-image-entrypoint.sh b/scripts/build-usb-image-entrypoint.sh index 336a12e2..94943f47 100644 --- a/scripts/build-usb-image-entrypoint.sh +++ b/scripts/build-usb-image-entrypoint.sh @@ -22,6 +22,9 @@ source "$version_env" python3 ./scripts/check-alpha8-ssh-hotfix.py python3 ./scripts/verify-systemd-ordering.py python3 -m py_compile \ + ./build/usb/includes.chroot/usr/local/sbin/rigos-recovery-access \ + ./build/usb/includes.chroot/usr/local/sbin/rigos-state-orchestrate \ + ./build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify \ ./build/usb/includes.chroot/usr/lib/rigos/rigos-randomx-msr \ ./build/usb/includes.chroot/usr/lib/rigos/rigos-miner-gate \ ./build/usb/includes.chroot/usr/lib/rigos/rigos-ssh-hostkeys \ @@ -46,6 +49,7 @@ cargo test --locked -p rigos-config --test randomx_build_entrypoint -- --nocaptu cargo test --locked -p rigos-config --test randomx_msr_authority -- --nocapture cargo test --locked -p rigos-config --test firstboot_tty -- --nocapture cargo test --locked -p rigos-config --test diagnostic_ssh -- --nocapture +cargo test --locked -p rigos-config --test state_resize_recovery -- --nocapture ./scripts/build-usb-image.sh @@ -53,3 +57,4 @@ image="./dist/usb/${RIGOS_IMAGE_ID}-${RIGOS_IMAGE_VERSION}.img" bash ./scripts/verify-randomx-performance-image.sh "$image" bash ./scripts/verify-miner-observer-image.sh "$image" bash ./scripts/verify-firstboot-image.sh "$image" +bash ./scripts/verify-state-recovery-image.sh "$image" diff --git a/scripts/check-alpha8-ssh-hotfix.py b/scripts/check-alpha8-ssh-hotfix.py index 654826e0..7324c2be 100644 --- a/scripts/check-alpha8-ssh-hotfix.py +++ b/scripts/check-alpha8-ssh-hotfix.py @@ -9,6 +9,7 @@ HOOK = ROOT / "build/usb/hooks/010-rigos.chroot" DOCKERFILE = ROOT / "build/usb/Dockerfile" RECOVERY_UNIT = ROOT / "build/usb/includes.chroot/etc/systemd/system/rigos-recovery-access.service" +RECOVERY_AUTHORITY = ROOT / "build/usb/includes.chroot/usr/local/sbin/rigos-recovery-access" RECOVERY_GATE = ROOT / "build/usb/includes.chroot/usr/lib/rigos/rigos-recovery-access-verify" STATE_READY_UNIT = ROOT / "build/usb/includes.chroot/etc/systemd/system/rigos-state-ready.service" HOSTKEY_UNIT = ROOT / "build/usb/includes.chroot/etc/systemd/system/rigos-ssh-hostkeys.service" @@ -130,12 +131,27 @@ def main() -> int: if required not in recovery_unit: raise RuntimeError(f"recovery access hotfix wiring is missing: {required}") + recovery_authority = RECOVERY_AUTHORITY.read_text(encoding="utf-8") + compile(recovery_authority, str(RECOVERY_AUTHORITY), "exec") + for required in ( + "def persistent_store_ready(status: dict) -> bool:", + '"credential_scope": "persistent" if persistent else "boot"', + "credential_persisted = persistent and CREDENTIAL_FILE.is_file()", + "if persistent:", + "This password is not persistent", + ): + if required not in recovery_authority: + raise RuntimeError(f"recovery credential authority contract is missing: {required}") + recovery_gate = RECOVERY_GATE.read_text(encoding="utf-8") compile(recovery_gate, str(RECOVERY_GATE), "exec") for required in ( 'status.get("boot_id") != boot_id', 'status.get("local_console_access") is not True', - 'status.get("credential_persisted") is not True', + 'scope = status.get("credential_scope")', + 'if scope == "persistent":', + 'elif scope == "boot":', + 'return deny("boot_credential_claims_persistence")', ): if required not in recovery_gate: raise RuntimeError(f"recovery access validator contract is missing: {required}") diff --git a/scripts/verify-state-recovery-image.sh b/scripts/verify-state-recovery-image.sh new file mode 100644 index 00000000..fab7186a --- /dev/null +++ b/scripts/verify-state-recovery-image.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -euo pipefail + +die() { + printf 'verify-state-recovery-image: %s\n' "$*" >&2 + exit 1 +} + +[[ $# -eq 1 ]] || die 'usage: verify-state-recovery-image.sh ' +image="$(readlink -f "$1")" +[[ -f "$image" ]] || die "image is missing: $image" +[[ "$(id -u)" -eq 0 ]] || die 'must run as root' + +partition_json="$(sfdisk --json "$image")" +start="$(jq -r '.partitiontable.partitions[1].start' <<<"$partition_json")" +size="$(jq -r '.partitiontable.partitions[1].size' <<<"$partition_json")" +[[ "$start" =~ ^[0-9]+$ && "$size" =~ ^[0-9]+$ ]] || die 'ROOT_A geometry is invalid' + +loop="$( + losetup --find --show --read-only \ + --offset $((start * 512)) \ + --sizelimit $((size * 512)) \ + "$image" +)" +temporary="$(mktemp -d)" +cleanup() { + set +e + mountpoint -q "$temporary/root-a" && umount "$temporary/root-a" + losetup -d "$loop" 2>/dev/null + rm -rf "$temporary" +} +trap cleanup EXIT + +mkdir -p "$temporary/root-a" "$temporary/squash" +mount -o ro "$loop" "$temporary/root-a" +squashfs="$temporary/root-a/live/filesystem.squashfs" +[[ -f "$squashfs" ]] || die 'ROOT_A squashfs is missing' + +unsquashfs -no-progress -d "$temporary/squash" "$squashfs" \ + usr/bin/python3 usr/bin/python3.11 \ + usr/local/sbin/rigos-state-orchestrate \ + usr/local/sbin/rigos-recovery-access \ + usr/lib/rigos/rigos-recovery-access-verify \ + etc/systemd/system/rigos-state.service \ + etc/rigos-release \ + >/dev/null + +root="$temporary/squash" +orchestrator="$root/usr/local/sbin/rigos-state-orchestrate" +recovery="$root/usr/local/sbin/rigos-recovery-access" +gate="$root/usr/lib/rigos/rigos-recovery-access-verify" +state_service="$root/etc/systemd/system/rigos-state.service" +release="$root/etc/rigos-release" + +[[ -x "$root/usr/bin/python3" ]] || die 'Python runtime is missing' +[[ -x "$orchestrator" ]] || die 'state orchestrator is missing or not executable' +[[ -x "$recovery" ]] || die 'recovery credential authority is missing or not executable' +[[ -f "$gate" ]] || die 'recovery credential gate is missing' +[[ -f "$state_service" ]] || die 'state service is missing' +[[ -f "$release" ]] || die 'release metadata is missing' + +python3 -m py_compile "$orchestrator" "$recovery" "$gate" +grep -Fqx 'VERSION_ID="0.0.4-alpha.11"' "$release" \ + || die 'embedded alpha.11 version is missing' +grep -Fqx 'TimeoutStartSec=12min' "$state_service" \ + || die 'state service recovery window is missing' + +for required in \ + 'FILESYSTEM_TIMEOUT_SECONDS = 300' \ + 'def complete_resize_after_timeout() -> bool:' \ + 'timeout=FILESYSTEM_TIMEOUT_SECONDS' \ + 'post-timeout ext4 check failed' \ + 'state filesystem resize failed' \ + 'resize2fs: timeout' +do + grep -Fq "$required" "$orchestrator" \ + || die "state resize recovery contract is missing: $required" +done + +for required in \ + 'def persistent_store_ready(status: dict) -> bool:' \ + '"credential_scope": "persistent" if persistent else "boot"' \ + 'credential_persisted = persistent and CREDENTIAL_FILE.is_file()' \ + 'This password is not persistent' +do + grep -Fq "$required" "$recovery" \ + || die "recovery credential contract is missing: $required" +done + +for required in \ + 'scope = status.get("credential_scope")' \ + 'if scope == "persistent":' \ + 'elif scope == "boot":' \ + 'boot_credential_claims_persistence' +do + grep -Fq "$required" "$gate" \ + || die "recovery credential gate contract is missing: $required" +done + +printf 'RIGOS state recovery and credential image verification passed: %s\n' "$image"