From a829bdc59eaa9c5a410ac6a2b80b9f949b956712 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Mon, 15 Jun 2026 01:32:37 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Fail closed on Codex account auth failures","authority":"manual"} --- README.md | 3 + apps/decodex/src/accounts.rs | 76 +++- apps/decodex/src/agent.rs | 2 +- apps/decodex/src/agent/app_server.rs | 49 ++- apps/decodex/src/agent/codex_accounts.rs | 347 +++++++++++++++++- apps/decodex/src/orchestrator/execution.rs | 5 + apps/decodex/src/orchestrator/selection.rs | 2 + .../src/orchestrator/tests/runtime/failure.rs | 20 + docs/reference/operator-control-plane.md | 2 +- docs/spec/app-server.md | 5 + docs/spec/runtime.md | 2 +- 11 files changed, 492 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index bec3d8680..c94f2d0b9 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,9 @@ Codex profile token stats for local Accounts displays. Bounded seven-day account usage estimates are kept in `~/.codex/decodex/account-usage-history.jsonl`; the file stores daily percentage snapshots plus non-secret capacity weights for local display and no token material. +Refresh authentication failures mark the account `auth_failed` in `accounts.jsonl`; +Decodex will not select or manually activate that account again until it is re-logged +or replaced. To switch the account used by the Codex CLI itself, run `decodex account use ` or use the Decodex App row action; this overwrites `$CODEX_HOME/auth.json` or `~/.codex/auth.json` from the matching `accounts.jsonl` diff --git a/apps/decodex/src/accounts.rs b/apps/decodex/src/accounts.rs index 72774b3f0..e46a387f6 100644 --- a/apps/decodex/src/accounts.rs +++ b/apps/decodex/src/accounts.rs @@ -229,6 +229,11 @@ impl AccountStore { if record.disabled { eyre::bail!("Decodex account `{selector}` is disabled and cannot be used by Codex."); } + if record.auth_failure().is_some() { + eyre::bail!( + "Decodex account `{selector}` is auth_failed and must be re-logged before Codex can use it." + ); + } record.validate_importable()?; @@ -1037,6 +1042,10 @@ struct AccountPoolRecord { #[serde(skip_serializing_if = "Option::is_none")] last_selected_at_unix_epoch: Option, #[serde(skip_serializing_if = "Option::is_none")] + auth_failed_at_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_failure: Option, + #[serde(skip_serializing_if = "Option::is_none")] auth_mode: Option, #[serde(rename = "OPENAI_API_KEY", skip_serializing_if = "Option::is_none")] openai_api_key: Option, @@ -1059,6 +1068,8 @@ impl AccountPoolRecord { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: auth.auth_mode, openai_api_key: auth.openai_api_key, tokens: auth.tokens, @@ -1102,6 +1113,14 @@ impl AccountPoolRecord { || self.account_id().map(redact_account_id).as_deref() == Some(selector) } + fn auth_failure(&self) -> Option<&str> { + self.auth_failure + .as_deref() + .map(str::trim) + .filter(|failure| !failure.is_empty()) + .or_else(|| self.auth_failed_at_unix_epoch.map(|_| "authentication failed")) + } + fn matches_account_identity(&self, identity: &AccountIdentity) -> bool { identity .account_id @@ -1170,6 +1189,8 @@ impl AccountPoolRecord { .is_some(); let status = if self.disabled { "disabled" + } else if self.auth_failure().is_some() { + "auth_failed" } else if self.cooldown_until_unix_epoch.is_some_and(|cooldown_until| cooldown_until > now) { "cooldown" @@ -1186,8 +1207,8 @@ impl AccountPoolRecord { let recovery_action = account_recovery_action( status, refresh_token_present, - None, - Some("local account pool"), + if self.auth_failure().is_some() { Some("auth_failed") } else { None }, + self.auth_failure().or(Some("local account pool")), ); AccountSummary { @@ -1206,7 +1227,10 @@ impl AccountPoolRecord { access_token_expires_at_unix_epoch, last_selected_at_unix_epoch: self.last_selected_at_unix_epoch, cooldown_until_unix_epoch: self.cooldown_until_unix_epoch, - note: Some(String::from("local account pool")), + note: Some( + self.auth_failure() + .map_or_else(|| String::from("local account pool"), ToOwned::to_owned), + ), plan_type: None, capacity_multiplier: DEFAULT_ACCOUNT_CAPACITY_MULTIPLIER, recovery_action, @@ -1275,6 +1299,10 @@ enum AccountPoolLine { cooldown_until: Option, #[serde(skip_serializing_if = "Option::is_none")] last_selected_at_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_failed_at_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_failure: Option, auth: AuthDotJson, }, Flat(AccountPoolRecord), @@ -1289,6 +1317,8 @@ impl AccountPoolLine { cooldown_until_unix_epoch, cooldown_until, last_selected_at_unix_epoch, + auth_failed_at_unix_epoch, + auth_failure, auth, } => { let mut record = AccountPoolRecord::from_auth(auth)?; @@ -1298,6 +1328,8 @@ impl AccountPoolLine { record.cooldown_until_unix_epoch = cooldown_until_unix_epoch; record.cooldown_until = cooldown_until; record.last_selected_at_unix_epoch = last_selected_at_unix_epoch; + record.auth_failed_at_unix_epoch = auth_failed_at_unix_epoch; + record.auth_failure = auth_failure; Ok(record) }, @@ -1835,6 +1867,9 @@ fn account_recovery_action( if status == "disabled" || status == "cooldown" { return None; } + if status == "auth_failed" || refresh_status == "auth_failed" { + return Some(String::from("login")); + } if !refresh_token_present { return Some(String::from("login")); } @@ -2061,6 +2096,8 @@ mod tests { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: None, openai_api_key: None, tokens: Some(CodexTokenData { @@ -2113,6 +2150,37 @@ mod tests { assert_eq!(tokens.account_id.as_deref(), Some("acct_123456")); } + #[test] + fn use_for_codex_rejects_auth_failed_account() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let codex_auth_path = temp_dir.path().join(".codex/auth.json"); + let store = AccountStore::new_with_codex_auth_path( + temp_dir.path().join("accounts.jsonl"), + temp_dir.path().join("config.toml"), + codex_auth_path, + ); + let mut record = account_record( + "copy@example.com", + "acct_123456", + "header.eyJleHAiOjQxMDI0NDQ4MDB9.sig", + "refresh-secret", + ); + + record.auth_failed_at_unix_epoch = Some(1_800_000_000); + record.auth_failure = Some(String::from( + "Codex account `copy@example.com` token refresh failed with HTTP 401 Unauthorized.", + )); + + store.save_records(&[record]).expect("records should save"); + + let error = match store.use_for_codex("copy@example.com", None) { + Ok(_) => panic!("auth failed account should reject"), + Err(error) => error, + }; + + assert!(error.to_string().contains("auth_failed")); + } + #[test] fn list_marks_codex_active_account() { let temp_dir = TempDir::new().expect("temp dir should create"); @@ -2513,6 +2581,8 @@ mod tests { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: None, openai_api_key: None, tokens: Some(CodexTokenData { diff --git a/apps/decodex/src/agent.rs b/apps/decodex/src/agent.rs index 2e744d33e..656c8280d 100644 --- a/apps/decodex/src/agent.rs +++ b/apps/decodex/src/agent.rs @@ -17,7 +17,7 @@ pub(crate) use self::{ TurnContinuationGuard, execute_app_server_run, probe_app_server, protocol_activity_idle_timeout, }, - codex_accounts::{CodexAccountPool, CodexAccountProvider}, + codex_accounts::{CodexAccountAuthFailure, CodexAccountPool, CodexAccountProvider}, decodex_tool_bridge::{DecodexRunContext, DecodexToolBridge}, json_rpc::{AppServerHomePreflightFailure, AppServerProcessEnv, AppServerTransportFailure}, tracker_tool_bridge::{ diff --git a/apps/decodex/src/agent/app_server.rs b/apps/decodex/src/agent/app_server.rs index 9b51731a7..775b905ad 100644 --- a/apps/decodex/src/agent/app_server.rs +++ b/apps/decodex/src/agent/app_server.rs @@ -39,7 +39,7 @@ use self::protocol::{ use crate::{ agent::{ app_server::protocol::LoginAccountResponse, - codex_accounts::{CodexAccountLogin, CodexAccountProvider}, + codex_accounts::{CodexAccountAuthFailure, CodexAccountLogin, CodexAccountProvider}, json_rpc, json_rpc::{ AppServerHomePreflightFailure, AppServerOutputTimeout, AppServerProcessEnv, @@ -3386,7 +3386,14 @@ fn login_codex_account_for_run( let Some(account_provider) = request.codex_account_provider else { return Ok(()); }; - let account = account_provider.select_account()?; + let account = match account_provider.select_account() { + Ok(account) => account, + Err(error) => { + record_codex_account_failure(recorder, "account/login/select/failed", &error); + + return Err(error); + }, + }; recorder.set_codex_account(account.summary(), account.account_summaries())?; @@ -3431,6 +3438,31 @@ fn login_codex_account_for_run( Ok(()) } +fn record_codex_account_failure(recorder: &mut RunRecorder<'_>, event_type: &str, error: &Report) { + let auth_failure = error.downcast_ref::(); + let error_class = + auth_failure.map(CodexAccountAuthFailure::error_class).unwrap_or("codex_account_failure"); + let account_fingerprint = auth_failure.and_then(CodexAccountAuthFailure::account_fingerprint); + let email = auth_failure.and_then(CodexAccountAuthFailure::email); + let reason = + auth_failure.map_or_else(|| error.to_string(), |failure| failure.reason().to_owned()); + let payload = serde_json::json!({ + "errorClass": error_class, + "accountFingerprint": account_fingerprint, + "email": email, + "reason": reason, + }); + + if let Err(record_error) = recorder.record(event_type, &payload.to_string()) { + tracing::warn!( + ?record_error, + event_type, + error_class, + "Failed to record Codex account failure event." + ); + } +} + fn start_or_resume_thread_session( client: &mut AppServerClient, recorder: &mut RunRecorder<'_>, @@ -5124,7 +5156,18 @@ fn dispatch_codex_account_refresh( ) })?; let params = serde_json::from_value::(request.params.clone())?; - let account = account_provider.refresh_account(params.previous_account_id.as_deref())?; + let account = match account_provider.refresh_account(params.previous_account_id.as_deref()) { + Ok(account) => account, + Err(error) => { + record_codex_account_failure( + recorder, + "account/chatgptAuthTokens/refresh/failed", + &error, + ); + + return Err(error); + }, + }; let response = ChatgptAuthTokensRefreshResponse { access_token: account.access_token().to_owned(), chatgpt_account_id: account.account_id().to_owned(), diff --git a/apps/decodex/src/agent/codex_accounts.rs b/apps/decodex/src/agent/codex_accounts.rs index 51c407ea4..0292ce0ad 100644 --- a/apps/decodex/src/agent/codex_accounts.rs +++ b/apps/decodex/src/agent/codex_accounts.rs @@ -13,6 +13,7 @@ use std::{ time::Duration, }; +use color_eyre::Report; use reqwest::{StatusCode, blocking::Client}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -45,6 +46,69 @@ pub(crate) trait CodexAccountProvider { ) -> crate::prelude::Result; } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CodexAccountAuthFailure { + account_fingerprint: Option, + email: Option, + reason: String, +} +impl CodexAccountAuthFailure { + fn from_record(record: &AccountPoolRecord, reason: impl Into) -> Self { + Self::new(record.account_fingerprint(), record.email(), reason) + } + + pub(crate) fn new( + account_fingerprint: Option, + email: Option, + reason: impl Into, + ) -> Self { + Self { account_fingerprint, email, reason: reason.into() } + } + + pub(crate) const fn error_class(&self) -> &'static str { + "codex_account_auth_failed" + } + + pub(crate) fn account_fingerprint(&self) -> Option<&str> { + self.account_fingerprint.as_deref() + } + + pub(crate) fn email(&self) -> Option<&str> { + self.email.as_deref() + } + + pub(crate) fn reason(&self) -> &str { + &self.reason + } + + fn account_label(&self) -> String { + self.email + .clone() + .or_else(|| self.account_fingerprint.clone()) + .unwrap_or_else(|| String::from("unknown account")) + } + + pub(crate) fn terminal_next_action(&self, recovery_gate: &str) -> String { + format!( + "re-login or remove Decodex Codex account `{}`, verify `decodex account list --json` reports no `auth_failed` selected account, then {recovery_gate}", + self.account_label() + ) + } +} + +impl Display for CodexAccountAuthFailure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { + write!( + formatter, + "Codex account `{}` authentication failed: {}", + self.account_label(), + self.reason + ) + } +} + +impl Error for CodexAccountAuthFailure {} + pub(crate) struct CodexAccountPool { path: PathBuf, usage_endpoint: String, @@ -329,6 +393,12 @@ impl CodexAccountPool { self.save_records(records)?; } if candidates.is_empty() { + if let Some(auth_failure) = + records.iter().find_map(AccountPoolRecord::auth_failed_error) + { + return Err(Report::new(auth_failure)); + } + eyre::bail!( "No usable Codex account was available from `{}`. Skipped entries: {}", self.path.display(), @@ -369,6 +439,11 @@ impl CodexAccountPool { let record_index = self.fixed_record_index(records, selector)?; let now = OffsetDateTime::now_utc().unix_timestamp(); let mut records_changed = false; + + if let Some(auth_failure) = records[record_index].auth_failed_error() { + return Err(Report::new(auth_failure)); + } + let mut selected = match self.account_candidate_from_record( &mut records[record_index], record_index + 1, @@ -411,6 +486,11 @@ impl CodexAccountPool { if record.disabled { return Ok(Err(format!("line {line_number} disabled"))); } + + if let Some(auth_failure) = record.auth_failure() { + return Ok(Err(format!("line {line_number} auth failed: {auth_failure}"))); + } + if record.cooldown_until_unix_epoch.is_some_and(|cooldown| cooldown > now) { return Ok(Err(format!("line {line_number} cooling down"))); } @@ -429,6 +509,11 @@ impl CodexAccountPool { status.as_str() }, + Err(error) if error.auth_failed => { + *records_changed = true; + + return Ok(Err(format!("{} auth failed: {}", record.display_name(), error.source))); + }, Err(error) if error.requires_skip => { return Ok(Err(format!( "{} proactive refresh failed: {}", @@ -442,7 +527,20 @@ impl CodexAccountPool { match self.probe_record_usage(record) { Ok(usage) => Ok(Ok(record.login_from_usage(usage, refresh_status)?)), Err(error) if error.unauthorized && record.refresh_token().is_some() => { - self.refresh_record(record)?; + if let Err(refresh_error) = self.refresh_record(record) { + if let Some(auth_failure) = + refresh_error.downcast_ref::() + { + *records_changed = true; + + return Ok(Err(format!( + "{} auth failed: {auth_failure}", + record.display_name() + ))); + } + + return Err(refresh_error); + } *records_changed = true; @@ -541,6 +639,18 @@ impl CodexAccountPool { status.as_str() }, + Err(error) if error.auth_failed => { + records_changed = true; + + let summary = record.auth_failed_activity_summary(now); + + return Ok(AccountActivityProbeResult { + index: input.index, + record, + summary, + records_changed, + }); + }, Err(error) if error.requires_skip => { let summary = record.probe_failed_activity_summary(now, "failed", &error.source); @@ -567,8 +677,19 @@ impl CodexAccountPool { record.probe_failed_activity_summary(now, "failed", &retry_error), } }, - Err(refresh_error) => - record.probe_failed_activity_summary(now, "failed", refresh_error.as_ref()), + Err(refresh_error) => { + if refresh_error.downcast_ref::().is_some() { + records_changed = true; + + record.auth_failed_activity_summary(now) + } else { + record.probe_failed_activity_summary( + now, + "failed", + refresh_error.as_ref(), + ) + } + }, } }, Err(error) => record.probe_failed_activity_summary(now, "probe_failed", &error), @@ -601,13 +722,17 @@ impl CodexAccountPool { return Err(ProactiveRefreshError { source: ReportableRefreshError::new(format!("missing refresh token for {reason}")), requires_skip: reason.requires_valid_token(), + auth_failed: false, }); } self.refresh_record(record).map(|()| RefreshStatus::Succeeded).map_err(|error| { + let auth_failed = error.downcast_ref::().is_some(); + ProactiveRefreshError { source: ReportableRefreshError::new(error.to_string()), requires_skip: reason.requires_valid_token(), + auth_failed, } }) } @@ -635,7 +760,16 @@ impl CodexAccountPool { })? }; - self.refresh_record(&mut records[record_index])?; + if let Some(auth_failure) = records[record_index].auth_failed_error() { + return Err(Report::new(auth_failure)); + } + if let Err(error) = self.refresh_record(&mut records[record_index]) { + if error.downcast_ref::().is_some() { + self.save_records(records)?; + } + + return Err(error); + } let usage = self.probe_record_usage(&records[record_index])?; let now = OffsetDateTime::now_utc().unix_timestamp(); @@ -775,10 +909,18 @@ impl CodexAccountPool { let status = response.status(); if !status.is_success() { - eyre::bail!( + let reason = format!( "Codex account `{}` token refresh failed with HTTP {status}.", display_name ); + + if token_refresh_auth_status(status) { + record.mark_auth_failed(OffsetDateTime::now_utc().unix_timestamp(), reason.clone()); + + return Err(Report::new(CodexAccountAuthFailure::from_record(record, reason))); + } + + eyre::bail!("{reason}"); } let refresh_response = response.json::()?; @@ -805,6 +947,7 @@ impl CodexAccountPool { record.last_refresh = Some(OffsetDateTime::now_utc().format(&Rfc3339)?); + record.clear_auth_failed(); self.sync_codex_auth_for_refreshed_record(record)?; Ok(()) @@ -961,6 +1104,10 @@ struct AccountPoolRecord { #[serde(skip_serializing_if = "Option::is_none")] last_selected_at_unix_epoch: Option, #[serde(skip_serializing_if = "Option::is_none")] + auth_failed_at_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_failure: Option, + #[serde(skip_serializing_if = "Option::is_none")] auth_mode: Option, #[serde(rename = "OPENAI_API_KEY", skip_serializing_if = "Option::is_none")] openai_api_key: Option, @@ -976,6 +1123,32 @@ impl AccountPoolRecord { .unwrap_or_else(|| String::from("unnamed account")) } + fn account_fingerprint(&self) -> Option { + self.account_id().map(redact_account_id).or_else(|| self.email()) + } + + fn auth_failure(&self) -> Option<&str> { + self.auth_failure + .as_deref() + .map(str::trim) + .filter(|failure| !failure.is_empty()) + .or_else(|| self.auth_failed_at_unix_epoch.map(|_| "authentication failed")) + } + + fn auth_failed_error(&self) -> Option { + self.auth_failure().map(|reason| CodexAccountAuthFailure::from_record(self, reason)) + } + + fn mark_auth_failed(&mut self, now_unix_epoch: i64, reason: impl Into) { + self.auth_failed_at_unix_epoch = Some(now_unix_epoch); + self.auth_failure = Some(reason.into()); + } + + fn clear_auth_failed(&mut self) { + self.auth_failed_at_unix_epoch = None; + self.auth_failure = None; + } + fn matches_account_selector(&self, selector: &str) -> bool { let selector = selector.trim(); @@ -1030,10 +1203,11 @@ impl AccountPoolRecord { &self, now_unix_epoch: i64, ) -> Option { - let account_fingerprint = - self.account_id().map(redact_account_id).or_else(|| self.email())?; + let account_fingerprint = self.account_fingerprint()?; let status = if self.disabled { "disabled" + } else if self.auth_failure().is_some() { + "auth_failed" } else if self .cooldown_until_unix_epoch .is_some_and(|cooldown_until| cooldown_until > now_unix_epoch) @@ -1049,9 +1223,16 @@ impl AccountPoolRecord { account_fingerprint, email: self.email(), status: String::from(status), - refresh_status: String::from("not_checked"), + refresh_status: if self.auth_failure().is_some() { + String::from("auth_failed") + } else { + String::from("not_checked") + }, cooldown_until_unix_epoch: self.cooldown_until_unix_epoch, - note: Some(String::from("configured account")), + note: Some( + self.auth_failure() + .map_or_else(|| String::from("configured account"), ToOwned::to_owned), + ), ..CodexAccountActivitySummary::default() }) } @@ -1167,6 +1348,16 @@ impl AccountPoolRecord { summary } + fn auth_failed_activity_summary(&self, now_unix_epoch: i64) -> CodexAccountActivitySummary { + let mut summary = self.configured_activity_summary(now_unix_epoch).unwrap_or_default(); + + summary.status = String::from("auth_failed"); + summary.refresh_status = String::from("auth_failed"); + summary.note = self.auth_failure().map(ToOwned::to_owned); + + summary + } + fn proactive_refresh_reason(&self, now_unix_epoch: i64) -> Option { let tokens = self.tokens.as_ref()?; @@ -1212,6 +1403,7 @@ struct RefreshResponse { struct ProactiveRefreshError { source: ReportableRefreshError, requires_skip: bool, + auth_failed: bool, } #[derive(Debug)] @@ -1351,6 +1543,10 @@ enum AccountPoolLine { cooldown_until: Option, #[serde(skip_serializing_if = "Option::is_none")] last_selected_at_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_failed_at_unix_epoch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_failure: Option, auth: AuthDotJson, }, Flat(AccountPoolRecord), @@ -1365,6 +1561,8 @@ impl AccountPoolLine { cooldown_until_unix_epoch, cooldown_until, last_selected_at_unix_epoch, + auth_failed_at_unix_epoch, + auth_failure, auth, } => AccountPoolRecord { email: first_nonblank_string(email, auth.email), @@ -1372,6 +1570,8 @@ impl AccountPoolLine { cooldown_until_unix_epoch, cooldown_until, last_selected_at_unix_epoch, + auth_failed_at_unix_epoch, + auth_failure, auth_mode: auth.auth_mode, openai_api_key: auth.openai_api_key, tokens: auth.tokens, @@ -1833,6 +2033,10 @@ fn account_summary_is_limited(summary: &CodexAccountActivitySummary) -> bool { || summary.secondary_remaining_percent == Some(0) } +fn token_refresh_auth_status(status: StatusCode) -> bool { + matches!(status, StatusCode::BAD_REQUEST | StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) +} + fn account_summaries( selected: &CodexAccountLogin, candidates: &[CodexAccountLogin], @@ -1868,9 +2072,10 @@ mod tests { use tempfile::TempDir; use crate::agent::codex_accounts::{ - self, AccountPoolRecord, CodexAccountActivitySummary, CodexAccountLogin, CodexAccountPool, - CodexAccountProvider, CodexTokenData, CreditsSnapshot, DEFAULT_REFRESH_ENDPOINT, Path, - ProactiveRefreshReason, UsageWindow, compare_account_candidates, + self, AccountPoolRecord, CodexAccountActivitySummary, CodexAccountAuthFailure, + CodexAccountLogin, CodexAccountPool, CodexAccountProvider, CodexTokenData, CreditsSnapshot, + DEFAULT_REFRESH_ENDPOINT, Path, ProactiveRefreshReason, UsageWindow, + compare_account_candidates, }; #[test] @@ -1898,6 +2103,8 @@ mod tests { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: Some(String::from("chatgpt")), openai_api_key: None, tokens: Some(CodexTokenData { @@ -1974,6 +2181,91 @@ mod tests { assert_eq!(summaries[0].note.as_deref(), Some("configured account")); } + #[test] + fn selection_marks_refresh_auth_failure_and_selects_next_account() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let accounts_path = temp_dir.path().join("accounts.jsonl"); + let refresh_endpoint = start_codex_refresh_status_fixture_server(vec![( + 401, + "Unauthorized", + r#"{"error":"invalid refresh token"}"#, + )]); + let usage_endpoint = start_codex_usage_fixture_server(vec![ + r#"{"plan_type":"pro","rate_limit":{"primary_window":{"used_percent":0},"secondary_window":{"used_percent":0}}}"#, + ]); + + fs::write( + &accounts_path, + r#"{"email":"bad@example.com","auth_mode":"chatgpt","tokens":{"access_token":"x.eyJleHAiOjEwMDB9.y","refresh_token":"refresh-bad","account_id":"acct_bad"}} +{"email":"good@example.com","auth_mode":"chatgpt","tokens":{"access_token":"access-good","refresh_token":"refresh-good","account_id":"acct_good"}} +"#, + ) + .expect("accounts fixture should write"); + + let pool = CodexAccountPool::new_with_fixed_account( + &accounts_path, + usage_endpoint, + refresh_endpoint, + None, + ) + .expect("account pool should initialize"); + let account = pool.select_account().expect("healthy fallback account should select"); + + assert_eq!(account.account_id(), "acct_good"); + + let records = fs::read_to_string(&accounts_path).expect("accounts should read"); + + assert!(records.contains(r#""auth_failed_at_unix_epoch":"#)); + assert!(records.contains("HTTP 401 Unauthorized")); + + let summaries = pool.account_activity_summaries_snapshot().expect("snapshot should load"); + let bad_summary = summaries + .iter() + .find(|summary| summary.email.as_deref() == Some("bad@example.com")) + .expect("bad account summary should exist"); + + assert_eq!(bad_summary.status, "auth_failed"); + assert_eq!(bad_summary.refresh_status, "auth_failed"); + assert!(bad_summary.note.as_deref().is_some_and(|note| note.contains("HTTP 401"))); + } + + #[test] + fn refresh_account_marks_auth_failure_and_returns_typed_error() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let accounts_path = temp_dir.path().join("accounts.jsonl"); + let refresh_endpoint = start_codex_refresh_status_fixture_server(vec![( + 401, + "Unauthorized", + r#"{"error":"invalid refresh token"}"#, + )]); + let usage_endpoint = start_codex_usage_fixture_server(Vec::new()); + + fs::write( + &accounts_path, + r#"{"email":"bad@example.com","auth_mode":"chatgpt","tokens":{"access_token":"access-bad","refresh_token":"refresh-bad","account_id":"acct_bad"}}"#, + ) + .expect("accounts fixture should write"); + + let pool = CodexAccountPool::new_with_fixed_account( + &accounts_path, + usage_endpoint, + refresh_endpoint, + None, + ) + .expect("account pool should initialize"); + let error = match pool.refresh_account(Some("acct_bad")) { + Ok(_) => panic!("refresh auth failure should surface"), + Err(error) => error, + }; + + assert!(error.downcast_ref::().is_some()); + + let records = fs::read_to_string(&accounts_path).expect("accounts should read"); + + assert!(records.contains(r#""auth_failed_at_unix_epoch":"#)); + assert!(records.contains("HTTP 401 Unauthorized")); + } + #[test] fn usage_summary_parses_codex_rate_limit_payload() { let payload = serde_json::json!({ @@ -2118,6 +2410,8 @@ mod tests { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: Some(String::from("chatgpt")), openai_api_key: None, tokens: Some(CodexTokenData { @@ -2196,6 +2490,8 @@ mod tests { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: Some(String::from("chatgpt")), openai_api_key: None, tokens: Some(CodexTokenData { @@ -2260,6 +2556,8 @@ mod tests { cooldown_until_unix_epoch: None, cooldown_until: None, last_selected_at_unix_epoch: None, + auth_failed_at_unix_epoch: None, + auth_failure: None, auth_mode: Some(String::from("chatgpt")), openai_api_key: None, tokens: Some(CodexTokenData { @@ -2614,4 +2912,29 @@ mod tests { format!("http://{address}/oauth/token") } + + fn start_codex_refresh_status_fixture_server( + responses: Vec<(u16, &'static str, &'static str)>, + ) -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("refresh fixture should bind"); + let address = listener.local_addr().expect("refresh fixture address should resolve"); + + thread::spawn(move || { + for (status, reason, body) in responses { + let (mut stream, _peer) = + listener.accept().expect("refresh fixture should accept request"); + let mut buffer = [0_u8; 4_096]; + let _bytes_read = stream.read(&mut buffer).expect("refresh request should read"); + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + + stream.write_all(response.as_bytes()).expect("refresh response should write"); + } + }); + + format!("http://{address}/oauth/token") + } } diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 353890b0e..9ddbdfa1a 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -1,4 +1,5 @@ use git_credentials::GitSigningConfig; +use agent::CodexAccountAuthFailure; use agent::CodexAccountPool; use agent::CodexAccountProvider; use agent::{AppServerThreadArchiveOutcome, AppServerThreadArchiveRequest}; @@ -511,6 +512,7 @@ pub(crate) fn run_failure_writeback_disposition( .downcast_ref::() .is_some_and(|failure| !failure.is_retryable_timeout()) || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() || error .downcast_ref::() .is_some_and(|failure| !failure.is_retryable_startup()) @@ -3573,6 +3575,7 @@ fn retained_progress_should_defer_to_terminal_intent(error: &Report) -> bool { || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() } fn retained_progress_source_error_class(error: &Report) -> Option<&'static str> { @@ -3590,6 +3593,8 @@ fn retained_progress_source_error_class(error: &Report) -> Option<&'static str> error.downcast_ref::() { Some(app_server_failure.error_class()) + } else if let Some(account_failure) = error.downcast_ref::() { + Some(account_failure.error_class()) } else if let Some(app_server_failure) = error.downcast_ref::() { diff --git a/apps/decodex/src/orchestrator/selection.rs b/apps/decodex/src/orchestrator/selection.rs index a2efacdf0..f27b62c61 100644 --- a/apps/decodex/src/orchestrator/selection.rs +++ b/apps/decodex/src/orchestrator/selection.rs @@ -280,6 +280,8 @@ fn terminal_failure_comment_details( error.downcast_ref::() { (app_server_failure.error_class(), app_server_failure.terminal_next_action(recovery_gate)) + } else if let Some(account_failure) = error.downcast_ref::() { + (account_failure.error_class(), account_failure.terminal_next_action(recovery_gate)) } else if error.downcast_ref::().is_some() { ( "stalled_run_detected", diff --git a/apps/decodex/src/orchestrator/tests/runtime/failure.rs b/apps/decodex/src/orchestrator/tests/runtime/failure.rs index c048fe230..f1bdc7c63 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/failure.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/failure.rs @@ -9,6 +9,8 @@ use orchestrator::LoopGuardrailStopRequested; use orchestrator::RunFailureWritebackDisposition; use orchestrator::StalledRunNeedsAttention; +use crate::agent::CodexAccountAuthFailure; + fn git_config_value( repo_root: &Path, key: &str, @@ -241,6 +243,15 @@ fn failure_writeback_disposition_marks_terminal_attention_classes() { )), RunFailureWritebackDisposition::TerminalAttention, ), + ( + "codex account auth failure", + Report::new(CodexAccountAuthFailure::new( + Some(String::from("...123456")), + Some(String::from("bad@example.com")), + "Codex account `bad@example.com` token refresh failed with HTTP 401 Unauthorized.", + )), + RunFailureWritebackDisposition::TerminalAttention, + ), ] { assert_eq!( orchestrator::run_failure_writeback_disposition(&error), @@ -1309,6 +1320,15 @@ fn app_server_terminal_failures_preserve_specific_error_classes() { "app_server_zero_evidence_start_failed", "verify `decodex probe stdio://`", ), + ( + Report::new(CodexAccountAuthFailure::new( + Some(String::from("...123456")), + Some(String::from("bad@example.com")), + "Codex account `bad@example.com` token refresh failed with HTTP 401 Unauthorized.", + )), + "codex_account_auth_failed", + "re-login or remove Decodex Codex account", + ), ( Report::new(AppServerPhaseGoalFailure::missing_terminal_path_for_test( PhaseGoalKind::HandoffEvidence, diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 80f0bcfe3..3fcf94569 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -308,7 +308,7 @@ protocol activity durable outside the local operator surface. | Section | Meaning | | --- | --- | -| `Accounts` | Shared Codex account pool and usage table from `~/.codex/decodex/accounts.jsonl` when `[codex.accounts]` is enabled for a project. Account identity can be obscured from the `Account` column header eye without changing the underlying snapshot. The row weight column shows the capacity multiplier used for pool usage estimates: `pro` accounts count as `20x`, and all other plans count as `1x`. Usage probes read Codex `/wham/usage` for window capacity and `/wham/profiles/me` for profile token stats such as lifetime tokens, peak daily tokens, longest task, streaks, and daily token activity. Selecting an account writes the global `[codex.accounts].fixed_account` selector in `~/.codex/decodex/config.toml`; clearing it returns all new account-pool runs to balanced account selection. Account display-name rerolls write `[codex.account_names.offsets]` in the same global config so Decodex App and the dashboard share the privacy-preserving names. Theme, sort, and identity-visibility preferences are client-local presentation state. The selector is global and does not pin a project to an account. | +| `Accounts` | Shared Codex account pool and usage table from `~/.codex/decodex/accounts.jsonl` when `[codex.accounts]` is enabled for a project. Account identity can be obscured from the `Account` column header eye without changing the underlying snapshot. The row weight column shows the capacity multiplier used for pool usage estimates: `pro` accounts count as `20x`, and all other plans count as `1x`. Usage probes read Codex `/wham/usage` for window capacity and `/wham/profiles/me` for profile token stats such as lifetime tokens, peak daily tokens, longest task, streaks, and daily token activity. Refresh authentication failures are persisted as `auth_failed` on the matching account, excluded from later selection, and surfaced with login recovery rather than retry-probe recovery. Selecting an account writes the global `[codex.accounts].fixed_account` selector in `~/.codex/decodex/config.toml`; clearing it returns all new account-pool runs to balanced account selection. Account display-name rerolls write `[codex.account_names.offsets]` in the same global config so Decodex App and the dashboard share the privacy-preserving names. Theme, sort, and identity-visibility preferences are client-local presentation state. The selector is global and does not pin a project to an account. | | `Projects` | Fleet-level project table. The section-level filter toggles between active project work and the full registry. Location is its own compact path column and can be obscured from the location header eye. `Activity` shows a relative timestamp or `-`; `Work` is `running/waiting/attention`. It should not duplicate per-lane details already shown below. | | `Running Lanes` | Active leased or live-executing issue lanes. A lane here is currently owned by this local control plane, or a live process/thread/protocol marker still explains active execution even when the queue lease is not held. It shows issue identity, phase, operation, attempt, queue lease state, execution liveness, thread/protocol status, child-agent activity when captured, phase-goal status when app-server reports it, timing, branch, and worktree. | | `Program Intake` | Read-only Program Intake and Execution Program progress. It shows each active program's public intake summary, status, mapped issue identifiers, summary counts, queue-label action counts, and sparse node diagnostics for queue decisions or held/blocked/stale/attention nodes. It does not expose graph editing, raw node-edge mutation controls, Decision Contract payloads, or private runtime evidence. | diff --git a/docs/spec/app-server.md b/docs/spec/app-server.md index 15ec768c0..ba5b0d16d 100644 --- a/docs/spec/app-server.md +++ b/docs/spec/app-server.md @@ -239,6 +239,11 @@ project-scoped fixed account. Successful selection writes `last_selected_at_unix_epoch` back to the JSONL file, and selection holds a pool-local lock so concurrent run dispatches observe the latest selector state instead of all choosing from the same stale snapshot. +If token refresh returns an authentication or authorization rejection, Decodex writes +`auth_failed_at_unix_epoch` and `auth_failure` to the matching JSONL record, excludes +that account from later selection, reports `status = "auth_failed"` in account +snapshots, and fails any active lane with `codex_account_auth_failed` instead of +retrying through no-diff or contract-boundary recovery. If a turn later fails with `codexErrorInfo = "usageLimitExceeded"`, Decodex treats it as a retryable capacity failure while retry budget remains. The current turn stops immediately, but the next attempt re-enters normal account selection so the pool can diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 1e4e7105f..284371765 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -640,7 +640,7 @@ mutations, or duplicate comment for that logical event. ## Local operational state -The local runtime store is the global Decodex SQLite database for one local installation. It lives at `~/.codex/decodex/runtime.sqlite3`, not inside any registered project checkout or worktree. Every row that belongs to a repo is scoped by `project_id`. Decodex logs live beside that database under `~/.codex/decodex/logs/`, the optional shared Codex account pool lives at `~/.codex/decodex/accounts.jsonl`, global operator config lives at `~/.codex/decodex/config.toml`, bounded local account usage estimates live at `~/.codex/decodex/account-usage-history.jsonl`, and agent-readable derived evidence lives under `~/.codex/decodex/agent-evidence//`; vendor-qualified app-data directories and per-project runtime databases are not part of the runtime contract. Global operator config owns account-pool routing and shared account display-name offsets. Account usage history owns local seven-day display estimates and non-secret account capacity weights only; it does not contain token material and does not decide scheduling. UI-only preferences such as theme, table sorting, and local privacy visibility are not runtime state. +The local runtime store is the global Decodex SQLite database for one local installation. It lives at `~/.codex/decodex/runtime.sqlite3`, not inside any registered project checkout or worktree. Every row that belongs to a repo is scoped by `project_id`. Decodex logs live beside that database under `~/.codex/decodex/logs/`, the optional shared Codex account pool lives at `~/.codex/decodex/accounts.jsonl`, global operator config lives at `~/.codex/decodex/config.toml`, bounded local account usage estimates live at `~/.codex/decodex/account-usage-history.jsonl`, and agent-readable derived evidence lives under `~/.codex/decodex/agent-evidence//`; vendor-qualified app-data directories and per-project runtime databases are not part of the runtime contract. Global operator config owns account-pool routing and shared account display-name offsets. The account pool also owns persisted `auth_failed` refresh-authentication state so scheduling does not route new lanes to accounts that must be re-logged or replaced. Account usage history owns local seven-day display estimates and non-secret account capacity weights only; it does not contain token material and does not decide scheduling. UI-only preferences such as theme, table sorting, and local privacy visibility are not runtime state. Project contracts live outside registered repositories under `~/.codex/decodex/projects//`. Each project directory must contain `project.toml` and `WORKFLOW.md`; arbitrary project file names such as `.toml` are not part of the contract. `project.toml` must set `[paths].repo_root` so the project contract is explicit. The `[github]` table owns the routed token environment variable and may also set `command_path` when the expected `gh` binary should be explicit for GUI-launched runs. The `[codex]` table owns app-server-adjacent runtime policy such as `review`. `review` accepts `"off"`, `"basic"`, `"standard"`, and `"strict"` and defaults to `"strict"` when omitted. Phase-scoped goal support is mandatory and is not project-configurable. Project registration stores the centralized `config_path`, target `repo_root`, `worktree_root`, and workflow path in the global runtime database. Commands that start inside a registered checkout or lane worktree resolve the project through that registry; they do not discover or trust worktree-local config files. Project config refreshes preserve an existing enabled or disabled registry toggle; only explicit operator commands such as `decodex project add `, `decodex project enable `, and `decodex project disable ` may change that toggle. `decodex serve` schedules and polls enabled registered projects from the global runtime database; the operator and App projections must still expose active runtime DB-backed attempts for disabled projects because pause is a future-dispatch control, not a visibility or ownership deletion. It must not scan `.codex` history, repo-local config files, or currently open worktrees to infer additional projects.