diff --git a/Cargo.lock b/Cargo.lock index b008115..49b91f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1975,6 +1975,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.1", + "socket2", "sqlx", "tempfile", "thiserror", diff --git a/crates/outer/Cargo.toml b/crates/outer/Cargo.toml index 3ebdfc9..b821598 100644 --- a/crates/outer/Cargo.toml +++ b/crates/outer/Cargo.toml @@ -58,7 +58,8 @@ sqlx = { version = "0.9", features = [ "chrono", "uuid", "migrate", -] } + ] } +socket2 = "0.6" tempfile = "3" thiserror = "2" tokio = { version = "1", features = ["full"] } diff --git a/crates/outer/src/deploy/mod.rs b/crates/outer/src/deploy/mod.rs index 7341fc5..8830ec3 100644 --- a/crates/outer/src/deploy/mod.rs +++ b/crates/outer/src/deploy/mod.rs @@ -12,5 +12,5 @@ pub mod pipeline; pub use check::{CheckOutput, CheckStep}; pub use pipeline::{ fail_interrupted, prune_old, scrub_interrupted_injections, DeployKind, DeployOutcome, - DeploySettings, Deployer, PurgeTasks, Step, + DeploySettings, Deployer, DrainRefused, PurgeTasks, Step, }; diff --git a/crates/outer/src/deploy/pipeline.rs b/crates/outer/src/deploy/pipeline.rs index ea2fbba..6690880 100644 --- a/crates/outer/src/deploy/pipeline.rs +++ b/crates/outer/src/deploy/pipeline.rs @@ -40,6 +40,8 @@ use crate::fleet::cgroups; use crate::fleet::manager::FleetStorage; use crate::fleet::sandbox::{self, Sandbox}; use crate::git::manifest::{self, GitConfig}; +use crate::handoff; +pub use crate::handoff::DrainRefused; use crate::moderation::gate; use crate::moderation::jev::JevClient; use crate::store::Store; @@ -388,6 +390,10 @@ pub struct Deployer { /// operator CLI) sets the boot-installed handle; the deploy entries /// refuse builds while a required policy is missing. egress: std::sync::RwLock, + /// Set on SIGTERM (drain in-flight deploys, refuse new ones) and read + /// alongside the handoff flag file, so a release handoff refuses new + /// deploys on both the old and the new process. + draining: std::sync::Arc, } /// Edge purges still in flight. Each runs detached so `deploy` never waits @@ -649,6 +655,7 @@ impl Deployer { app_locks: Mutex::new(HashMap::new()), purges: PurgeTasks::default(), egress: std::sync::RwLock::new(crate::fleet::egress::EgressFirewall::disabled()), + draining: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -674,6 +681,27 @@ impl Deployer { self.purges.clone() } + /// Mark this process draining: new deploys refuse with [`DrainRefused`] + /// while in-flight ones run to a terminal row. The shutdown path sets + /// this on SIGTERM; the handoff flag file covers the newcomer. + pub fn set_draining(&self) { + self.draining + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + /// True while this process drains (SIGTERM) or a release handoff is + /// open (flag file): no new deploy may start here. + pub fn is_draining(&self) -> bool { + self.draining.load(std::sync::atomic::Ordering::SeqCst) + || handoff::is_active(&self.settings.data_dir) + } + + /// The drain flag for the shutdown path: setting it refuses new deploys + /// while the process finishes in-flight ones before exiting. + pub fn drain_handle(&self) -> std::sync::Arc { + self.draining.clone() + } + /// Deploy `sha` of `app`: row `queued` → `building` → `reviewing` → /// `deploying` → `success`|`failed`|`blocked`. A blocked deploy writes /// `status = 'blocked'` with findings JSON, leaves `last_deploy_sha` @@ -689,6 +717,9 @@ impl Deployer { app: &App, sha: &str, ) -> anyhow::Result { + if self.is_draining() { + return Err(DrainRefused.into()); + } // Fail closed: no build runs as the app uid while the required // policy is missing. self.egress() @@ -712,6 +743,9 @@ impl Deployer { app: &App, sha: &str, ) -> anyhow::Result { + if self.is_draining() { + return Err(DrainRefused.into()); + } // Fail closed: no build runs as the app uid while the required // policy is missing. self.egress() @@ -3854,6 +3888,61 @@ mod tests { ); } + /// A release handoff or SIGTERM drain refuses new deploys retryably + /// before any row is written; a fresh deployer passes the same gate. + #[tokio::test] + async fn draining_refuses_new_deploys_without_a_row() { + let data = tempfile::tempdir().unwrap(); + let deployer = Deployer::new(test_settings(data.path().to_path_buf()), 1); + let (_ddir, pool) = test_pool().await; + let app = test_app(10001); + let git = crate::git::manifest::GitConfig::new(data.path().join("git"), None); + let sha = "a".repeat(40); + assert!( + !deployer.is_draining(), + "a fresh deployer with no flag file drains nothing" + ); + deployer.set_draining(); + assert!(deployer.is_draining(), "SIGTERM marks the drain"); + let err = deployer + .deploy(&pool, &Store::memory(), &git, &app, &sha) + .await + .expect_err("a draining deployer refuses"); + assert!( + err.is::(), + "the refusal is typed for a retryable mapping: {err:?}" + ); + let (rows,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM deploy") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 0, "a refused deploy writes no row"); + } + + /// The handoff flag file alone drains the deployer: the newcomer + /// refuses new deploys while the old process finishes its in-flight + /// ones, with no signal involved. + #[tokio::test] + async fn handoff_flag_file_drains_the_deployer() { + let data = tempfile::tempdir().unwrap(); + let deployer = Deployer::new(test_settings(data.path().to_path_buf()), 1); + let since_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + .to_string(); + std::fs::write(data.path().join(crate::handoff::FLAG), since_epoch).unwrap(); + assert!( + deployer.is_draining(), + "a fresh flag file refuses new deploys" + ); + std::fs::remove_file(data.path().join(crate::handoff::FLAG)).unwrap(); + assert!( + !deployer.is_draining(), + "a cleared handoff lets deploys through again" + ); + } + #[tokio::test] async fn failed_deploy_issues_no_purge() { let (_mdir, cfg, store, _sync, _sha_a, sha_b) = seed_mirror("grocery").await; diff --git a/crates/outer/src/handoff.rs b/crates/outer/src/handoff.rs new file mode 100644 index 0000000..8a99373 --- /dev/null +++ b/crates/outer/src/handoff.rs @@ -0,0 +1,377 @@ +//! Zero-downtime release handoff (loopback flag file + deploy drain). +//! +//! Releases hand a live `outer` to a newcomer with no 443 blackout and no +//! killed in-flight deploys: the newcomer binds the same port with +//! `SO_REUSEPORT` while the old container is still serving, then the old +//! container drains and exits. The two processes share one SQLite file and +//! one bucket claim, so the handoff needs exactly-one-writer discipline: +//! +//! - [`is_active`]: true while `release.sh` holds the handoff open. A fresh +//! boot under an active handoff skips [`fail_interrupted`] bookkeeping +//! (the old process still owns its in-flight deploy rows), refuses new +//! deploys with a retryable error, and delays the background writers +//! (cron, backups, sweeps) until the flag clears. The snapshot sync and +//! the bucket-owner fence keep running: the newcomer claims at boot and +//! the old process stands down on its next tick. +//! - [`in_flight`]: deploy rows no terminal state has claimed. The old +//! process waits for zero before exiting on SIGTERM, so no agent deploy + +//! dies mid-release. +//! +//! The flag file holds the heartbeat epoch the release script refreshes +//! while the handoff is open. A file older than [`FRESH_SECS`] reads as +//! inactive (and is removed best-effort): a crashed release heals itself +//! instead of refusing deploys forever. Recovery is `rm /data/handoff`. +//! +//! [`fail_interrupted`]: crate::deploy::fail_interrupted + +use anyhow::Context as _; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use sqlx::SqlitePool; + +/// Bind `listen` with `SO_REUSEPORT` (and `SO_REUSEADDR`): during a release +/// handoff the newcomer binds 443 while the old container is still serving, +/// and the kernel splits new connections between them. Either process +/// exiting never drops a SYN — at least one socket stays bound throughout +/// the swap — and the overlap needs no proxy component or fd passing. +pub fn bind_reuseport(listen: std::net::SocketAddr) -> anyhow::Result { + let domain = if listen.is_ipv4() { + socket2::Domain::IPV4 + } else { + socket2::Domain::IPV6 + }; + let socket = socket2::Socket::new(domain, socket2::Type::STREAM, None)?; + socket.set_reuse_address(true)?; + #[cfg(unix)] + socket.set_reuse_port(true)?; + if listen.is_ipv6() { + socket.set_only_v6(false)?; + } + socket + .bind(&listen.into()) + .with_context(|| format!("bind {listen}"))?; + // Tokio (both `axum::serve` and axum-server's `from_tcp`) takes this + // socket over and requires non-blocking mode; a blocking listener is a + // boot error there, not a slowdown. + socket.set_nonblocking(true)?; + socket.listen(1024)?; + Ok(socket.into()) +} + +/// The flag file name inside the data dir (`release.sh` owns it). +pub const FLAG: &str = "handoff"; + +/// A handoff flag older than this many seconds reads as inactive: the +/// release script refreshes the heartbeat every minute, so anything older +/// is a crashed release, not an open handoff. +pub const FRESH_SECS: i64 = 180; + +/// What a refused deploy tells the caller to wait before retrying. The +/// handoff window is minutes (in-flight deploys plus the container swap), +/// so one minute backs the retry past most of it without a long stall. +pub const DRAIN_RETRY_AFTER_S: u64 = 60; + +/// Refused while a release handoff is open (or this process is draining): +/// the agent committed nothing, burned nothing, and should retry shortly. +#[derive(Debug, thiserror::Error)] +#[error("outer is handing off to a new release; retry the deploy shortly")] +pub struct DrainRefused; + +pub fn path(data_dir: &Path) -> PathBuf { + data_dir.join(FLAG) +} + +fn now_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// True while a release handoff is open: the flag file exists and its +/// heartbeat is fresh. A stale file reads as inactive and is removed +/// best-effort, so a crashed release heals itself. +pub fn is_active(data_dir: &Path) -> bool { + let raw = match std::fs::read_to_string(path(data_dir)) { + Ok(raw) => raw, + Err(_) => return false, + }; + let marked: i64 = match raw.trim().parse() { + Ok(marked) => marked, + Err(_) => { + let _ = std::fs::remove_file(path(data_dir)); + return false; + } + }; + if now_secs() - marked < FRESH_SECS { + return true; + } + let _ = std::fs::remove_file(path(data_dir)); + false +} + +/// Deploy rows no terminal state has claimed (`queued`, `building`, +/// `reviewing`, `deploying`, or any future non-terminal state): the same +/// predicate [`fail_interrupted`] closes, inverted. +/// +/// [`fail_interrupted`]: crate::deploy::fail_interrupted +pub async fn in_flight(pool: &SqlitePool) -> anyhow::Result { + let (count,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM deploy WHERE status NOT IN ('success', 'failed', 'blocked')", + ) + .fetch_one(pool) + .await?; + Ok(count.max(0) as u64) +} + +/// How the handoff ended: the release script removed the flag after the old +/// process exited (`Cleared`), or the heartbeat went stale and this process +/// reaped it (`Stale`, likely a crashed release with the old process still +/// alive — its in-flight deploy rows may still be live work). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HandoffEnd { + Cleared, + Stale, +} + +/// Wait until no handoff is open, polling every five seconds. Returns at +/// once when none is (or the flag went stale): the normal boot path never +/// blocks here. A missing flag is `Cleared`; a stale one is reaped and +/// reports `Stale`. +pub async fn wait_until_clear(data_dir: &std::path::Path) -> HandoffEnd { + loop { + let raw = std::fs::read_to_string(path(data_dir)); + let marked: Option = raw.ok().and_then(|text| text.trim().parse().ok()); + match marked { + None => return HandoffEnd::Cleared, + Some(marked) if now_secs() - marked < FRESH_SECS => {} + _ => { + let _ = std::fs::remove_file(path(data_dir)); + return HandoffEnd::Stale; + } + } + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } +} + +/// First-signal shutdown gate: mark this process draining (new deploys +/// refuse with [`DrainRefused`]) and wait for in-flight deploys to reach a +/// terminal row, so a release handoff — or any stop — never kills an agent +/// deploy mid-flight. `cancel` is the second signal: a wedged deploy must +/// not wedge the stop, so it exits the process at once (the next boot's +/// `fail_interrupted` closes whatever is left behind). +pub async fn drain_and_wait( + pool: &SqlitePool, + drain: Option>, + cancel: impl Future, +) { + let mut cancel = Box::pin(cancel); + if let Some(drain) = drain { + drain.store(true, std::sync::atomic::Ordering::SeqCst); + } + loop { + match in_flight(pool).await { + Ok(0) => return, + Ok(n) => { + tracing::info!(n, "draining: waiting for in-flight deploys before shutdown"); + } + Err(err) => { + tracing::warn!( + error = %format!("{err:#}"), + "drain poll failed; shutting down anyway" + ); + return; + } + } + tokio::select! { + () = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} + () = &mut cancel => { + tracing::warn!("second shutdown signal: exiting with deploys in flight"); + std::process::exit(1); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dir() -> tempfile::TempDir { + tempfile::tempdir().unwrap() + } + + #[test] + fn missing_flag_is_inactive() { + assert!(!is_active(dir().path()), "no flag file means no handoff"); + } + + #[test] + fn fresh_flag_is_active() { + let dir = dir(); + std::fs::write(path(dir.path()), now_secs().to_string()).unwrap(); + assert!(is_active(dir.path()), "a fresh heartbeat means handoff"); + } + + #[test] + fn stale_flag_heals_itself() { + let dir = dir(); + std::fs::write(path(dir.path()), (now_secs() - FRESH_SECS - 1).to_string()).unwrap(); + assert!( + !is_active(dir.path()), + "a stale heartbeat is a crashed release, not a handoff" + ); + assert!( + !path(dir.path()).exists(), + "the stale flag is removed so deploys resume" + ); + } + + #[test] + fn unparseable_flag_heals_itself() { + let dir = dir(); + std::fs::write(path(dir.path()), "not-an-epoch").unwrap(); + assert!(!is_active(dir.path()), "garbage never blocks deploys"); + assert!(!path(dir.path()).exists(), "garbage is removed"); + } + + #[test] + fn boundary_heartbeat_is_stale() { + let dir = dir(); + std::fs::write(path(dir.path()), (now_secs() - FRESH_SECS).to_string()).unwrap(); + assert!( + !is_active(dir.path()), + "exactly FRESH_SECS old is stale, not active" + ); + } + + #[tokio::test] + async fn in_flight_counts_only_non_terminal_rows() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::query( + "CREATE TABLE deploy (id TEXT PRIMARY KEY, app_id TEXT, sha TEXT, status TEXT, created_at INTEGER)", + ) + .execute(&pool) + .await + .unwrap(); + for (id, status) in [ + ("d1", "queued"), + ("d2", "building"), + ("d3", "reviewing"), + ("d4", "deploying"), + ("d5", "success"), + ("d6", "failed"), + ("d7", "blocked"), + ] { + sqlx::query("INSERT INTO deploy (id, app_id, sha, status, created_at) VALUES (?, 'app1', 'abc', ?, 0)") + .bind(id) + .bind(status) + .execute(&pool) + .await + .unwrap(); + } + assert_eq!( + in_flight(&pool).await.unwrap(), + 4, + "only the non-terminal rows block the handoff" + ); + } + + /// The handoff overlap: two processes bind the same port at once, and a + /// connection to it succeeds while both are bound. + #[test] + fn reuseport_binds_twice_and_serves() { + let first = bind_reuseport("127.0.0.1:0".parse().unwrap()).unwrap(); + assert!( + socket2::SockRef::from(&first).nonblocking().unwrap(), + "Tokio takes this socket over and requires non-blocking mode" + ); + let port = first.local_addr().unwrap().port(); + let second = bind_reuseport(std::net::SocketAddr::from(([127, 0, 0, 1], port))).unwrap(); + assert_eq!( + second.local_addr().unwrap().port(), + port, + "the newcomer shares the port instead of failing with EADDRINUSE" + ); + let conn = std::net::TcpStream::connect_timeout( + &std::net::SocketAddr::from(([127, 0, 0, 1], port)), + std::time::Duration::from_secs(5), + ); + assert!(conn.is_ok(), "a SYN lands while both are bound"); + } + + #[tokio::test] + async fn wait_returns_cleared_without_a_flag() { + assert_eq!( + wait_until_clear(dir().path()).await, + HandoffEnd::Cleared, + "no flag means no wait" + ); + } + + #[tokio::test] + async fn wait_reaps_a_stale_flag() { + let dir = dir(); + std::fs::write(path(dir.path()), (now_secs() - FRESH_SECS - 1).to_string()).unwrap(); + assert_eq!( + wait_until_clear(dir.path()).await, + HandoffEnd::Stale, + "a stale flag unblocks the sweeps but is reported" + ); + assert!(!path(dir.path()).exists(), "the stale flag is reaped"); + } + + async fn deploy_table(pool: &SqlitePool) { + sqlx::query( + "CREATE TABLE deploy (id TEXT PRIMARY KEY, app_id TEXT, sha TEXT, status TEXT, created_at INTEGER)", + ) + .execute(pool) + .await + .unwrap(); + } + + #[tokio::test] + async fn drain_waits_for_in_flight_then_returns() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + deploy_table(&pool).await; + sqlx::query("INSERT INTO deploy (id, app_id, sha, status, created_at) VALUES ('d1', 'app1', 'abc', 'building', 0)") + .execute(&pool) + .await + .unwrap(); + let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flip = pool.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + sqlx::query("UPDATE deploy SET status = 'success' WHERE id = 'd1'") + .execute(&flip) + .await + .unwrap(); + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + drain_and_wait(&pool, Some(flag.clone()), std::future::pending()), + ) + .await + .expect("the drain returns once the deploy lands"); + assert!( + flag.load(std::sync::atomic::Ordering::SeqCst), + "the drain marks the process so new deploys refuse" + ); + assert_eq!(in_flight(&pool).await.unwrap(), 0); + } + + #[tokio::test] + async fn drain_returns_at_once_when_idle() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + deploy_table(&pool).await; + tokio::time::timeout( + std::time::Duration::from_secs(10), + drain_and_wait(&pool, None, std::future::pending()), + ) + .await + .expect("an idle drain never blocks"); + } +} diff --git a/crates/outer/src/lib.rs b/crates/outer/src/lib.rs index 80635c0..7d8b2c6 100644 --- a/crates/outer/src/lib.rs +++ b/crates/outer/src/lib.rs @@ -21,6 +21,7 @@ pub mod deploy; pub mod examples; pub mod fleet; pub mod git; +pub mod handoff; pub mod http; pub mod inactivity; pub mod limits; diff --git a/crates/outer/src/main.rs b/crates/outer/src/main.rs index babda99..8eff71a 100644 --- a/crates/outer/src/main.rs +++ b/crates/outer/src/main.rs @@ -6,7 +6,7 @@ //! The `app` and `templates` commands run app operations against the same //! database and bucket; `serve` picks their results up (it reads app rows per //! request, and fleets adopt new deploys on reload or their own poll). - +//! use std::future::Future; use std::net::SocketAddr; use std::process::ExitCode; @@ -583,26 +583,42 @@ async fn serve(config: Config) -> anyhow::Result<()> { tracing::info!("restored platform state from the bucket snapshot"); } let pool = db::open(&config.db_path()).await?; - let interrupted = outer::deploy::fail_interrupted(&pool).await?; - if interrupted > 0 { - tracing::warn!( - interrupted, - "closed deploy rows a restart cut off (marked failed)" - ); - } - // A crash mid-upload leaves injected plaintext in the worktree: put the - // stashed pre-inject bytes back and delete the sidecars (next boot - // retries whatever still fails the symlink check). - let scrubbed = outer::deploy::scrub_interrupted_injections(&config.data_dir)?; - if scrubbed > 0 { + // A release handoff shares this SQLite file with the still-serving old + // process: its in-flight deploy rows are live work, not crash leftovers. + // Skip the restart bookkeeping (and the secret-injection scrub, which + // would race the old process's running deploys) until the handoff + // clears; the next handoff-free boot closes whatever is left behind. + if outer::handoff::is_active(&config.data_dir) { tracing::warn!( - scrubbed, - "restored deploy configs a restart cut off mid secret-injection" + "handoff flag is active: skipping interrupted-deploy bookkeeping, \ + refusing new deploys and delaying background writers until it clears" ); + } else { + let interrupted = outer::deploy::fail_interrupted(&pool).await?; + if interrupted > 0 { + tracing::warn!( + interrupted, + "closed deploy rows a restart cut off (marked failed)" + ); + } + // A crash mid-upload leaves injected plaintext in the worktree: put the + // stashed pre-inject bytes back and delete the sidecars (next boot + // retries whatever still fails the symlink check). + let scrubbed = outer::deploy::scrub_interrupted_injections(&config.data_dir)?; + if scrubbed > 0 { + tracing::warn!( + scrubbed, + "restored deploy configs a restart cut off mid secret-injection" + ); + } } // Bind before claiming the bucket: a second instance that can't listen - // must not take ownership away from the one that's serving. - let listener = tokio::net::TcpListener::bind(config.listen).await?; + // must not take ownership away from the one that's serving. The bind is + // SO_REUSEPORT, so during a release handoff the newcomer shares 443 with + // the old container and the kernel splits new connections; the claim + // below then moves the snapshot-owner fence to the newcomer, and the old + // process stands down on its next tick. + let listener = outer::handoff::bind_reuseport(config.listen)?; // Read the TLS triple right after binding (still before the bucket // claim): an unreadable cert must fail the boot, never the claim. let tls_files = match (&config.tls_cert, &config.tls_key, &config.origin_pull_ca) { @@ -671,20 +687,6 @@ async fn serve(config: Config) -> anyhow::Result<()> { } let git = git_config(&config); let git_sync = outer::git::GitSync::default(); - outer::cron::spawn_cron_task( - pool.clone(), - fleets.clone(), - outer::cron::TICK_EVERY, - outer::cron::WAKE_LEAD, - ); - // Daily D1 backups (wave 3, feature #4): dump each active app's - // databases to `outer/backups/{slug}/` and prune past retention. - outer::d1_backup::spawn_backup_task( - pool.clone(), - store.clone(), - fleets.clone(), - outer::d1_backup::TICK_EVERY, - ); let secrets_key = outer::secrets::SecretsKey::derive(&config.outer_secret); // One client for the pipeline gate and the slug/name check alike: both @@ -763,74 +765,10 @@ async fn serve(config: Config) -> anyhow::Result<()> { config.outer_secret.as_bytes().to_vec(), )?, }; - // Past-due grace (spec §14.2): day-5 reminders and the downgrade at - // day 7, plus the downgrade-retention pass (day-23 warnings, day-30 - // purges). A missed tick only delays mail and the downgrade; the next - // tick catches up. - let retention = outer::billing::RetentionDeps { - store: store.clone(), - git: git.clone(), - sync: git_sync.clone(), - fleets: fleets.clone(), - deploys: mcp.deploys.clone(), - rollback_locks: mcp.rollback_locks.clone(), - delete_cfg: mcp.delete_cfg.clone(), - }; - outer::billing::spawn_grace_sweep( - mcp.billing.clone(), - pool.clone(), - mcp.auth.config.email.clone(), - mcp.egress.clone(), - retention.clone(), - ); - // Sleep, preview-expiry, keep-warm, log flush and prefix-cleanup ticks - // (every minute) with the daily maintenance block: retention prunes - // plus the inactivity sweep (spec §7.8: reminders, expiry, restores, - // purges). Spawned after the MCP state so the daily block owns the - // delete handles and the mail handles. - fleets.spawn_sleep_sweep( - pool.clone(), - store.clone(), - git.clone(), - git_sync.clone(), - Duration::from_secs(60.min(rt.sleep_after_secs.max(1))), - outer::inactivity::SweepDeps { - sender: mcp.auth.config.email.clone(), - issuer: mcp.auth.config.issuer.clone(), - unsub_key: mcp.unsub_key.clone(), - retention: retention.clone(), - windows: outer::inactivity::Windows::new(rt.inactive_days, rt.inactive_grace_days), - }, - ); - // Email alerts (wave 3, feature #2): error spikes are evaluated from - // `error_issue` every five minutes; deploy failures and quota - // thresholds notify inline at their trigger sites. - outer::alerts::spawn_sweep( - pool.clone(), - mcp.auth.config.email.clone(), - mcp.auth.config.issuer.clone(), - mcp.unsub_key.clone(), - std::time::Duration::from_secs(300), - ); - // Request metrics (spec §13): owned by the proxy state, flushed to - // SQLite every few seconds by the sweep below (and once more on - // shutdown), pruned per plan by the same sweep. - let metrics = outer::observe::metrics::MetricsAgg::new(); - let metrics_sweep = outer::observe::metrics::spawn_sweep(pool.clone(), metrics.clone()); - // Plan limits (spec §3, owner-set 2026-10-26): hourly request rollup + - // soft-cap email and storage freeze. A missed tick only delays mail and - // flags; the rollup watermark never double-counts. - outer::limits::spawn_limits_sweep(outer::limits::LimitsSweep { - pool: pool.clone(), - store: store.clone(), - sender: mcp.auth.config.email.clone(), - issuer: mcp.auth.config.issuer.clone(), - unsub_key: mcp.unsub_key.clone(), - billing: mcp.billing.clone(), - }); // After-live moderation (spec §15.2): the proxy's report path and the // periodic sweep share the Jev client, the crawl config and the effect - // deps (mailbox, fleet handle, edge purge). + // deps (mailbox, fleet handle, edge purge). The sweep itself starts in + // the waiter below; the hooks serve requests from boot. let crawl_effects = outer::moderation::crawl::EffectDeps { sender: mcp.auth.config.email.clone(), apps_domain: rt.apps_domain.clone(), @@ -842,12 +780,157 @@ async fn serve(config: Config) -> anyhow::Result<()> { cfg: config.crawl.clone(), effects: crawl_effects.clone(), }; - outer::moderation::crawl::spawn_sweep(outer::moderation::crawl::SweepDeps { - pool: pool.clone(), - cfg: config.crawl.clone(), - jev: mcp.jev.clone(), - effects: crawl_effects, - }); + // Background writers wait out a release handoff: while the flag file is + // fresh the old process still owns its fleets and runs its own sweeps, + // so the newcomer must not wake, mail, prune or back up twice. Request + // serving, metrics and the snapshot sync run from boot; this waiter only + // delays the writers. Returns at once when no handoff is open. + { + let data_dir = config.data_dir.clone(); + let sweep_data_dir = data_dir.clone(); + let sweep_pool = pool.clone(); + let sweep_store = store.clone(); + let sweep_fleets = fleets.clone(); + let sweep_git = git.clone(); + let sweep_sync = git_sync.clone(); + let sweep_billing = mcp.billing.clone(); + let sweep_email = mcp.auth.config.email.clone(); + let sweep_issuer = mcp.auth.config.issuer.clone(); + let sweep_unsub = mcp.unsub_key.clone(); + let sweep_egress = mcp.egress.clone(); + let sweep_deploys = mcp.deploys.clone(); + let sweep_locks = mcp.rollback_locks.clone(); + let sweep_delete = mcp.delete_cfg.clone(); + let sweep_jev = mcp.jev.clone(); + let sweep_crawl = config.crawl.clone(); + let sweep_effects = crawl_effects.clone(); + let sleep_every = Duration::from_secs(60.min(rt.sleep_after_secs.max(1))); + let sweep_windows = + outer::inactivity::Windows::new(rt.inactive_days, rt.inactive_grace_days); + tokio::spawn(async move { + match outer::handoff::wait_until_clear(&data_dir).await { + // The old process exited before the flag cleared: any + // non-terminal deploy row is a kill leftover (the normal + // SIGTERM drain leaves none), so close it here instead of + // at boot, when the old process still owned those rows. + outer::handoff::HandoffEnd::Cleared => { + match outer::deploy::fail_interrupted(&sweep_pool).await { + Ok(0) => {} + Ok(closed) => tracing::warn!( + closed, + "closed deploy rows the handoff cut off (marked failed)" + ), + Err(err) => tracing::warn!( + error = %format!("{err:#}"), + "handoff catch-up bookkeeping failed" + ), + } + if let Err(err) = outer::deploy::scrub_interrupted_injections(&sweep_data_dir) { + tracing::warn!( + error = %format!("{err:#}"), + "handoff catch-up scrub failed" + ); + } + } + outer::handoff::HandoffEnd::Stale => { + tracing::warn!( + "handoff flag went stale (crashed release?): sweeps start, \ + but check for a stray old container before releasing again" + ); + } + } + outer::cron::spawn_cron_task( + sweep_pool.clone(), + sweep_fleets.clone(), + outer::cron::TICK_EVERY, + outer::cron::WAKE_LEAD, + ); + // Daily D1 backups (wave 3, feature #4): dump each active app's + // databases to `outer/backups/{slug}/` and prune past retention. + outer::d1_backup::spawn_backup_task( + sweep_pool.clone(), + sweep_store.clone(), + sweep_fleets.clone(), + outer::d1_backup::TICK_EVERY, + ); + // Past-due grace (spec §14.2): day-5 reminders and the downgrade + // at day 7, plus the downgrade-retention pass (day-23 warnings, + // day-30 purges). A missed tick only delays mail and the + // downgrade; the next tick catches up. + let retention = outer::billing::RetentionDeps { + store: sweep_store.clone(), + git: sweep_git.clone(), + sync: sweep_sync.clone(), + fleets: sweep_fleets.clone(), + deploys: sweep_deploys.clone(), + rollback_locks: sweep_locks.clone(), + delete_cfg: sweep_delete.clone(), + }; + outer::billing::spawn_grace_sweep( + sweep_billing.clone(), + sweep_pool.clone(), + sweep_email.clone(), + sweep_egress.clone(), + retention.clone(), + ); + // Sleep, preview-expiry, keep-warm, log flush and prefix-cleanup + // ticks (every minute) with the daily maintenance block: + // retention prunes plus the inactivity sweep (spec §7.8: + // reminders, expiry, restores, purges). + sweep_fleets.spawn_sleep_sweep( + sweep_pool.clone(), + sweep_store.clone(), + sweep_git.clone(), + sweep_sync.clone(), + sleep_every, + outer::inactivity::SweepDeps { + sender: sweep_email.clone(), + issuer: sweep_issuer.clone(), + unsub_key: sweep_unsub.clone(), + retention: retention.clone(), + windows: sweep_windows, + }, + ); + // Email alerts (wave 3, feature #2): error spikes are evaluated + // from `error_issue` every five minutes; deploy failures and + // quota thresholds notify inline at their trigger sites. + outer::alerts::spawn_sweep( + sweep_pool.clone(), + sweep_email.clone(), + sweep_issuer.clone(), + sweep_unsub.clone(), + std::time::Duration::from_secs(300), + ); + // Plan limits (spec §3, owner-set 2026-10-26): hourly request + // rollup + soft-cap email and storage freeze. A missed tick only + // delays mail and flags; the rollup watermark never double-counts. + outer::limits::spawn_limits_sweep(outer::limits::LimitsSweep { + pool: sweep_pool.clone(), + store: sweep_store.clone(), + sender: sweep_email.clone(), + issuer: sweep_issuer.clone(), + unsub_key: sweep_unsub.clone(), + billing: sweep_billing.clone(), + }); + outer::moderation::crawl::spawn_sweep(outer::moderation::crawl::SweepDeps { + pool: sweep_pool.clone(), + cfg: sweep_crawl.clone(), + jev: sweep_jev.clone(), + effects: sweep_effects, + }); + }); + } + // Request metrics (spec §13): owned by the proxy state, flushed to + // SQLite every few seconds by the sweep below (and once more on + // shutdown), pruned per plan by the same sweep. Runs from boot, handoff + // or not: recording is a local SQLite write, never a double-acting + // external effect. + let metrics = outer::observe::metrics::MetricsAgg::new(); + let metrics_sweep = outer::observe::metrics::spawn_sweep(pool.clone(), metrics.clone()); + // The drain flag refuses new deploys from here on: the shutdown path + // sets it on the first signal and waits out the in-flight ones before + // stopping fleets, so a release handoff never kills an agent deploy. + let drain = mcp.deploys.drain_handle(); let app = http::app( http::AppState { pool: pool.clone(), @@ -886,13 +969,15 @@ async fn serve(config: Config) -> anyhow::Result<()> { // handshake drops just that connection and never stops serving. let rustls_config = axum_server::tls_rustls::RustlsConfig::from_config(std::sync::Arc::new(rustls_config)); - let std_listener = listener.into_std()?; + let std_listener = listener; let handle = axum_server::Handle::new(); let shutdown_handle = handle.clone(); + let shutdown_pool = pool.clone(); + let shutdown_drain = drain.clone(); tokio::spawn(async move { - shutdown_signal().await; - // Bounded drain: stop_all and the final snapshot run after - // serve returns, all inside compose's 35 s stop_grace_period. + drain_and_wait(shutdown_pool, shutdown_drain).await; + // Bounded HTTP drain: stop_all and the final snapshot run after + // serve returns, all inside compose's 15 min stop_grace_period. shutdown_handle.graceful_shutdown(Some(std::time::Duration::from_secs(5))); }); axum_server::tls_rustls::from_tcp_rustls(std_listener, rustls_config)? @@ -900,11 +985,12 @@ async fn serve(config: Config) -> anyhow::Result<()> { .serve(app.into_make_service_with_connect_info::()) .await?; } else { + let shutdown_pool = pool.clone(); axum::serve( - listener, + tokio::net::TcpListener::from_std(listener)?, app.into_make_service_with_connect_info::(), ) - .with_graceful_shutdown(shutdown_signal()) + .with_graceful_shutdown(drain_and_wait(shutdown_pool, drain)) .await?; } @@ -947,3 +1033,12 @@ async fn shutdown_signal() { () = terminate => {} } } + +/// First shutdown signal (Ctrl-C/SIGTERM): the lib gate refuses new deploys +/// and waits for in-flight ones to finish, so a release handoff — or any +/// stop — never kills an agent deploy mid-flight. The second signal exits +/// at once; the next boot's `fail_interrupted` closes whatever is left. +async fn drain_and_wait(pool: sqlx::SqlitePool, drain: Option>) { + shutdown_signal().await; + outer::handoff::drain_and_wait(&pool, drain, shutdown_signal()).await; +} diff --git a/crates/outer/src/mcp/mod.rs b/crates/outer/src/mcp/mod.rs index dcd039e..f75bef2 100644 --- a/crates/outer/src/mcp/mod.rs +++ b/crates/outer/src/mcp/mod.rs @@ -186,6 +186,29 @@ impl Deploys { DeploysInner::Fake(fake) => fake.deploy_preview(pool, app, sha).await, } } + + /// True while this process drains (SIGTERM) or a release handoff is + /// open: [`run_deploy`] refuses new deploys retryably up front. The + /// test fake never drains. + /// + /// [`run_deploy`]: crate::mcp::tools::run_deploy + pub fn is_draining(&self) -> bool { + match &*self.inner { + DeploysInner::Live(deployer) => deployer.is_draining(), + #[cfg(test)] + DeploysInner::Fake(_) => false, + } + } + + /// The live pipeline's drain flag for the shutdown path (`None` for the + /// test fake, which never drains). + pub fn drain_handle(&self) -> Option> { + match &*self.inner { + DeploysInner::Live(deployer) => Some(deployer.drain_handle()), + #[cfg(test)] + DeploysInner::Fake(_) => None, + } + } } /// What the fake deploy answers with. #[cfg(test)] diff --git a/crates/outer/src/mcp/tools/add_example.rs b/crates/outer/src/mcp/tools/add_example.rs index 4414161..48b7418 100644 --- a/crates/outer/src/mcp/tools/add_example.rs +++ b/crates/outer/src/mcp/tools/add_example.rs @@ -46,6 +46,8 @@ pub async fn add_example(input: AddExampleInput, ctx: &Ctx<'_>) -> Result) -> Result) -> Result) -> Result Result<(), ToolError> { + if state.deploys.is_draining() { + return Err(ToolError::rate_limited( + "outer is handing off to a new release; retry shortly", + crate::handoff::DRAIN_RETRY_AFTER_S, + )); + } + Ok(()) +} + /// Give back a claimed change when the commit never happened (bad base, /// moved head, storage failure). One guarded decrement; a missing or empty /// row stays empty. @@ -861,6 +875,17 @@ async fn run_deploy_inner( // claims already count their change/rollback; a created app or commit // already exists when this refuses, and the error names the reset). check_build_quota(&state.pool, &state.billing, &app.owner_id).await?; + // Release handoff or SIGTERM drain: the last backstop before a deploy + // row is written or a build is spawned, retryably. Callers that claim + // quota check [`check_no_drain`] first, so this usually fires for + // claim-free paths (seed deploys, promotes, rollbacks, console retries). + // Covers MCP and console alike: both funnel through here. + if state.deploys.is_draining() { + return Err(ToolError::rate_limited( + "outer is handing off to a new release; retry shortly", + crate::handoff::DRAIN_RETRY_AFTER_S, + )); + } check_storage_writable(&state.pool, &state.billing, app).await?; let deploys = state.deploys.clone(); let pool = state.pool.clone(); @@ -980,6 +1005,12 @@ async fn run_deploy_inner( &outcome, kind.is_preview(), )), + Ok(Ok((_, Err(e)))) if e.is::() => { + Err(ToolError::rate_limited( + "outer is handing off to a new release; retry shortly", + crate::handoff::DRAIN_RETRY_AFTER_S, + )) + } Ok(Ok((_, Err(e)))) => Err(internal(e)), Ok(Err(e)) => Err(internal(anyhow::Error::new(e))), Err(_) => Ok(json!({ diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 2e4003e..c0331e1 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -19,10 +19,13 @@ services: image: ${OUTER_IMAGE:?set OUTER_IMAGE in /srv/outer/release.env (deploy/release.sh writes it)} restart: unless-stopped network_mode: host - # Above outer's stop budget: fleets stop concurrently (each at most ~20 s: - # StopGrace::NORMAL is 15 s SIGTERM + 5 s SIGKILL), then outer writes the - # final state snapshot before exiting. - stop_grace_period: 35s + # Above outer's stop budget: on SIGTERM outer refuses new deploys and + # waits out the in-flight ones (a release handoff drains this way), and + # only then stops fleets concurrently (each at most ~20 s: StopGrace is + # 15 s SIGTERM + 5 s SIGKILL) and writes the final state snapshot. Stops + # with no in-flight deploys exit at once; the grace period is a cap, not + # a delay. `docker stop -t` on the command line overrides this per stop. + stop_grace_period: 15m # Tenant isolation: dropping to the app/fleet uids, owning worktrees, the # nft egress policy (needs NET_ADMIN), and per-fleet cgroup leaves. # The cgroup leaves need a writable cgroupfs mount: CAP_SYS_ADMIN lets diff --git a/deploy/handoff.sh b/deploy/handoff.sh new file mode 100755 index 0000000..d26531c --- /dev/null +++ b/deploy/handoff.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# +# Outer zero-downtime handoff helpers. Sourced (not executed) by +# deploy/release.sh and deploy/rollback.sh inside their remote heredocs — +# every function runs on the VPS, where /srv/outer is the state dir. +# +# How a handoff works: the newcomer container binds 443 with SO_REUSEPORT +# while the old one is still serving (the kernel splits new connections), +# the old container drains on SIGTERM (refuses new deploys, exits once +# in-flight ones finish), and the compose service is recreated last onto the +# same image, overlapping the newcomer the same way. At no point is 443 +# without a listener, and no in-flight deploy dies with its process. +# +# Coordination is the handoff flag file (/srv/outer/data/handoff): the +# release script writes the heartbeat epoch into it every minute while the +# handoff is open, and holds it until outer-next is stopped. outer refuses +# new deploys (retryable, 60 s backoff) and delays its background writers +# while the flag is fresh; a flag older than 180 s reads as inactive, so a +# crashed release heals itself. Recovery from a crashed release is +# `rm /srv/outer/data/handoff` plus `docker rm -f outer-next`, then a fresh +# release. +# +# shellcheck disable=SC2034 + +HANDOFF_FILE=/srv/outer/data/handoff +HEARTBEAT_PID="" + +# Write one heartbeat epoch into the flag file, atomically: readers must +# never see a truncated file (an empty read deletes the flag and unblocks +# the newcomer mid-handoff). +handoff_flag() { + date +%s >"$HANDOFF_FILE.tmp" && mv -f "$HANDOFF_FILE.tmp" "$HANDOFF_FILE" +} + +# Open the handoff and keep the heartbeat fresh in the background. The +# subshell's stdio is detached so a lingering sleep never holds the ssh +# channel open after the script exits. +handoff_heartbeat_start() { + handoff_flag + ( + while true; do + sleep 60 + handoff_flag + done + ) >/dev/null 2>&1 /dev/null || true + wait "$HEARTBEAT_PID" 2>/dev/null || true + fi + HEARTBEAT_PID="" +} + +# Close the handoff: stop the heartbeat and remove the flag. Idempotent. +handoff_clear() { + handoff_heartbeat_stop + rm -f "$HANDOFF_FILE" +} + +# Abort a failed handoff: clear the flag, and remove outer-next only while +# a compose-managed outer still serves. Past the old container's removal +# outer-next may be the only listener on 443, so it stays up and the +# operator recovers with a fresh release. +handoff_abort() { + handoff_clear + local old + old="$(docker compose --env-file /srv/outer/release.env \ + -f /srv/outer/compose.yaml ps -q outer 2>/dev/null | head -n 1 || true)" + if [[ -n "$old" ]] && docker logs "$old" 2>&1 | grep -q 'listening on'; then + docker rm -f outer-next >/dev/null 2>&1 || true + else + echo "remote: handoff aborted with no healthy compose outer; outer-next is left running — recover with a fresh release" >&2 + fi +} + +# wait_for_listening : the container is running on +# and outer logged its "listening on" line. mTLS admits no +# unauthenticated HTTP probe, so the log line is the signal (same as the +# old stop-start rollout). 60 s deadline, then failure. +wait_for_listening() { + local name="$1" image="$2" + local cid deadline + deadline=$((SECONDS + 60)) + while true; do + cid="$(docker ps -q -f "name=^${name}$")" + if [[ -n "$cid" ]] \ + && [[ "$(docker inspect -f '{{.State.Running}} {{.Config.Image}}' "$cid")" == "true $image" ]] \ + && docker logs "$cid" 2>&1 | grep -q 'listening on'; then + echo "remote: $name on $image up ($cid)" + return 0 + fi + if [[ $SECONDS -ge $deadline ]]; then + echo "remote: $name did not come up; last log lines:" >&2 + docker logs --tail 50 "$name" 2>&1 || true + return 1 + fi + sleep 2 + done +} + +# In-flight deploys (non-terminal rows) block nothing by themselves — the +# old container drains them on SIGTERM — but an old container that predates +# the drain still dies with them, so wait for an idle moment first (bounded: +# the drain and the newcomer's catch-up cover whatever starts after). +wait_for_idle_deploys() { + local deadline count + deadline=$((SECONDS + 900)) + while true; do + count="$(sqlite3 /srv/outer/data/outer.sqlite \ + "SELECT COUNT(*) FROM deploy WHERE status NOT IN ('success', 'failed', 'blocked');")" + if [[ "$count" == "0" ]]; then + return 0 + fi + if [[ $SECONDS -ge $deadline ]]; then + echo "remote: $count deploy(s) still in flight after 15 min; proceeding (the drain covers them)" >&2 + return 0 + fi + sleep 10 + done +} + +# Record the new image, keeping the previous tag for rollback. Same +# semantics as the old stop-start rollout: re-releasing the running image +# keeps the existing rollback target. +record_image() { + local image="$1" + local prev prev_rollback tmp + touch /srv/outer/release.env + chmod 600 /srv/outer/release.env + prev="$(grep -E '^OUTER_IMAGE=' /srv/outer/release.env | tail -n 1 | cut -d= -f2- || true)" + prev_rollback="$(grep -E '^OUTER_PREVIOUS_IMAGE=' /srv/outer/release.env | tail -n 1 | cut -d= -f2- || true)" + tmp="$(mktemp)" + grep -v -E '^(OUTER_IMAGE|OUTER_PREVIOUS_IMAGE|OUTER_ROLLED_BACK_FROM)=' \ + /srv/outer/release.env >"$tmp" || true + { + cat "$tmp" + echo "OUTER_IMAGE=$image" + if [[ -n "$prev" && "$prev" != "$image" ]]; then + echo "OUTER_PREVIOUS_IMAGE=$prev" + elif [[ -n "$prev_rollback" ]]; then + # Re-releasing the running image keeps the existing rollback target. + echo "OUTER_PREVIOUS_IMAGE=$prev_rollback" + fi + } >/srv/outer/release.env + rm -f "$tmp" +} + +# handoff_to : roll out with no 443 blackout. +handoff_to() { + local image="$1" + local old + docker image inspect "$image" >/dev/null 2>&1 || { + echo "remote: image $image not loaded" >&2 + return 1 + } + wait_for_idle_deploys + handoff_heartbeat_start + docker rm -f outer-next >/dev/null 2>&1 || true + # Same volumes, env, caps and host network as the compose service + # (deploy/compose.yaml): the newcomer is the service, started by hand. + docker run -d --name outer-next --network host \ + --cap-add NET_ADMIN --cap-add SETUID --cap-add SETGID --cap-add CHOWN --cap-add SYS_ADMIN \ + --security-opt apparmor:unconfined \ + --env-file /srv/outer/outer.env \ + -v /srv/outer/data:/data \ + -v /srv/outer/tls:/tls:ro \ + "$image" serve >/dev/null + old="$(docker compose --env-file /srv/outer/release.env \ + -f /srv/outer/compose.yaml ps -q outer)" + [[ -n "$old" ]] || { + echo "remote: no outer container to drain" >&2 + handoff_abort + return 1 + } + if ! wait_for_listening outer-next "$image"; then + # First-handoff fallback: an old container that predates SO_REUSEPORT + # cannot share 443, so the newcomer bind fails with EADDRINUSE. + # Stop/start once (seconds of blackout; the idle wait above keeps + # in-flight deploys off the old build) — every later handoff + # overlaps. + if docker logs outer-next 2>&1 | grep -qi "address already in use" \ + && [[ "$(docker inspect -f '{{.State.Running}}' "$old")" == "true" ]]; then + echo "remote: old container predates SO_REUSEPORT; one-time stop/start fallback" >&2 + docker stop -t 60 "$old" + docker rm "$old" + docker start outer-next >/dev/null + wait_for_listening outer-next "$image" || { + handoff_abort + return 1 + } + old="" + else + handoff_abort + return 1 + fi + fi + # Drain the old container: SIGTERM refuses new deploys and the process + # exits once in-flight ones finish. -t 900 bounds a wedged deploy with + # SIGKILL; the newcomer's handoff catch-up closes whatever is left. + # (Skipped on the fallback path above: the old container is already + # gone.) + if [[ -n "$old" ]]; then + docker stop -t 900 "$old" || { + handoff_abort + return 1 + } + docker rm "$old" || { + handoff_abort + return 1 + } + fi + record_image "$image" + # Recreate the compose service onto the same image: it overlaps the + # newcomer via SO_REUSEPORT (same-version, no schema skew), then the + # newcomer drains and goes away. The flag stays up throughout, so the + # recreated service skips boot bookkeeping and waits out its writers + # until outer-next is gone. Steady state stays compose-managed. + docker compose --env-file /srv/outer/release.env \ + -f /srv/outer/compose.yaml up -d || { + handoff_abort + return 1 + } + wait_for_listening outer-outer-1 "$image" || { + handoff_abort + return 1 + } + docker stop -t 300 outer-next || { + handoff_abort + return 1 + } + docker rm outer-next || { + handoff_abort + return 1 + } + handoff_clear + # Keep the last 3 outer images, prune older (in-use images fail, ignored). + docker images --format '{{.CreatedAt}} {{.Repository}}:{{.Tag}}' \ + | grep ' outer:' | sort -r | awk '{print $NF}' | tail -n +4 \ + | xargs -r docker rmi >/dev/null 2>&1 || true + echo "remote: handed off to $image with no downtime" +} diff --git a/deploy/release.sh b/deploy/release.sh index fd60c14..ed2509b 100755 --- a/deploy/release.sh +++ b/deploy/release.sh @@ -1,19 +1,29 @@ #!/usr/bin/env bash # -# Outer release: build the image from the current HEAD and roll it out. +# Outer release: build the image from the current HEAD and roll it out with +# no downtime (deploy/handoff.sh). # # deploy/release.sh # # Refuses a dirty tree. Prefers podman, falls back to docker. Streams the # image with save|load (no registry), uploads compose.yaml + the firewall -# files to /srv/outer, records OUTER_IMAGE (keeping the previous tag as -# OUTER_PREVIOUS_IMAGE), runs `docker compose up -d`, then health-checks. +# files + handoff.sh to /srv/outer, then hands the new image out: the +# newcomer shares 443 via SO_REUSEPORT while the old container drains +# (refuses new deploys, exits once in-flight ones finish), and the compose +# service is recreated last onto the same image. At no point is 443 without +# a listener, and no in-flight deploy dies with its process. # # Health check from the box: outer serves TLS with required client # certificates (mTLS), so no unauthenticated probe can pass the handshake — # not even `curl --resolve ... https://outer.now/health` from here. The check -# is therefore: the container is running and outer logged its "listening on" -# line. Rollback: deploy/rollback.sh . +# is therefore: the container is running on the new image and outer logged +# its "listening on" line. Rollback: deploy/rollback.sh (same +# handoff, previous image). +# +# Migrations are append-only within a release line: the newcomer runs them +# at boot while the old container is still serving, so a release's +# migrations must be additive (nullable columns, new tables). A destructive +# migration needs an announced maintenance release, not this script. # # No secrets in args or logs (only image tags travel in release.env). @@ -29,7 +39,7 @@ cd "$ROOT" } SHA="$(git rev-parse --short HEAD)" [[ "$SHA" =~ ^[0-9a-f]+$ ]] || { - echo "release.sh: unexpected sha: $SHA" >&2 + echo "release.sh: unexpected short SHA: $SHA" >&2 exit 1 } @@ -41,7 +51,7 @@ SHA="$(git rev-parse --short HEAD)" MIGRATIONS="crates/outer/migrations" APPLIED="$(ssh "$TARGET" "sqlite3 -separator ' ' /srv/outer/data/outer.sqlite \ 'SELECT version, lower(hex(checksum)) FROM _sqlx_migrations ORDER BY version'" 2>/dev/null)" || { - echo "release.sh: could not read applied migrations from $TARGET (sqlite3 on the host?)" >&2 + echo "release.sh: cannot read prod migrations from $TARGET (is outer up?)" >&2 exit 1 } while read -r version checksum; do @@ -73,7 +83,7 @@ else fi ssh "$TARGET" mkdir -p /srv/outer -scp deploy/compose.yaml deploy/firewall.sh deploy/firewall.service \ +scp deploy/compose.yaml deploy/handoff.sh deploy/firewall.sh deploy/firewall.service \ deploy/firewall-refresh.service deploy/firewall-refresh.timer \ "$TARGET:/srv/outer/" @@ -102,26 +112,6 @@ for f in /srv/outer/tls/origin.pem /srv/outer/tls/origin.key /srv/outer/tls/orig } done -# Record the new image, keeping the previous tag for rollback. -touch /srv/outer/release.env -chmod 600 /srv/outer/release.env -prev="$(grep -E '^OUTER_IMAGE=' /srv/outer/release.env | tail -n 1 | cut -d= -f2- || true)" -prev_rollback="$(grep -E '^OUTER_PREVIOUS_IMAGE=' /srv/outer/release.env | tail -n 1 | cut -d= -f2- || true)" -tmp="$(mktemp)" -grep -v -E '^(OUTER_IMAGE|OUTER_PREVIOUS_IMAGE|OUTER_ROLLED_BACK_FROM)=' \ - /srv/outer/release.env >"$tmp" || true -{ - cat "$tmp" - echo "OUTER_IMAGE=outer:$SHA" - if [[ -n "$prev" && "$prev" != "outer:$SHA" ]]; then - echo "OUTER_PREVIOUS_IMAGE=$prev" - elif [[ -n "$prev_rollback" ]]; then - # Re-releasing the running image keeps the existing rollback target. - echo "OUTER_PREVIOUS_IMAGE=$prev_rollback" - fi -} > /srv/outer/release.env -rm -f "$tmp" - # outer runs on the host network: the ingress filter must be in place before # it starts (a first install may predate it). firewall.sh exits nonzero on # failure, which stops the rollout here. @@ -129,38 +119,13 @@ if ! nft list table inet outer_fw >/dev/null 2>&1; then /srv/outer/firewall.sh fi -docker compose --env-file /srv/outer/release.env \ - -f /srv/outer/compose.yaml up -d - - -# Wait for the rollout: container running + outer's boot line. mTLS means no -# HTTP probe is possible without a client key, so the log line is the signal. -cid="$(docker compose --env-file /srv/outer/release.env \ - -f /srv/outer/compose.yaml ps -q outer)" -[[ -n "$cid" ]] || { - echo "remote: no outer container after up" >&2 - exit 1 -} -deadline=$((SECONDS + 60)) -while true; do - # The image check catches a compose that kept the old container. - if [[ "$(docker inspect -f '{{.State.Running}} {{.Config.Image}}' "$cid")" == "true outer:$SHA" ]] \ - && docker logs "$cid" 2>&1 | grep -q 'listening on'; then - break - fi - if [[ $SECONDS -ge $deadline ]]; then - echo "remote: outer did not come up; last log lines:" >&2 - docker logs --tail 50 "$cid" >&2 || true - exit 1 - fi - sleep 2 -done -echo "remote: outer:$SHA up ($cid)" - -# Keep the last 3 outer images, prune older (in-use images fail, ignored). -docker images --format '{{.CreatedAt}} {{.Repository}}:{{.Tag}}' \ - | grep ' outer:' | sort -r | awk '{print $NF}' | tail -n +4 \ - | xargs -r docker rmi >/dev/null 2>&1 || true +# shellcheck disable=SC1091 +source /srv/outer/handoff.sh +# Whatever fails below, the heartbeat must not outlive the release: abort +# clears the flag and removes outer-next only while a compose outer serves. +trap 'handoff_abort' EXIT +handoff_to "outer:$SHA" +trap - EXIT EOF echo "released $IMAGE to $TARGET" diff --git a/deploy/rollback.sh b/deploy/rollback.sh index 6573f15..8f4dd4e 100755 --- a/deploy/rollback.sh +++ b/deploy/rollback.sh @@ -6,9 +6,10 @@ # # Points OUTER_IMAGE at OUTER_PREVIOUS_IMAGE in /srv/outer/release.env # (recording the abandoned image as OUTER_ROLLED_BACK_FROM, and leaving no -# rollback target behind), runs `docker compose up -d`, and waits for the same signal as release.sh -# (container running + outer's "listening on" log line — mTLS admits no -# unauthenticated HTTP probe). +# rollback target behind), then hands that image out with the same +# zero-downtime flow as a release (deploy/handoff.sh): the replacement +# container just runs the previous image, so the rollback itself drops no +# connection and kills no in-flight deploy. # # No secrets in args or logs (only image tags travel in release.env). @@ -27,9 +28,11 @@ cd /srv/outer # Sourced, NOT exported: an exported OUTER_IMAGE would override # --env-file below and compose would keep the current image. # shellcheck disable=SC1091 -. /srv/outer/release.env -# One-way: a retry after a failed rollout re-applies the same target instead -# of flipping back. Going forward again is a fresh release.sh. +source /srv/outer/release.env +# shellcheck disable=SC1091 +source /srv/outer/handoff.sh +# One-way: a retry after a failed rollout re-applies the same target +# instead of flipping back. Going forward again is a fresh release.sh. if [[ -n "${OUTER_PREVIOUS_IMAGE:-}" ]]; then target="$OUTER_PREVIOUS_IMAGE" tmp="$(mktemp)" @@ -54,28 +57,7 @@ echo "remote: rolling back to $target" # drop it here; a build that has the firewall reinstalls it at boot. nft delete table inet outer 2>/dev/null || true -docker compose --env-file /srv/outer/release.env \ - -f /srv/outer/compose.yaml up -d - -cid="$(docker compose --env-file /srv/outer/release.env \ - -f /srv/outer/compose.yaml ps -q outer)" -[[ -n "$cid" ]] || { - echo "remote: no outer container after up" >&2 - exit 1 -} -deadline=$((SECONDS + 60)) -while true; do - # The image check catches a compose that kept the old container. - if [[ "$(docker inspect -f '{{.State.Running}} {{.Config.Image}}' "$cid")" == "true $target" ]] \ - && docker logs "$cid" 2>&1 | grep -q 'listening on'; then - break - fi - if [[ $SECONDS -ge $deadline ]]; then - echo "remote: outer did not come back; last log lines:" >&2 - docker logs --tail 50 "$cid" >&2 || true - exit 1 - fi - sleep 2 -done -echo "remote: rolled back to $target ($cid)" +trap 'handoff_abort' EXIT +handoff_to "$target" +trap - EXIT EOF