From 2e1c66520d1df7a8e304a98a2f3ec5198db9a111 Mon Sep 17 00:00:00 2001 From: Santiago Date: Tue, 8 Sep 2026 17:26:54 -0300 Subject: [PATCH 1/2] feat: add Kubernetes deployment units --- README.md | 5 + k8s/README.md | 10 + k8s/dolos-publisher/.driver-backup.patch | 1184 +++++++++++++++++ k8s/dolos-publisher/.gitignore | 2 + k8s/dolos-publisher/README.md | 99 ++ k8s/dolos-publisher/chart/.helmignore | 5 + k8s/dolos-publisher/chart/Chart.yaml | 13 + k8s/dolos-publisher/chart/templates/NOTES.txt | 17 + .../chart/templates/_helpers.tpl | 14 + .../chart/templates/configmap.yaml | 48 + k8s/dolos-publisher/chart/templates/job.yaml | 59 + k8s/dolos-publisher/chart/values.yaml | 72 + k8s/registry/.gitignore | 1 + k8s/registry/README.md | 106 ++ k8s/registry/chart/.helmignore | 5 + k8s/registry/chart/Chart.yaml | 13 + k8s/registry/chart/templates/NOTES.txt | 24 + k8s/registry/chart/templates/_helpers.tpl | 46 + k8s/registry/chart/templates/configmap.yaml | 65 + k8s/registry/chart/templates/deployment.yaml | 98 ++ k8s/registry/chart/templates/ingress.yaml | 31 + k8s/registry/chart/templates/pvc.yaml | 21 + .../chart/templates/service-public.yaml | 28 + k8s/registry/chart/templates/service.yaml | 20 + k8s/registry/chart/values.yaml | 159 +++ .../zot-build/4350-drop-index-walk.patch | 568 ++++++++ k8s/registry/zot-build/Dockerfile | 48 + 27 files changed, 2761 insertions(+) create mode 100644 k8s/README.md create mode 100644 k8s/dolos-publisher/.driver-backup.patch create mode 100644 k8s/dolos-publisher/.gitignore create mode 100644 k8s/dolos-publisher/README.md create mode 100644 k8s/dolos-publisher/chart/.helmignore create mode 100644 k8s/dolos-publisher/chart/Chart.yaml create mode 100644 k8s/dolos-publisher/chart/templates/NOTES.txt create mode 100644 k8s/dolos-publisher/chart/templates/_helpers.tpl create mode 100644 k8s/dolos-publisher/chart/templates/configmap.yaml create mode 100644 k8s/dolos-publisher/chart/templates/job.yaml create mode 100644 k8s/dolos-publisher/chart/values.yaml create mode 100644 k8s/registry/.gitignore create mode 100644 k8s/registry/README.md create mode 100644 k8s/registry/chart/.helmignore create mode 100644 k8s/registry/chart/Chart.yaml create mode 100644 k8s/registry/chart/templates/NOTES.txt create mode 100644 k8s/registry/chart/templates/_helpers.tpl create mode 100644 k8s/registry/chart/templates/configmap.yaml create mode 100644 k8s/registry/chart/templates/deployment.yaml create mode 100644 k8s/registry/chart/templates/ingress.yaml create mode 100644 k8s/registry/chart/templates/pvc.yaml create mode 100644 k8s/registry/chart/templates/service-public.yaml create mode 100644 k8s/registry/chart/templates/service.yaml create mode 100644 k8s/registry/chart/values.yaml create mode 100644 k8s/registry/zot-build/4350-drop-index-walk.patch create mode 100644 k8s/registry/zot-build/Dockerfile diff --git a/README.md b/README.md index af36e17..770a68b 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,11 @@ producer and consumer; it carries no Cardano assumption. The boundary proof lives in the test suite: `stelae/tests/toy_profile.rs` implements a second, trivial profile against the protocol surface alone. +## Kubernetes + +[`k8s/`](k8s/) contains the instance-agnostic Helm deployment units for the +Dolos publisher and the Stelae registry. + ## Specification The normative specification is [`SPEC.md`](SPEC.md). The Dolos profile's own diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..f7e7afa --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,10 @@ +# Kubernetes + +Helm-packaged deployment units related to the Stelae protocol: + +- [`dolos-publisher/`](dolos-publisher/) runs Dolos backfill Jobs that publish + network steles. +- [`registry/`](registry/) runs the zot registry that stores and serves them. + +Both charts are instance-agnostic. Deployment-specific values and secrets stay +in the operator's infrastructure repository. diff --git a/k8s/dolos-publisher/.driver-backup.patch b/k8s/dolos-publisher/.driver-backup.patch new file mode 100644 index 0000000..194cc4e --- /dev/null +++ b/k8s/dolos-publisher/.driver-backup.patch @@ -0,0 +1,1184 @@ +diff --git a/src/bin/dolos/bootstrap/mithril.rs b/src/bin/dolos/bootstrap/mithril.rs +index 0a2886c9..64c401ad 100644 +--- a/src/bin/dolos/bootstrap/mithril.rs ++++ b/src/bin/dolos/bootstrap/mithril.rs +@@ -15,35 +15,35 @@ use dolos::prelude::*; + #[derive(Debug, clap::Args, Clone)] + pub struct Args { + #[arg(long, default_value = "./snapshot")] +- download_dir: String, ++ pub(crate) download_dir: String, + + /// Skip the Mithril certificate validation + #[arg(long, action)] +- skip_validation: bool, ++ pub(crate) skip_validation: bool, + + /// Assume the snapshot is already available in the download dir + #[arg(long, action)] +- skip_download: bool, ++ pub(crate) skip_download: bool, + + /// Retain downloaded snapshot instead of deleting it + #[arg(long, action)] +- retain_snapshot: bool, ++ pub(crate) retain_snapshot: bool, + + /// Number of blocks to process in each chunk, more is faster but uses more + /// memory + #[arg(long, default_value = "500")] +- chunk_size: usize, ++ pub(crate) chunk_size: usize, + + #[arg(long)] +- start_from: Option, ++ pub(crate) start_from: Option, + + /// Start downloading from this immutable file number (inclusive) + #[arg(long)] +- download_start: Option, ++ pub(crate) download_start: Option, + + /// Download up to this immutable file number (inclusive) + #[arg(long)] +- download_end: Option, ++ pub(crate) download_end: Option, + } + + impl Default for Args { +@@ -170,7 +170,7 @@ impl mithril_client::feedback::FeedbackReceiver for MithrilFeedback { + } + + /// Scan the immutable directory for the highest immutable file number present. +-fn highest_existing_immutable(immutable_dir: &Path) -> Option { ++pub(crate) fn highest_existing_immutable(immutable_dir: &Path) -> Option { + let entries = std::fs::read_dir(immutable_dir).ok()?; + let mut max: Option = None; + for entry in entries.flatten() { +@@ -239,17 +239,39 @@ fn plan_download(args: &Args, immutable_dir: &Path, last_immutable: u64) -> Down + } + } + +-async fn fetch_snapshot( ++/// One aggregator client configuration, shared by the fetch and the beacon ++/// query so the two cannot drift apart on discovery or key handling. ++fn client_builder(config: &MithrilConfig) -> ClientBuilder { ++ ClientBuilder::new(AggregatorDiscoveryType::Url(config.aggregator.clone())) ++ .set_genesis_verification_key(mithril_client::GenesisVerificationKey::JsonHex( ++ config.genesis_key.clone(), ++ )) ++} ++ ++/// The highest immutable file number any aggregator snapshot covers. ++/// ++/// What `snapshot backfill` sizes its next download window against, and how it ++/// knows the aggregator has nothing past the files already on disk. ++pub(crate) async fn latest_immutable_file(config: &MithrilConfig) -> MithrilResult { ++ let client = client_builder(config).build()?; ++ ++ let snapshots = client.cardano_database_v2().list().await?; ++ ++ snapshots ++ .iter() ++ .map(|snapshot| snapshot.beacon.immutable_file_number) ++ .max() ++ .ok_or(MithrilError::msg("no snapshot available")) ++} ++ ++pub(crate) async fn fetch_snapshot( + args: &Args, + config: &MithrilConfig, + feedback: &Feedback, + ) -> MithrilResult<()> { + let feedback = MithrilFeedback::new(feedback); + +- let client = ClientBuilder::new(AggregatorDiscoveryType::Url(config.aggregator.clone())) +- .set_genesis_verification_key(mithril_client::GenesisVerificationKey::JsonHex( +- config.genesis_key.clone(), +- )) ++ let client = client_builder(config) + .add_feedback_receiver(Arc::new(feedback)) + .build()?; + +diff --git a/src/bin/dolos/bootstrap/mod.rs b/src/bin/dolos/bootstrap/mod.rs +index 78a60fab..4ef0bf28 100644 +--- a/src/bin/dolos/bootstrap/mod.rs ++++ b/src/bin/dolos/bootstrap/mod.rs +@@ -8,7 +8,7 @@ use tracing::info; + use crate::feedback::Feedback; + use dolos_core::{StateStore, WalStore}; + +-mod mithril; ++pub(crate) mod mithril; + mod ranged; + mod relay; + mod snapshot; +diff --git a/src/bin/dolos/common.rs b/src/bin/dolos/common.rs +index 8e74c9bf..5969ac75 100644 +--- a/src/bin/dolos/common.rs ++++ b/src/bin/dolos/common.rs +@@ -148,6 +148,19 @@ pub fn stele_scratch_dir(config: &StorageConfig, chosen: Option<&std::path::Path + } + + pub fn setup_domain(config: &RootConfig) -> miette::Result { ++ setup_domain_with_stop_epoch(config, None) ++} ++ ++/// The same domain [`setup_domain`] assembles, with `chain.stop_epoch` forced. ++/// ++/// For callers that replay to a chosen epoch boundary over the live stores — ++/// `snapshot backfill` — the way `doctor rebuild-state` forces it on the ++/// domain it hand-builds. A `Some` here overrides whatever the configuration ++/// says; `None` leaves it alone. ++pub fn setup_domain_with_stop_epoch( ++ config: &RootConfig, ++ stop_epoch: Option, ++) -> miette::Result { + let stores = open_data_stores(config).map_err(|e| match e { + Error::WalError(WalError::IncompatibleVersion { found, expected }) => miette::miette!( + help = format!( +@@ -162,7 +175,11 @@ pub fn setup_domain(config: &RootConfig) -> miette::Result { + let (tip_broadcast, _) = tokio::sync::broadcast::channel(100); + let chain = config.chain.clone(); + +- let ChainConfig::Cardano(chain_config) = chain; ++ let ChainConfig::Cardano(mut chain_config) = chain; ++ ++ if stop_epoch.is_some() { ++ chain_config.stop_epoch = stop_epoch; ++ } + + let chain = dolos_cardano::CardanoLogic::initialize::( + chain_config, +@@ -333,7 +350,7 @@ pub fn open_genesis_files(config: &GenesisConfig) -> miette::Result { + + #[inline] + #[cfg(unix)] +-async fn wait_for_exit_signal() { ++pub(crate) async fn wait_for_exit_signal() { + let mut sigterm = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap(); + +@@ -349,7 +366,7 @@ async fn wait_for_exit_signal() { + + #[inline] + #[cfg(windows)] +-async fn wait_for_exit_signal() { ++pub(crate) async fn wait_for_exit_signal() { + tokio::signal::ctrl_c().await.unwrap() + } + +diff --git a/src/bin/dolos/snapshot/backfill.rs b/src/bin/dolos/snapshot/backfill.rs +new file mode 100644 +index 00000000..e16a37f4 +--- /dev/null ++++ b/src/bin/dolos/snapshot/backfill.rs +@@ -0,0 +1,846 @@ ++//! `dolos snapshot backfill` — replay mithril history one epoch at a time, ++//! publishing a stele at each boundary. ++//! ++//! The publisher driver: one restart-safe loop that acquires immutable files ++//! from mithril in bounded windows, replays them to the next epoch boundary ++//! under `stop_epoch`, publishes the resulting sequence into an OCI repository ++//! in-process, prunes behind itself, and repeats until the aggregator has ++//! nothing further. A premature rerun — cron firing before a new epoch is ++//! available — finds the repository up to date and exits zero without writing. ++//! ++//! Downloads resume from the directory's own contents when any immutable ++//! files are present. When none are — a cold container start, whose disk ++//! keeps nothing and whose store was just restored from the registry — the ++//! start is derived from the cursor's chunk file instead, a margin early, ++//! so a restart costs one window rather than the chain so far. ++//! ++//! The reader never opens the highest downloaded file — pallas pops it as ++//! "not really immutable" — so at the aggregator tip the replay stands at ++//! most one chunk (~21600 slots, about six hours) behind the mithril beacon. ++//! That lag is steady state, not loss: the next run picks the chunk up once ++//! the beacon moves past it. ++//! ++//! Each iteration *opens* by publishing the sequence the cursor already stands ++//! at, then extends the replay by one epoch. Publishing on entry rather than ++//! right after the boundary import is what makes every crash window resumable: ++//! a rerun that finds the boundary reached but unpublished publishes it before ++//! moving on, instead of replaying past it and leaving a gap no later stele ++//! could close. ++//! ++//! This module is orchestration only: it composes the mithril fetch, the ++//! import lifecycle, and the publish path `snapshot publish --repo` uses, and ++//! changes none of them. ++ ++use std::path::{Path, PathBuf}; ++use std::sync::Arc; ++ ++use clap::Parser; ++use dolos_core::config::RootConfig; ++use dolos_core::{Domain as _, DomainError, ImportExt as _, StateStore as _, WalStore as _}; ++use dolos_snapshot::{export, registry::Repository}; ++use indicatif::ProgressBar; ++use itertools::Itertools as _; ++use miette::{bail, Context as _, IntoDiagnostic as _}; ++use tokio_util::sync::CancellationToken; ++use tracing::info; ++ ++use crate::feedback::Feedback; ++use dolos::adapters::DomainAdapter; ++ ++/// Blocks handed to `import_blocks` per batch. ++const IMPORT_CHUNK: usize = 100; ++ ++/// Files of margin kept around the cursor's own immutable file, on both of ++/// the convention's uses: cleanup spares this many files behind the consumed ++/// threshold, and a download start derived from the cursor backs up this ++/// many files. Early is the cheap direction — a re-downloaded file's blocks ++/// at or before the cursor are skipped on import — while late would leave ++/// the immutable reader without the cursor's own chunk. ++const IMMUTABLE_FILE_MARGIN: u64 = 2; ++ ++/// Slots per immutable chunk file — a node packaging convention, not a ++/// protocol invariant. Used to pick files safe to delete and, on a cold ++/// start whose download dir is empty, to derive where downloading resumes; ++/// never to plan how far a replay goes. Both uses carry ++/// [`IMMUTABLE_FILE_MARGIN`], and a derivation that still lands past the ++/// cursor's chunk is caught by the stalled-window check rather than trusted. ++const SLOTS_PER_IMMUTABLE_FILE: u64 = 21_600; ++ ++/// Where the mithril window lands when the operator names nowhere: beside the ++/// stores, so the bytes stay on the data mount. ++const DOWNLOAD_DIR: &str = "mithril"; ++ ++const INTERRUPTED: &str = ++ "interrupted by a shutdown signal; the stores are consistent and a rerun resumes here"; ++ ++#[derive(Debug, Parser)] ++pub struct Args { ++ /// OCI repository to publish into, e.g. ++ /// `oci://ghcr.io/txpipe/dolos-mainnet` ++ #[arg(long, value_name = "OCI_URL")] ++ repo: Repository, ++ ++ /// talk to the repository over plaintext HTTP rather than HTTPS; for a ++ /// registry on a loopback address or a mirror inside a cluster, and for ++ /// nothing reachable from outside one ++ #[arg(long, action)] ++ insecure: bool, ++ ++ /// directory to stage layers in while they are uploaded; defaults to ++ /// `/scratch` ++ #[arg(long, value_name = "DIR")] ++ scratch_dir: Option, ++ ++ /// directory the mithril immutable files are downloaded into; defaults to ++ /// `/mithril` ++ #[arg(long, value_name = "DIR")] ++ download_dir: Option, ++ ++ /// immutable files fetched per download round ++ #[arg(long, default_value = "40")] ++ window: u64, ++ ++ /// stop after publishing this sequence; for smoke tests ++ #[arg(long, value_name = "N")] ++ until_epoch: Option, ++ ++ /// skip the mithril digest and merkle validation; the certificate chain ++ /// is still verified. local smoke tests only ++ #[arg(long, action)] ++ skip_validation: bool, ++} ++ ++/// What an iteration's opening publish decided about the run. ++enum Step { ++ /// Replay toward `target`'s boundary. `prune` says whether history behind ++ /// the cursor may be dropped first — true only once the publish step has ++ /// run, because pruning at tip T is safe exactly when everything below T ++ /// is already in the repository. ++ Extend { target: u64, prune: bool }, ++ /// `--until-epoch` is published; the run is over. ++ Done, ++} ++ ++/// How an epoch's replay ended. ++enum Advance { ++ /// `stop_epoch` fired: the cursor stands on the target epoch's first ++ /// block. ++ Boundary { cursor_slot: u64 }, ++ /// The local files ran out and the aggregator has nothing newer. ++ MithrilExhausted, ++ /// A shutdown signal arrived; everything imported so far is committed. ++ Cancelled, ++} ++ ++/// One import pass over the files on disk. ++enum Import { ++ Boundary, ++ /// The files ran out before the boundary. Deliberately silent about how ++ /// many blocks the pass imported: on a sparse chain zero is an ordinary ++ /// answer, so nothing downstream may treat it as evidence. ++ Exhausted, ++ Cancelled, ++} ++ ++/// The epoch the next replay stops at: one past the cursor's, or 1 from a ++/// fresh store. ++fn target_epoch(cursor_epoch: Option) -> u64 { ++ cursor_epoch.map_or(1, |epoch| epoch + 1) ++} ++ ++/// The file a download round resumes from, or `None` for the beginning. ++/// ++/// The directory's own contents stay authoritative when any files are ++/// present — that is what re-fetches a possibly truncated highest file. An ++/// empty directory with a cursor is a cold container start: the store was ++/// restored from the registry onto a disk that keeps nothing, and resuming ++/// from file zero would re-download the whole chain, so the start is derived ++/// from the cursor's own chunk file instead, a margin early. ++fn resume_file(highest: Option, cursor_slot: Option) -> Option { ++ highest.or_else(|| { ++ cursor_slot ++ .map(|slot| (slot / SLOTS_PER_IMMUTABLE_FILE).saturating_sub(IMMUTABLE_FILE_MARGIN)) ++ }) ++} ++ ++/// The next download round, as `(download_start, download_end)`, or `None` ++/// when the aggregator has nothing past what is on disk. ++/// ++/// The resume file is deliberately re-fetched rather than skipped: an ++/// interrupted download may have left it truncated, and it has not been ++/// verified yet. ++fn next_window(resume: Option, window: u64, beacon: u64) -> Option<(Option, u64)> { ++ match resume { ++ Some(resume) if beacon <= resume => None, ++ Some(resume) => Some((Some(resume), beacon.min(resume + window))), ++ None => Some((None, beacon.min(window))), ++ } ++} ++ ++/// Whether a download round actually landed new files. ++/// ++/// The stall test, and it is asked of file numbers rather than block counts ++/// on purpose: a window of legitimately empty chunks advances the files while ++/// importing nothing, and that is an ordinary round on a sparse chain. A ++/// fetch that left the highest file where it was is the hard error. `Option` ++/// ordering puts an absent highest below every present one, so the first ++/// window into an empty directory counts as an advance. ++fn fetch_advanced(before: Option, after: Option) -> bool { ++ after > before ++} ++ ++/// Immutable files strictly below this number sit wholly behind the cursor, ++/// margin included, and are safe to delete. ++fn consumed_below(cursor_slot: u64) -> u64 { ++ (cursor_slot / SLOTS_PER_IMMUTABLE_FILE).saturating_sub(IMMUTABLE_FILE_MARGIN) ++} ++ ++/// What the immutable directory holds, for a diagnostic. ++fn dir_contents(immutable_dir: &Path) -> String { ++ let Ok(entries) = std::fs::read_dir(immutable_dir) else { ++ return "an unreadable immutable dir".to_owned(); ++ }; ++ ++ let mut numbers: Vec = entries ++ .flatten() ++ .filter_map(|entry| { ++ let name = entry.file_name(); ++ let name = name.to_string_lossy(); ++ name.split('.').next().and_then(|s| s.parse().ok()) ++ }) ++ .collect(); ++ ++ numbers.sort_unstable(); ++ numbers.dedup(); ++ ++ match (numbers.first(), numbers.last()) { ++ (Some(first), Some(last)) => { ++ format!("files {first:05}..={last:05} ({} of them)", numbers.len()) ++ } ++ _ => "no immutable files".to_owned(), ++ } ++} ++ ++/// Delete the numbered immutable files the replay has consumed. ++fn cleanup_consumed(immutable_dir: &Path, cursor_slot: u64) -> miette::Result<()> { ++ let threshold = consumed_below(cursor_slot); ++ ++ if threshold == 0 { ++ return Ok(()); ++ } ++ ++ let Ok(entries) = std::fs::read_dir(immutable_dir) else { ++ return Ok(()); ++ }; ++ ++ let mut removed = 0u64; ++ ++ for entry in entries.flatten() { ++ let name = entry.file_name(); ++ let name = name.to_string_lossy(); ++ ++ let Some(number) = name.split('.').next().and_then(|s| s.parse::().ok()) else { ++ continue; ++ }; ++ ++ if number < threshold { ++ std::fs::remove_file(entry.path()) ++ .into_diagnostic() ++ .with_context(|| format!("removing the consumed immutable file {name}"))?; ++ ++ removed += 1; ++ } ++ } ++ ++ if removed > 0 { ++ info!(removed, threshold, "removed consumed immutable files"); ++ } ++ ++ Ok(()) ++} ++ ++/// SIGTERM/SIGINT as a token the synchronous loop polls between chunks. ++/// ++/// The driver has no ambient tokio runtime for `hook_exit_token`, so the ++/// signal wait gets a dedicated thread with a current-thread runtime of its ++/// own. ++fn spawn_exit_watcher() -> miette::Result { ++ let cancel = CancellationToken::new(); ++ let hooked = cancel.clone(); ++ ++ std::thread::Builder::new() ++ .name("exit-signal".to_owned()) ++ .spawn(move || { ++ let runtime = tokio::runtime::Builder::new_current_thread() ++ .enable_all() ++ .build() ++ .expect("building the signal-wait runtime"); ++ ++ runtime.block_on(async { ++ crate::common::wait_for_exit_signal().await; ++ tracing::warn!("shutdown requested; stopping at the next chunk"); ++ hooked.cancel(); ++ }); ++ }) ++ .into_diagnostic() ++ .context("spawning the signal-wait thread")?; ++ ++ Ok(cancel) ++} ++ ++fn dir_argument(dir: &Path) -> miette::Result { ++ dir.to_str() ++ .map(str::to_owned) ++ .ok_or_else(|| miette::miette!("the download dir {} is not valid UTF-8", dir.display())) ++} ++ ++/// Everything an iteration needs and the CLI already settled. ++struct Driver<'a> { ++ config: &'a RootConfig, ++ args: &'a Args, ++ feedback: &'a Feedback, ++ /// For the async mithril calls only. The registry client owns a ++ /// current-thread runtime of its own and must never run inside this one, ++ /// so every publish stays on the plain thread. ++ runtime: tokio::runtime::Runtime, ++ cancel: CancellationToken, ++ download_dir: PathBuf, ++ immutable_dir: PathBuf, ++} ++ ++impl Driver<'_> { ++ /// Publish the sequence the cursor stands at, and name the next target. ++ /// ++ /// Also reseeds the WAL from the state cursor before anything else opens ++ /// the domain: `import_blocks` skips the WAL by design, so a run that ++ /// died mid-import left the state ahead of it, and the next domain open ++ /// would refuse with `InconsistentState`. ++ fn publish_pending(&self) -> miette::Result { ++ let stores = crate::common::open_data_stores(self.config) ++ .into_diagnostic() ++ .context("opening the data stores")?; ++ ++ let cursor = stores ++ .state ++ .read_cursor() ++ .into_diagnostic() ++ .context("reading the state cursor")?; ++ ++ let Some(cursor) = cursor else { ++ return Ok(Step::Extend { ++ target: target_epoch(None), ++ prune: false, ++ }); ++ }; ++ ++ if cursor.is_fully_defined() { ++ stores ++ .wal ++ .reset_to(&cursor) ++ .into_diagnostic() ++ .context("seeding the WAL from the state cursor")?; ++ } ++ ++ let summary = dolos_cardano::eras::load_chain_summary_from_state(&stores.state) ++ .map_err(|err| miette::miette!("loading the chain summary: {err:?}"))?; ++ ++ let (epoch, _) = summary.slot_epoch(cursor.slot()); ++ ++ // Nothing publishable yet: a sequence-0 stele would be epoch 0's ++ // mid-epoch sliver, which no consumer chains from. ++ if epoch == 0 { ++ return Ok(Step::Extend { ++ target: target_epoch(Some(epoch)), ++ prune: false, ++ }); ++ } ++ ++ let genesis = crate::common::open_genesis_files(&self.config.genesis)?; ++ ++ let plan = export::plan( ++ &stores.state, ++ u64::from(genesis.network_magic()), ++ super::retained_epochs(self.config)?, ++ ) ++ .into_diagnostic() ++ .context("planning the publish")?; ++ ++ super::report_plan(&plan)?; ++ ++ let publish = super::publish::RepositoryPublish { ++ repo: &self.args.repo, ++ insecure: self.args.insecure, ++ scratch_dir: self.args.scratch_dir.as_deref(), ++ rebuild: false, ++ dry_run: false, ++ require_new: false, ++ }; ++ ++ super::publish::to_repository(self.config, &publish, &plan, &stores, self.feedback)?; ++ ++ if self ++ .args ++ .until_epoch ++ .is_some_and(|until| plan.sequence >= until) ++ { ++ println!( ++ "sequence {} published; stopping at --until-epoch", ++ plan.sequence ++ ); ++ ++ return Ok(Step::Done); ++ } ++ ++ Ok(Step::Extend { ++ target: target_epoch(Some(epoch)), ++ prune: true, ++ }) ++ } ++ ++ /// Replay toward `target`'s boundary inside a domain that stops there. ++ fn extend(&self, target: u64, prune: bool) -> miette::Result { ++ let domain = crate::common::setup_domain_with_stop_epoch(self.config, Some(target))?; ++ ++ let result = self.advance_domain(&domain, prune); ++ ++ // Shut down even when the replay failed: fjall in particular has ++ // background work to flush before the handle drops. ++ let shutdown = domain.shutdown(); ++ ++ let advance = result?; ++ shutdown.map_err(|e| miette::miette!("shutting down the domain: {e}"))?; ++ ++ Ok(advance) ++ } ++ ++ /// Import what is on disk, fetching windows from mithril whenever the ++ /// files run out, until the boundary, the aggregator's tip, or a signal. ++ fn advance_domain(&self, domain: &DomainAdapter, prune: bool) -> miette::Result { ++ let mithril = self ++ .config ++ .mithril ++ .as_ref() ++ .ok_or_else(|| miette::miette!("missing mithril config"))?; ++ ++ // After the publish and before the next epoch goes in, never between ++ // a boundary and its publish: pruning at tip T drops history below ++ // `T - max_history` only, and every later publish reads blocks at or ++ // above the T it was standing at when its predecessor was published. ++ if prune { ++ let rounds = domain ++ .drain_housekeeping(None) ++ .map_err(|e| miette::miette!("{e}")) ++ .context("pruning excess history")?; ++ ++ info!(rounds, "housekeeping drained"); ++ } ++ ++ let progress = self.feedback.slot_progress_bar(); ++ progress.set_message("replaying immutable blocks"); ++ ++ let outcome = loop { ++ if self.cancel.is_cancelled() { ++ break Advance::Cancelled; ++ } ++ ++ match self.import_available(domain, &progress)? { ++ Import::Boundary => { ++ let cursor_slot = domain ++ .state ++ .read_cursor() ++ .into_diagnostic() ++ .context("reading the state cursor at the boundary")? ++ .map(|cursor| cursor.slot()) ++ .unwrap_or_default(); ++ ++ break Advance::Boundary { cursor_slot }; ++ } ++ Import::Cancelled => break Advance::Cancelled, ++ // Zero blocks imported is not a stall: on a sparse chain a ++ // whole window of chunks can legitimately be empty, and the ++ // replay simply needs those slot ranges walked. Keep ++ // fetching; the stall check below speaks in file numbers. ++ Import::Exhausted => {} ++ } ++ ++ let Some(beacon) = self ++ .runtime ++ .block_on(async { ++ tokio::select! { ++ beacon = crate::bootstrap::mithril::latest_immutable_file(mithril) => { ++ beacon.map(Some) ++ } ++ _ = self.cancel.cancelled() => Ok(None), ++ } ++ }) ++ .map_err(|err| miette::miette!(err.to_string())) ++ .context("listing mithril snapshots")? ++ else { ++ break Advance::Cancelled; ++ }; ++ ++ let highest = ++ crate::bootstrap::mithril::highest_existing_immutable(&self.immutable_dir); ++ ++ let cursor_slot = domain ++ .state ++ .read_cursor() ++ .into_diagnostic() ++ .context("reading the state cursor")? ++ .map(|cursor| cursor.slot()); ++ ++ let resume = resume_file(highest, cursor_slot); ++ ++ let Some((start, end)) = next_window(resume, self.args.window, beacon) else { ++ break Advance::MithrilExhausted; ++ }; ++ ++ info!( ++ ?start, ++ end, beacon, "fetching an immutable window from mithril" ++ ); ++ ++ let fetch = crate::bootstrap::mithril::Args { ++ download_dir: dir_argument(&self.download_dir)?, ++ skip_validation: self.args.skip_validation, ++ download_start: start, ++ download_end: Some(end), ++ ..Default::default() ++ }; ++ ++ let fetched = self ++ .runtime ++ .block_on(async { ++ tokio::select! { ++ fetched = crate::bootstrap::mithril::fetch_snapshot( ++ &fetch, ++ mithril, ++ self.feedback, ++ ) => fetched.map(Some), ++ _ = self.cancel.cancelled() => Ok(None), ++ } ++ }) ++ .map_err(|err| miette::miette!(err.to_string())) ++ .context("fetching a mithril immutable window")?; ++ ++ if fetched.is_none() { ++ break Advance::Cancelled; ++ } ++ ++ // The one stall that is a hard error, and it is measured in file ++ // numbers, never blocks: a fetch that left the highest file ++ // where it was returned nothing new — a misconfigured range or ++ // a download failure — and every later round would only repeat ++ // it. ++ let after = crate::bootstrap::mithril::highest_existing_immutable(&self.immutable_dir); ++ ++ if !fetch_advanced(highest, after) { ++ let cursor_slot = cursor_slot.unwrap_or_default(); ++ ++ bail!( ++ "the fetched immutable window {:05}..={end:05} did not advance the \ ++ downloaded files (highest was {highest:?}, still {after:?}); mithril \ ++ returned nothing new for the state cursor at slot {cursor_slot} — the \ ++ immutable dir holds {}", ++ start.unwrap_or(0), ++ dir_contents(&self.immutable_dir), ++ ); ++ } ++ }; ++ ++ // Whatever ended the replay, the chunks it committed are in the state ++ // and the WAL must agree before the next domain open. ++ self.seed_wal(domain)?; ++ ++ progress.abandon_with_message("replay round complete"); ++ ++ Ok(outcome) ++ } ++ ++ /// Import everything on disk past the cursor, in chunks. ++ fn import_available( ++ &self, ++ domain: &DomainAdapter, ++ progress: &ProgressBar, ++ ) -> miette::Result { ++ use pallas::network::miniprotocols::Point; ++ ++ // Before the first download the immutable dir does not exist at all; ++ // that is the caller's cue to fetch, not an error. ++ if !self.immutable_dir.is_dir() { ++ return Ok(Import::Exhausted); ++ } ++ ++ // Nothing to walk yet, same cue. Deliberately *not* `get_tip`: on a ++ // sparse chain the second-highest chunk can be empty, and `get_tip` ++ // reads only that one chunk and answers `None` for the whole db — ++ // with full unimported chunks sitting right there. The walk from the ++ // cursor is the only reader that tells the truth here. ++ if crate::bootstrap::mithril::highest_existing_immutable(&self.immutable_dir).is_none() { ++ return Ok(Import::Exhausted); ++ } ++ ++ let cursor = domain ++ .state ++ .read_cursor() ++ .into_diagnostic() ++ .context("reading the state cursor")?; ++ ++ let point: Point = cursor ++ .map(|c| c.try_into().unwrap()) ++ .unwrap_or(Point::Origin); ++ ++ let mut iter = pallas::interop::hardano::storage::immutable::read_blocks_from_point( ++ &self.immutable_dir, ++ point.clone(), ++ ) ++ .map_err(|err| miette::miette!(err.to_string())) ++ .context("iterating the local immutable db")?; ++ ++ // unless we're starting from the origin of the chain, the iterator ++ // stands on the last block already imported; skip it rather than ++ // import it twice ++ if point != Point::Origin { ++ iter.next(); ++ } ++ ++ for batch in iter.chunks(IMPORT_CHUNK).into_iter() { ++ let batch: Vec<_> = batch ++ .try_collect() ++ .into_diagnostic() ++ .context("reading block data")?; ++ ++ let batch: Vec<_> = batch.into_iter().map(Arc::new).collect(); ++ ++ match domain.import_blocks(batch) { ++ Ok(last) => progress.set_position(last), ++ Err(DomainError::StopEpochReached) => return Ok(Import::Boundary), ++ Err(e) => { ++ return Err(miette::miette!("{e}")) ++ .context("importing an immutable block chunk") ++ } ++ } ++ ++ if self.cancel.is_cancelled() { ++ return Ok(Import::Cancelled); ++ } ++ } ++ ++ // A yield of nothing is not a verdict: the walk exhausts silently at ++ // the retained edge, empty chunks contribute no blocks, and a chunk ++ // read error truncates the iterator the same way. Whether anything ++ // more is coming is the fetch loop's question, answered in file ++ // numbers, never in block counts. ++ Ok(Import::Exhausted) ++ } ++ ++ /// Reseed the WAL from the state cursor, so the next domain open finds ++ /// the two agreeing. ++ fn seed_wal(&self, domain: &DomainAdapter) -> miette::Result<()> { ++ let cursor = domain ++ .state ++ .read_cursor() ++ .into_diagnostic() ++ .context("reading the state cursor")?; ++ ++ let Some(cursor) = cursor else { ++ return Ok(()); ++ }; ++ ++ if !cursor.is_fully_defined() { ++ bail!( ++ "state cursor at slot {} has no block hash, cannot seed the WAL", ++ cursor.slot(), ++ ); ++ } ++ ++ domain ++ .wal ++ .reset_to(&cursor) ++ .into_diagnostic() ++ .context("seeding the WAL from the state cursor") ++ } ++} ++ ++pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Result<()> { ++ crate::common::setup_tracing(&config.logging, &config.telemetry)?; ++ ++ if args.window == 0 { ++ bail!("--window must be at least 1"); ++ } ++ ++ if config.mithril.is_none() { ++ bail!("missing mithril config"); ++ } ++ ++ let download_dir = args ++ .download_dir ++ .clone() ++ .unwrap_or_else(|| config.storage.path.join(DOWNLOAD_DIR)); ++ ++ std::fs::create_dir_all(&download_dir) ++ .into_diagnostic() ++ .with_context(|| format!("creating the download dir {}", download_dir.display()))?; ++ ++ let driver = Driver { ++ config, ++ args, ++ feedback, ++ runtime: tokio::runtime::Runtime::new() ++ .into_diagnostic() ++ .context("creating the tokio runtime for mithril downloads")?, ++ cancel: spawn_exit_watcher()?, ++ immutable_dir: download_dir.join("immutable"), ++ download_dir, ++ }; ++ ++ loop { ++ if driver.cancel.is_cancelled() { ++ bail!(INTERRUPTED); ++ } ++ ++ let (target, prune) = match driver.publish_pending()? { ++ Step::Done => return Ok(()), ++ Step::Extend { target, prune } => (target, prune), ++ }; ++ ++ info!(target, "replaying toward the next epoch boundary"); ++ ++ match driver.extend(target, prune)? { ++ Advance::Boundary { cursor_slot } => { ++ cleanup_consumed(&driver.immutable_dir, cursor_slot)?; ++ } ++ Advance::MithrilExhausted => { ++ println!("the repository is up to date with mithril; nothing left to backfill"); ++ return Ok(()); ++ } ++ Advance::Cancelled => bail!(INTERRUPTED), ++ } ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ ++ #[test] ++ fn the_first_target_is_epoch_one_and_every_later_one_follows_the_cursor() { ++ assert_eq!(target_epoch(None), 1); ++ assert_eq!(target_epoch(Some(0)), 1); ++ assert_eq!(target_epoch(Some(499)), 500); ++ } ++ ++ #[test] ++ fn an_empty_download_dir_resumes_from_the_cursors_own_file() { ++ // empty dir with a cursor: the cursor's chunk file, a margin early ++ assert_eq!( ++ resume_file(None, Some(SLOTS_PER_IMMUTABLE_FILE * 6000)), ++ Some(5998), ++ ); ++ ++ // mid-file slots land in the same file before the margin applies ++ assert_eq!( ++ resume_file(None, Some(SLOTS_PER_IMMUTABLE_FILE * 6000 + 5)), ++ Some(5998), ++ ); ++ ++ // the margin floors at the first file ++ assert_eq!(resume_file(None, Some(SLOTS_PER_IMMUTABLE_FILE)), Some(0)); ++ assert_eq!(resume_file(None, Some(0)), Some(0)); ++ ++ // empty dir, fresh store: the beginning ++ assert_eq!(resume_file(None, None), None); ++ assert_eq!( ++ next_window(resume_file(None, None), 40, 1000), ++ Some((None, 40)) ++ ); ++ ++ // files on disk stay authoritative, wherever the cursor is ++ assert_eq!( ++ resume_file(Some(120), Some(SLOTS_PER_IMMUTABLE_FILE * 6000)), ++ Some(120), ++ ); ++ } ++ ++ #[test] ++ fn only_a_fetch_that_leaves_the_files_where_they_were_is_a_stall() { ++ // the first window into an empty dir is an advance ++ assert!(fetch_advanced(None, Some(0))); ++ ++ // new files landed — even when every one of them is an empty chunk ++ // and the import that follows adds zero blocks, the loop goes on ++ assert!(fetch_advanced(Some(5), Some(6))); ++ ++ // nothing new on disk after a fetch: the hard error ++ assert!(!fetch_advanced(Some(5), Some(5))); ++ assert!(!fetch_advanced(Some(5), None)); ++ assert!(!fetch_advanced(None, None)); ++ } ++ ++ #[test] ++ fn windows_advance_from_the_highest_existing_file() { ++ // a fresh dir starts at the beginning, one window deep ++ assert_eq!(next_window(None, 40, 1000), Some((None, 40))); ++ ++ // a short chain clamps to the beacon ++ assert_eq!(next_window(None, 40, 7), Some((None, 7))); ++ ++ // resuming re-fetches the highest file: it may be truncated ++ assert_eq!(next_window(Some(100), 40, 1000), Some((Some(100), 140))); ++ ++ // the last window clamps to the beacon ++ assert_eq!(next_window(Some(990), 40, 1000), Some((Some(990), 1000))); ++ ++ // nothing newer than what is on disk ++ assert_eq!(next_window(Some(1000), 40, 1000), None); ++ assert_eq!(next_window(Some(1001), 40, 1000), None); ++ } ++ ++ #[test] ++ fn cleanup_keeps_a_margin_behind_the_cursor() { ++ assert_eq!(consumed_below(0), 0); ++ assert_eq!(consumed_below(SLOTS_PER_IMMUTABLE_FILE * 2), 0); ++ assert_eq!(consumed_below(SLOTS_PER_IMMUTABLE_FILE * 3), 1); ++ assert_eq!(consumed_below(SLOTS_PER_IMMUTABLE_FILE * 10 + 5), 8); ++ } ++ ++ #[test] ++ fn cleanup_removes_only_consumed_numbered_files() { ++ let dir = tempfile::tempdir().unwrap(); ++ ++ for n in 0..6u64 { ++ for ext in ["chunk", "primary", "secondary"] { ++ std::fs::write(dir.path().join(format!("{n:05}.{ext}")), []).unwrap(); ++ } ++ } ++ ++ std::fs::write(dir.path().join("lock"), []).unwrap(); ++ ++ // threshold 3: files 0..=2 consumed, 3..=5 and non-numeric names stay ++ cleanup_consumed(dir.path(), SLOTS_PER_IMMUTABLE_FILE * 5).unwrap(); ++ ++ let mut remaining: Vec = std::fs::read_dir(dir.path()) ++ .unwrap() ++ .flatten() ++ .map(|entry| entry.file_name().to_string_lossy().into_owned()) ++ .collect(); ++ ++ remaining.sort(); ++ ++ assert_eq!( ++ remaining, ++ [ ++ "00003.chunk", ++ "00003.primary", ++ "00003.secondary", ++ "00004.chunk", ++ "00004.primary", ++ "00004.secondary", ++ "00005.chunk", ++ "00005.primary", ++ "00005.secondary", ++ "lock", ++ ], ++ ); ++ } ++} +diff --git a/src/bin/dolos/snapshot/mod.rs b/src/bin/dolos/snapshot/mod.rs +index c86d87fe..f0fda476 100644 +--- a/src/bin/dolos/snapshot/mod.rs ++++ b/src/bin/dolos/snapshot/mod.rs +@@ -27,6 +27,8 @@ use miette::{Context as _, IntoDiagnostic as _}; + + use crate::feedback::Feedback; + ++#[cfg(feature = "mithril")] ++mod backfill; + mod digest; + mod inspect; + mod publish; +@@ -37,6 +39,11 @@ pub enum Command { + /// writes a stele to a local directory or an OCI repository + Publish(publish::Args), + ++ /// replays mithril history one epoch at a time, publishing a stele at ++ /// each boundary into an OCI repository ++ #[cfg(feature = "mithril")] ++ Backfill(backfill::Args), ++ + /// computes a stele's inscription and identity without writing one + Digest(digest::Args), + +@@ -61,6 +68,8 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res + // that waits on a store walk and a network, and the other three are + // over in the time it takes to print what they found. + Command::Publish(x) => publish::run(config, x, feedback), ++ #[cfg(feature = "mithril")] ++ Command::Backfill(x) => backfill::run(config, x, feedback), + Command::Digest(x) => digest::run(config, x), + Command::Verify(x) => verify::run(config, x), + Command::Inspect(x) => inspect::run(config, x), +diff --git a/src/bin/dolos/snapshot/publish.rs b/src/bin/dolos/snapshot/publish.rs +index 32f8c1df..4f653395 100644 +--- a/src/bin/dolos/snapshot/publish.rs ++++ b/src/bin/dolos/snapshot/publish.rs +@@ -89,13 +89,50 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res + super::report_plan(&plan)?; + + match (&args.repo, &args.output_dir) { +- (Some(repo), _) => to_repository(config, args, repo, &plan, &stores, feedback), ++ (Some(repo), _) => { ++ let publish = RepositoryPublish { ++ repo, ++ insecure: args.insecure, ++ scratch_dir: args.scratch_dir.as_deref(), ++ rebuild: args.rebuild, ++ dry_run: args.dry_run, ++ require_new: args.require_new, ++ }; ++ ++ to_repository(config, &publish, &plan, &stores, feedback) ++ } + (None, Some(dir)) => to_directory(args, dir, &plan, &stores, feedback), + // The required `destination` group already refuses this. + (None, None) => unreachable!("one of --output-dir and --repo is required"), + } + } + ++/// The repository arm's settings, freed of the CLI that spelled them. ++/// ++/// Factored so `snapshot backfill` publishes through exactly the code path ++/// `snapshot publish --repo` does — same standing check, same preflight, same ++/// chained predecessor, same report — rather than a second telling of it that ++/// would drift. ++pub(super) struct RepositoryPublish<'a> { ++ /// The repository to publish into. ++ pub repo: &'a Repository, ++ ++ /// Talk plaintext HTTP rather than HTTPS. ++ pub insecure: bool, ++ ++ /// Where to stage layers; `None` takes `/scratch`. ++ pub scratch_dir: Option<&'a std::path::Path>, ++ ++ /// Build every layer instead of carrying forward published ones. ++ pub rebuild: bool, ++ ++ /// Report what would be written and stop. ++ pub dry_run: bool, ++ ++ /// Fail when the repository is already at this node's sequence. ++ pub require_new: bool, ++} ++ + fn to_directory( + args: &Args, + dir: &std::path::Path, +@@ -145,14 +182,15 @@ fn to_directory( + /// The report is what a publisher wants to check rather than trust: how much of + /// this stele was inherited rather than built, and how much of it moved. Both + /// are numbers the code counted, not an inference from a duration. +-fn to_repository( ++pub(super) fn to_repository( + config: &RootConfig, +- args: &Args, +- repo: &Repository, ++ publish: &RepositoryPublish, + plan: &export::Plan, + stores: &crate::common::Stores, + feedback: &Feedback, + ) -> miette::Result<()> { ++ let repo = publish.repo; ++ + // A publisher's credentials come from `STELAE_REGISTRY_USER` / + // `STELAE_REGISTRY_PASSWORD`, which override anything configured. The + // configured user is still the fallback: it is read-only, so authenticating +@@ -160,9 +198,9 @@ fn to_repository( + // is the honest place for "these credentials cannot publish" to be said. + let auth = crate::common::stele_registry_auth(&config.stelae)?; + +- let scratch = crate::common::stele_scratch_dir(&config.storage, args.scratch_dir.as_deref()); ++ let scratch = crate::common::stele_scratch_dir(&config.storage, publish.scratch_dir); + +- let registry = registry::open(repo, args.insecure, auth, scratch) ++ let registry = registry::open(repo, publish.insecure, auth, scratch) + .into_diagnostic() + .context("opening the repository")?; + +@@ -172,11 +210,11 @@ fn to_repository( + // along with everything else. + let publishing = registry::Publishing::new(®istry) + .recording_in(&config.storage.path) +- .rebuilding(args.rebuild); ++ .rebuilding(publish.rebuild); + + // Before anything is built, and before the dry run too: a publisher asking + // what a publish would do wants the same answer the publish gives. +- if !standing(®istry, plan, args)? { ++ if !standing(®istry, plan, publish.require_new)? { + return Ok(()); + } + +@@ -190,7 +228,7 @@ fn to_repository( + .into_diagnostic() + .context("sizing the staging directory")?; + +- if args.dry_run { ++ if publish.dry_run { + // `None` here and `None` at the `publish` below are one decision: a dry + // run describes the publish that follows it, so the two calls are + // handed the same digest records or the number is about something else. +@@ -273,7 +311,7 @@ fn to_repository( + /// to invent. What is new is that the message names the distance alongside + /// both sequences, so "the publisher has been down for a day" and "the + /// publisher has been down for a month" do not read the same. +-fn standing(registry: &Registry, plan: &export::Plan, args: &Args) -> miette::Result { ++fn standing(registry: &Registry, plan: &export::Plan, require_new: bool) -> miette::Result { + let standing = registry::standing(registry, plan) + .into_diagnostic() + .context("reading the repository's latest stele")?; +@@ -294,7 +332,7 @@ fn standing(registry: &Registry, plan: &export::Plan, args: &Args) -> miette::Re + plan.sequence, + ); + +- if args.require_new { ++ if require_new { + return Err(miette::miette!("{message}")); + } + diff --git a/k8s/dolos-publisher/.gitignore b/k8s/dolos-publisher/.gitignore new file mode 100644 index 0000000..97fe62e --- /dev/null +++ b/k8s/dolos-publisher/.gitignore @@ -0,0 +1,2 @@ +# Nothing local lives here: the registry pair is the third Secret in +# ../../../ops/secrets.local.yaml, ignored there. diff --git a/k8s/dolos-publisher/README.md b/k8s/dolos-publisher/README.md new file mode 100644 index 0000000..24025b9 --- /dev/null +++ b/k8s/dolos-publisher/README.md @@ -0,0 +1,99 @@ +--- +provenance: authored +owner: org/founder +tags: [] +--- +# Dolos publisher — deployment unit + +The publisher is one Kubernetes Job per network, rendered from the Helm +chart in [`chart/`](chart/) with a values file per network in +[`solution/stelae/ops/`](../../../ops/README.md), and deployed by the same +two commands as the registry. The +[`stelae-registry-ops`](../../../skills/registry-ops/SKILL.md) skill +holds those commands with the re-run, first-run and monitoring procedures. +Each Job runs the official `ghcr.io/txpipe/dolos` image on the Demeter m2 +EKS cluster: it restores its network's latest stele (or starts from +genesis), replays to the next epoch boundary, publishes, prunes, repeats, +and exits 0 at the aggregator tip. Each publish is its own checkpoint, so a +pod restart costs one epoch and nothing more. + +## What the chart renders + +Release `stelae-publisher-` in namespace `stelae-publisher` — the +registry's namespace, which the chart does not create; the deploy passes +`--namespace`. Two objects: + +- **ConfigMap** `-config`, mounted at `/etc/publisher`: + `entrypoint.sh`, the restore-or-genesis script templated from the chart + (`concurrency` and `--insecure` from values), and `dolos.toml`, the + network's config carried **verbatim** from the values string `dolosToml`. + Its `[snapshot] state_epochs` list is signed input, frozen at the + network's first publish (decisions 0028, 0030, 0038) and extended only + above the then-current tip. Carrying the file verbatim means no template + can change it silently, and its comments stay in the values file. +- **Job** `-backfill-`: the official image pinned by + `image.tag`, `/data` an `emptyDir`, the ConfigMap at `/etc/publisher`, the + registry pair from the Secret named `publisherSecretName` + (`stelae-registry-publisher` — referenced, never templated). + +## The `run` mechanism + +A Job's pod template is immutable: it cannot be upgraded in place. The +chart makes the re-run a deliberate act by putting a counter in the Job's +name. Bumping `run` in the values file and upgrading makes Helm create the +new Job and remove the old one; the new pod restores from `latest` and the +epoch cost is paid on purpose. Changing anything else in the pod template +without a bump fails the upgrade against the immutable field — the guard +the raw manifests lacked. Deleting a Job by hand instead of bumping leaves +the release history and the cluster disagreeing: the next upgrade recreates +it under the old name. Bump, never delete. + +## Values a network must supply + +| value | holds | +|---|---| +| `network` | prefixes the Job name, labels the pod | +| `run` | the re-run counter | +| `repo` | the `oci://` URL the publisher writes | +| `concurrency` | uploads in flight per publish — measured per network, never a default | +| `dolosToml` | the network's `dolos.toml`, verbatim | +| `image.tag` | the `sha-` pin; there is no chart-wide dolos version | +| `resources` | the working set; no default | + +Everything else defaults in [`chart/values.yaml`](chart/values.yaml) +with its reason beside it: `insecure` on (the write path is in-cluster plain +HTTP), `publisherSecretName`, `backoffLimit`, +`terminationGracePeriodSeconds`, and empty placement. + +## Constraints + +- **Concurrency is a network fact.** Mainnet 16 because zot parallelises + where the old Worker did not; the testnets 4 because in-cluster zot + serialises blob commits, and a deeper queue only adds latency to any + mainnet window beside it. Never lowered to make bursts smaller. +- **Single-publisher discipline.** Exactly one writer per + `cardano/`, ever. +- **Placement is a value.** The dedicated `stele-backfill` nodegroup was + deleted on 2026-08-31; every network runs on the shared best-effort pool + today, and a dedicated node again is a `nodeSelector` and `tolerations` + change in that network's values file. + +Local check, no cluster needed: + +```bash +helm lint solution/stelae/codebase/k8s/dolos-publisher/chart \ + --values solution/stelae/ops/values.publisher-preprod.yaml +helm template stelae-publisher-preprod solution/stelae/codebase/k8s/dolos-publisher/chart \ + --namespace stelae-publisher --values solution/stelae/ops/values.publisher-preprod.yaml +``` + +Its predecessors are in git history: the raw manifests this chart replaced +(one Job and ConfigMap per network under `k8s/`, applied by hand, retired +under [publisher-chart](../../../../../archive/plans/dolos-stelae-publication-ops-publisher-chart.md)), +and before them a Cloudflare Worker with a container-backed Durable Object. +The registry moved to the cluster +(`decisions/0037-stelae-registry-on-eks.md`), the Jobs followed, the +Worker's cron was stood down (`decisions/0038-preprod-stele-publication.md`), +and the infrastructure was deleted on 2026-09-06 under +[registry-decommission](../../../../../plans/stelae-registry-decommission.md), +tier 5. diff --git a/k8s/dolos-publisher/chart/.helmignore b/k8s/dolos-publisher/chart/.helmignore new file mode 100644 index 0000000..e5912b7 --- /dev/null +++ b/k8s/dolos-publisher/chart/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +.git/ +.gitignore +*.tmp +*.orig diff --git a/k8s/dolos-publisher/chart/Chart.yaml b/k8s/dolos-publisher/chart/Chart.yaml new file mode 100644 index 0000000..6139889 --- /dev/null +++ b/k8s/dolos-publisher/chart/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: dolos-publisher +description: one stele publisher — a Dolos backfill Job that restores, replays, publishes and prunes one network's stele repository +type: application +# Chart version: this chart. There is no appVersion on purpose: the dolos +# build a release runs is image.tag in the network's values file, and the +# three networks need not agree. +version: 0.1.0 +home: https://github.com/txpipe/stelae +keywords: + - stelae + - oci + - publisher diff --git a/k8s/dolos-publisher/chart/templates/NOTES.txt b/k8s/dolos-publisher/chart/templates/NOTES.txt new file mode 100644 index 0000000..ec63e6b --- /dev/null +++ b/k8s/dolos-publisher/chart/templates/NOTES.txt @@ -0,0 +1,17 @@ +{{ include "dolos-publisher.jobName" . }} in {{ .Release.Namespace }} — {{ .Values.network }}, run {{ .Values.run }}, concurrency {{ .Values.concurrency }}, {{ .Values.image.repository }}:{{ .Values.image.tag }} + +Follow the run: + kubectl -n {{ .Release.Namespace }} logs -f job/{{ include "dolos-publisher.jobName" . }} + +Lines worth reading: + restored own latest stele a pod start paid a full restore + no restorable stele — starting from genesis the repository was empty + replaying toward the next epoch boundary target=N replay started for N + sequence: N (tag epoch-N) publish window opened + the registry failed a round trip; making it again attempt=K + a retry; attempt=3 is worth attention + +Re-run on purpose: bump `run` in the values file and upgrade. Any other +change to the pod template without a bump fails the upgrade against the +Job's immutable template — that failure is the guard, and the answer is +to bump `run`. Never delete the Job by hand. diff --git a/k8s/dolos-publisher/chart/templates/_helpers.tpl b/k8s/dolos-publisher/chart/templates/_helpers.tpl new file mode 100644 index 0000000..45b92fb --- /dev/null +++ b/k8s/dolos-publisher/chart/templates/_helpers.tpl @@ -0,0 +1,14 @@ +{{- define "dolos-publisher.jobName" -}} +{{ required "network is required" .Values.network }}-backfill-{{ required "run is required" .Values.run }} +{{- end -}} + +{{- define "dolos-publisher.configMapName" -}} +{{ .Release.Name }}-config +{{- end -}} + +{{- define "dolos-publisher.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/k8s/dolos-publisher/chart/templates/configmap.yaml b/k8s/dolos-publisher/chart/templates/configmap.yaml new file mode 100644 index 0000000..f9a3b3d --- /dev/null +++ b/k8s/dolos-publisher/chart/templates/configmap.yaml @@ -0,0 +1,48 @@ +{{/* +The restore-or-genesis entrypoint and the network's dolos.toml, mounted at +/etc/publisher. The entrypoint is templated (concurrency, --insecure); the +dolos.toml is the values string verbatim. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolos-publisher.configMapName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "dolos-publisher.labels" . | nindent 4 }} +data: + entrypoint.sh: | + #!/bin/sh + # Restore-or-genesis, then run the backfill loop. + # + # /data is an emptyDir — every pod start begins empty. If the repo holds + # a stele, restore it (pruned by sync.max_history) and continue from + # there; an empty repo means genesis. A transient restore failure + # degrades to a genesis run whose publish lands on Standing::UpToDate + # and no-ops — wasted CPU, never a corrupt repo. + # + # Rendered by the dolos-publisher chart: the concurrency and the + # --insecure flag come from the network's values file, with their + # reasons. + set -eu + + REPO="${PUBLISHER_REPO:?PUBLISHER_REPO is required}" + CONFIG=/etc/publisher/dolos.toml + DATA=/data/db + + if [ ! -d "$DATA" ] || [ -z "$(ls -A "$DATA" 2>/dev/null || true)" ]; then + echo "fresh disk: attempting restore of own latest stele from $REPO" + if dolos --config "$CONFIG" bootstrap stelae --source "$REPO" --point latest{{ if .Values.insecure }} --insecure{{ end }}; then + echo "restored own latest stele" + else + echo "no restorable stele — starting from genesis" + fi + fi + + exec dolos --config "$CONFIG" snapshot backfill --repo "$REPO" \ + {{- if .Values.insecure }} + --insecure \ + {{- end }} + --concurrency {{ required "concurrency is required" .Values.concurrency }} + dolos.toml: | + {{- required "dolosToml is required" .Values.dolosToml | nindent 4 }} diff --git a/k8s/dolos-publisher/chart/templates/job.yaml b/k8s/dolos-publisher/chart/templates/job.yaml new file mode 100644 index 0000000..2852d87 --- /dev/null +++ b/k8s/dolos-publisher/chart/templates/job.yaml @@ -0,0 +1,59 @@ +{{/* +The backfill Job. Its name carries the run counter: a Job's pod template is +immutable, so a re-run is a new Job, and Helm removes the old one in the +same upgrade. Everything else about the pod is a value. +*/}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dolos-publisher.jobName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "dolos-publisher.labels" . | nindent 4 }} +spec: + backoffLimit: {{ .Values.backoffLimit }} + template: + metadata: + labels: + app: {{ .Values.network }}-backfill + spec: + restartPolicy: OnFailure + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + containers: + - name: publisher + image: {{ .Values.image.repository }}:{{ required "image.tag is required" .Values.image.tag }} + command: ["/bin/sh", "/etc/publisher/entrypoint.sh"] + env: + - name: PUBLISHER_REPO + value: {{ required "repo is required" .Values.repo | quote }} + - name: DOLOS_STELAE_REGISTRY_USER + valueFrom: + secretKeyRef: + name: {{ .Values.publisherSecretName }} + key: DOLOS_STELAE_REGISTRY_USER + - name: DOLOS_STELAE_REGISTRY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.publisherSecretName }} + key: DOLOS_STELAE_REGISTRY_PASSWORD + resources: + {{- required "resources is required" .Values.resources | toYaml | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + - name: {{ .Values.network }}-config + mountPath: /etc/publisher + volumes: + - name: data + emptyDir: {} + - name: {{ .Values.network }}-config + configMap: + name: {{ include "dolos-publisher.configMapName" . }} diff --git a/k8s/dolos-publisher/chart/values.yaml b/k8s/dolos-publisher/chart/values.yaml new file mode 100644 index 0000000..bb92b88 --- /dev/null +++ b/k8s/dolos-publisher/chart/values.yaml @@ -0,0 +1,72 @@ +# One stele publisher: a Kubernetes Job running the official dolos image, +# which restores its network's latest stele (or starts from genesis), +# replays to the next epoch boundary, publishes, prunes, and repeats until +# it reaches the aggregator tip. One release per network. +# +# These defaults are instance-agnostic — no network, registry, image pin or +# credential is assumed. Everything a network must supply is empty here and +# `required` in the templates, so a missing value fails at render rather +# than starting a half-configured publisher. This domain's instances are +# The operator supplies one values file per network. +# +# Two values are deliberately without a default. `concurrency` was measured +# per network against the registry it writes to. `dolosToml` carries the +# `[snapshot] state_epochs` list, which is signed input frozen at the +# network's first publish. Neither may be inherited from a chart. + +# The network name. Prefixes the Job name and labels the pod. +network: "" + +# The deliberate re-run counter. The Job is named -backfill-, +# so bumping it makes the upgrade create a new Job and remove the old one, +# paying the restart cost on purpose. A pod template change without a bump +# fails the upgrade against the Job's immutable template, which is the guard +# against re-running by accident. See README.md. +run: "" + +# Where the publisher writes: an oci:// URL to the network's stele +# repository. +repo: "" + +# Uploads in flight during a publish. Measured per network against the +# registry it writes to; never lowered to make bursts smaller. +concurrency: "" + +# The network's dolos.toml, carried verbatim into the ConfigMap — comments +# included — so the frozen state_epochs list and its reasoning stay readable +# in the values file and no template can change signed input. +dolosToml: "" + +image: + # The official image: CI-built from every merge to main, every network's + # genesis files at /etc/genesis//, multi-arch. + repository: ghcr.io/txpipe/dolos + # Pin to the sha- tag of a known commit — never latest. Each + # network's values file holds its own pin; there is no chart-wide dolos + # version. + tag: "" + +# The registry runs in-cluster and the write path is plain HTTP, so both the +# restore and the backfill pass --insecure. Turn off only for a TLS registry. +insecure: true + +# Secret holding DOLOS_STELAE_REGISTRY_USER and DOLOS_STELAE_REGISTRY_PASSWORD. +# Referenced by name, never templated. +publisherSecretName: stelae-registry-publisher + +# Restart budget. Every pod restart costs one epoch by design (/data is an +# emptyDir; the pod restores from latest), and on spot capacity an eviction +# counts here too. +backoffLimit: 200 + +# Covers the driver's SIGTERM handler finishing its chunk and flushing the +# stores. +terminationGracePeriodSeconds: 600 + +# Placement. Empty means anywhere the scheduler likes, which is rarely right +# on a tainted cluster. +nodeSelector: {} +tolerations: [] + +# No default: the working set is a network fact. +resources: {} diff --git a/k8s/registry/.gitignore b/k8s/registry/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/k8s/registry/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/k8s/registry/README.md b/k8s/registry/README.md new file mode 100644 index 0000000..ce68a2a --- /dev/null +++ b/k8s/registry/README.md @@ -0,0 +1,106 @@ +--- +provenance: authored +owner: org/founder +tags: [] +--- +# Stelae registry + +The registry software that serves stele repositories: **zot over an S3 +object store**, packaged so that no cluster, bucket, hostname or credential +is assumed. This directory is the instance-agnostic half. This domain's own +instance — m2 EKS over R2, `oci.stelae.store` — is +[`solution/stelae/ops/`](../../../ops/) +and is operated through the +[`stelae-registry-ops`](../../../skills/registry-ops/SKILL.md) skill. + +Why zot: real registry software over an object store pays ~44 ms per request +where the stock `cloudflare/serverless-registry` Worker paid ~3 s per `PATCH` +and capped a whole mainnet publish at 2.40 MB/s +([`kb/registry-alternatives/`](../../../kb/registry-alternatives/), decision +[0034](../../../../../decisions/0034-stelae-registry-front-end.md)). Why a cluster: +the first zot deployment, in a Cloudflare Container behind a Worker, killed +its own write path under sustained upload and could not be probed when it +broke ([`kb/zot-instance-disconnects/`](../../../kb/zot-instance-disconnects/), +decision [0037](../../../../../decisions/0037-stelae-registry-on-eks.md)). + +``` +publisher ── in-cluster, plain HTTP ──▶ zot ──▶ object store (S3 API) +client ──── public name, TLS ─────────▶ zot ──▶ 307 to a presigned store URL +``` + +| path | holds | +|---|---| +| `chart/` | the Helm chart: one zot replica, `Recreate`, a PVC for the metaDB, a ClusterIP Service for the write path, and two mutually exclusive public-read shapes (ingress or LoadBalancer), both off by default | +| `zot-build/` | an interim zot image, v2.1.20 plus upstream's unreleased blob-`HEAD` walk fix (#4350); retired at v2.1.21 by [stelae-zot-upstream-repin](../../../../../plans/stelae-zot-upstream-repin.md) | + +## The chart + +`values.yaml` is deliberately unusable as-is: every instance fact — store +endpoint and bucket, usernames, placement, public hostname — is empty and +`required` in the templates, so a missing value fails at render rather than +deploying a half-configured registry. The values that look arbitrary are +load-bearing, and each carries its reason in the file: + +- **`redirectBlobURL: true`.** A blob `GET` answers 307 to a presigned store + URL, so payload bytes never enter the pod. Proxied reads would meter every + restore through the cluster and give up the store's egress terms; a blob + `GET` answering `200` means the deployment is wrong. +- **`rootDirectory: "/"`.** The S3 driver applies it twice — `"/zot"` puts + keys at `zot/zot/…` and an empty value is rejected — so `"/"` is the only + value that yields one un-prefixed layout at the bucket root, which is the + layout the bucket holds. `chunkSize` and `multipartCopyThresholdSize` are + JSON **strings** for the same kind of reason: as numbers, zot refuses to + start. +- **No dedupe, no cache driver, no gc.** Dedupe needs a cache driver, which + for a remote store means a remote dependency, and it is an optimization + rather than a correctness property. gc is off because every blob is + referenced by its own `epoch-{E}` tag and nothing is collectable anyway. +- **The metaDB on a PVC.** zot rebuilds its metaDB by parsing every manifest + in the bucket unless it finds its fast-restart stamp. On ephemeral storage + that parse ran on every start and grew with the repository, which is what + crash-looped the registry on its previous substrate; persisted, boot is + seconds. +- **10-minute HTTP timeouts.** A monolithic layer `PUT` has to stream and + commit to the store inside this window; at zot's 60 s default, mid-size + layers behind a busy commit queue were killed and retried forever. + +Secrets are referenced, never templated: the htpasswd Secret (bcrypt only — +zot reads nothing else) and the store credentials, whose keys must be named +`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` because they reach the S3 +driver through the AWS credential chain. The usernames live in both the +values and the htpasswd lines and only convention keeps them matched; +divergence authenticates a user and grants nothing. + +Render and install are the instance's business — for this domain, the two +commands in the ops skill. `templates/NOTES.txt` prints the smoke test: a +`401` on `/v2/`, a sub-second blob `HEAD`, and a `307` on a blob `GET`. + +## The interim image + +zot v2.1.17–v2.1.20 resolve a blob `HEAD`'s Content-Type by walking the +repository's manifests out of the object store — O(tags) per request, 143 s +on `cardano/mainnet` at ~300 tags, which made publishing impossible. +Upstream fixed it in fe0679da (#4350), unreleased as of 2026-08-29, and no +released version both has `redirectBlobURL` and lacks the walk. `zot-build/` +carries exactly that one patch on top of v2.1.20; its Dockerfile holds the +build-and-push commands, and the pushed digest goes into the instance's +values. The acceptance test is the sub-second blob `HEAD`: stock takes 143 s +on the reference digest, patched takes 0.8 s. + +## History + +Two Cloudflare front ends preceded this: the stock `serverless-registry` +Worker over the `stelae-registry` bucket, and the first zot deployment in a +Cloudflare Container behind a Worker router. Both were decommissioned under +[stelae-registry-decommission](../../../../../plans/stelae-registry-decommission.md). +The attempts that localized the Container fault are recorded in +[`kb/zot-instance-disconnects/`](../../../kb/zot-instance-disconnects/); the deploy +unit itself lives in git history at the 2026-08-29 commit. + +The steles themselves were moved once, on 2026-08-28, from the retired +Worker's R2 layout into zot's by a control-plane copy tool that lived here +as `migrate/`. It was removed on 2026-09-06: its source layout no longer +exists and everything it carried over has since been re-published from +scratch, so nothing remained for it to convert or verify. The account of +that job — cost, method, byte-for-byte verification — is +[`kb/registry-alternatives/migration.md`](../../../kb/registry-alternatives/migration.md). diff --git a/k8s/registry/chart/.helmignore b/k8s/registry/chart/.helmignore new file mode 100644 index 0000000..e5912b7 --- /dev/null +++ b/k8s/registry/chart/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +.git/ +.gitignore +*.tmp +*.orig diff --git a/k8s/registry/chart/Chart.yaml b/k8s/registry/chart/Chart.yaml new file mode 100644 index 0000000..d7f59ad --- /dev/null +++ b/k8s/registry/chart/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: stelae-registry +description: zot as the stelae OCI registry, serving stele repositories over R2's S3 API +type: application +# Chart version: this chart. appVersion: the zot build it deploys — bump both +# at the v2.1.21 re-pin (plans/stelae-zot-upstream-repin.md). +version: 0.1.0 +appVersion: v2.1.20-pr4350 +home: https://github.com/txpipe/stelae +keywords: + - stelae + - oci + - registry diff --git a/k8s/registry/chart/templates/NOTES.txt b/k8s/registry/chart/templates/NOTES.txt new file mode 100644 index 0000000..ed2e383 --- /dev/null +++ b/k8s/registry/chart/templates/NOTES.txt @@ -0,0 +1,24 @@ +{{ .Chart.Name }} {{ .Chart.Version }} — zot {{ .Chart.AppVersion }} + +Write path (publishers, in-cluster, plain HTTP): + oci://{{ include "stelae-registry.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }}/cardano/mainnet +{{ if .Values.ingress.enabled }} +Read path (public, ingress shape): + https://{{ .Values.ingress.host }}/v2/ + Needs a DNS record for {{ .Values.ingress.host }} pointing at the class + `{{ .Values.ingress.className }}` load balancer before cert-manager can pass http01. +{{ end }} +{{- if .Values.loadBalancer.enabled }} +Read path (public, LoadBalancer shape): + kubectl -n {{ .Release.Namespace }} get svc {{ include "stelae-registry.fullname" . }}-public \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' + Point the public hostname at that address (DNS-only / grey cloud). +{{- end }} +Smoke test: + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "stelae-registry.fullname" . }} 15000:{{ .Values.service.port }} + curl -sS -o /dev/null -w '%{http_code} %{time_total}s\n' http://localhost:15000/v2/ # 401, fast + curl -sS -o /dev/null -w '%{http_code} %{time_total}s\n' -u USER:PASS -I \ + http://localhost:15000/v2/cardano/mainnet/blobs/sha256: # 200, sub-second + +A blob GET must answer 307, never 200: reads redirect to presigned R2 URLs, +and a 200 means payload bytes are crossing the pod. diff --git a/k8s/registry/chart/templates/_helpers.tpl b/k8s/registry/chart/templates/_helpers.tpl new file mode 100644 index 0000000..4a6466a --- /dev/null +++ b/k8s/registry/chart/templates/_helpers.tpl @@ -0,0 +1,46 @@ +{{- define "stelae-registry.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "stelae-registry.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "stelae-registry.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "stelae-registry.labels" -}} +helm.sh/chart: {{ include "stelae-registry.chart" . }} +{{ include "stelae-registry.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels are the bare `app:` key the pre-chart manifests used, not the +app.kubernetes.io pair. A Deployment's selector is immutable, so keeping it +lets the running registry be adopted by `helm upgrade --take-ownership` +instead of being deleted and recreated mid-publish. +*/}} +{{- define "stelae-registry.selectorLabels" -}} +app: {{ include "stelae-registry.fullname" . }} +{{- end -}} + +{{- define "stelae-registry.image" -}} +{{- $img := .Values.image -}} +{{- if $img.digest -}} +{{- printf "%s:%s@%s" $img.repository $img.tag $img.digest -}} +{{- else -}} +{{- printf "%s:%s" $img.repository $img.tag -}} +{{- end -}} +{{- end -}} diff --git a/k8s/registry/chart/templates/configmap.yaml b/k8s/registry/chart/templates/configmap.yaml new file mode 100644 index 0000000..d510d98 --- /dev/null +++ b/k8s/registry/chart/templates/configmap.yaml @@ -0,0 +1,65 @@ +{{/* +zot's config.json. The htpasswd file it references is mounted alongside from +a pre-created Secret, projected into the same directory by the Deployment. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "stelae-registry.fullname" . }}-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "stelae-registry.labels" . | nindent 4 }} +data: + config.json: | + { + "storage": { + "rootDirectory": {{ .Values.persistence.mountPath | quote }}, + "dedupe": {{ .Values.storage.dedupe }}, + "remoteCache": {{ .Values.storage.remoteCache }}, + "gc": {{ .Values.storage.gc }}, + "commit": {{ .Values.storage.commit }}, + "redirectBlobURL": {{ .Values.storage.redirectBlobURL }}, + "fastRestart": {{ .Values.storage.fastRestart }}, + "storageDriver": { + "name": "s3", + "rootdirectory": {{ .Values.s3.rootDirectory | quote }}, + "region": {{ .Values.s3.region | quote }}, + "regionendpoint": {{ required "s3.endpoint is required" .Values.s3.endpoint | quote }}, + "bucket": {{ required "s3.bucket is required" .Values.s3.bucket | quote }}, + "forcepathstyle": true, + "secure": true, + "skipverify": false, + "chunksize": {{ .Values.s3.chunkSize | quote }}, + "multipartcopythresholdsize": {{ .Values.s3.multipartCopyThresholdSize | quote }} + } + }, + "http": { + "address": "0.0.0.0", + "port": {{ .Values.http.port | quote }}, + "readTimeout": {{ .Values.http.readTimeout | quote }}, + "writeTimeout": {{ .Values.http.writeTimeout | quote }}, + "auth": { + "htpasswd": { "path": "/etc/zot/htpasswd" }, + "failDelay": 1 + }, + "accessControl": { + "repositories": { + "**": { + "policies": [ + { + "users": [{{ required "auth.readonlyUsername is required" .Values.auth.readonlyUsername | quote }}], + "actions": ["read"] + } + ], + "defaultPolicy": [], + "anonymousPolicy": [] + } + }, + "adminPolicy": { + "users": [{{ required "auth.publisherUsername is required" .Values.auth.publisherUsername | quote }}], + "actions": ["read", "create", "update", "delete"] + } + } + }, + "log": { "level": {{ .Values.log.level | quote }} } + } diff --git a/k8s/registry/chart/templates/deployment.yaml b/k8s/registry/chart/templates/deployment.yaml new file mode 100644 index 0000000..10d1efd --- /dev/null +++ b/k8s/registry/chart/templates/deployment.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "stelae-registry.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "stelae-registry.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "stelae-registry.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "stelae-registry.selectorLabels" . | nindent 8 }} + annotations: + # Roll the pod when config.json changes; zot reads it only at start. + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: zot + image: {{ include "stelae-registry.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [{{ .Values.image.binary | quote }}, "serve", "/etc/zot/config.json"] + ports: + - containerPort: {{ .Values.http.port | int }} + name: http + envFrom: + # R2 keys as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY: the S3 + # driver picks them up from the AWS credential chain, so nothing + # lands in config.json or the image. + - secretRef: + name: {{ .Values.auth.r2SecretName }} + startupProbe: + tcpSocket: { port: {{ .Values.http.port | int }} } + periodSeconds: {{ .Values.probes.startup.periodSeconds }} + failureThreshold: {{ .Values.probes.startup.failureThreshold }} + readinessProbe: + tcpSocket: { port: {{ .Values.http.port | int }} } + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + livenessProbe: + tcpSocket: { port: {{ .Values.http.port | int }} } + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: zot-config + mountPath: /etc/zot + readOnly: true + - name: state + mountPath: {{ .Values.persistence.mountPath }} + volumes: + # config.json is chart-rendered, htpasswd is pre-created; zot wants + # both in one directory, which is what the projection gives. + - name: zot-config + projected: + sources: + - configMap: + name: {{ include "stelae-registry.fullname" . }}-config + items: + - key: config.json + path: config.json + - secret: + name: {{ .Values.auth.htpasswdSecretName }} + items: + - key: htpasswd + path: htpasswd + - name: state + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "stelae-registry.fullname" . }}-state + {{- else }} + emptyDir: {} + {{- end }} diff --git a/k8s/registry/chart/templates/ingress.yaml b/k8s/registry/chart/templates/ingress.yaml new file mode 100644 index 0000000..3e88f17 --- /dev/null +++ b/k8s/registry/chart/templates/ingress.yaml @@ -0,0 +1,31 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "stelae-registry.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "stelae-registry.labels" . | nindent 4 }} + annotations: + cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }} + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - backend: + service: + name: {{ include "stelae-registry.fullname" . }} + port: + number: {{ .Values.service.port }} + path: / + pathType: Prefix + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tlsSecretName }} +{{- end }} diff --git a/k8s/registry/chart/templates/pvc.yaml b/k8s/registry/chart/templates/pvc.yaml new file mode 100644 index 0000000..4a72a90 --- /dev/null +++ b/k8s/registry/chart/templates/pvc.yaml @@ -0,0 +1,21 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "stelae-registry.fullname" . }}-state + namespace: {{ .Release.Namespace }} + labels: + {{- include "stelae-registry.labels" . | nindent 4 }} + annotations: + # The metaDB is expensive to rebuild and the whole point of this volume; + # a helm uninstall must not take it. + helm.sh/resource-policy: keep +spec: + accessModes: [ReadWriteOnce] + {{- with .Values.persistence.storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/k8s/registry/chart/templates/service-public.yaml b/k8s/registry/chart/templates/service-public.yaml new file mode 100644 index 0000000..f1e3e0c --- /dev/null +++ b/k8s/registry/chart/templates/service-public.yaml @@ -0,0 +1,28 @@ +{{/* +The public read path, LoadBalancer shape: a second Service so the in-cluster +ClusterIP one (the publishers' write path) is never widened. The cloud's LB +controller owns the actual load balancer's lifecycle — it exists because +this object exists and dies with it, which is what keeps the exit an +annotation-and-DNS change rather than a teardown project. +*/}} +{{- if .Values.loadBalancer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "stelae-registry.fullname" . }}-public + namespace: {{ .Release.Namespace }} + labels: + {{- include "stelae-registry.labels" . | nindent 4 }} + {{- with .Values.loadBalancer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: LoadBalancer + selector: + {{- include "stelae-registry.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.loadBalancer.port }} + targetPort: {{ .Values.http.port | int }} + name: public +{{- end }} diff --git a/k8s/registry/chart/templates/service.yaml b/k8s/registry/chart/templates/service.yaml new file mode 100644 index 0000000..f133c39 --- /dev/null +++ b/k8s/registry/chart/templates/service.yaml @@ -0,0 +1,20 @@ +{{/* +The publishers' write path: + oci://stelae-registry.stelae-publisher.svc.cluster.local:5000/ +reached with dolos --insecure, so no TLS and no Cloudflare hop touches it. +*/}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "stelae-registry.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "stelae-registry.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + selector: + {{- include "stelae-registry.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.http.port | int }} + name: http diff --git a/k8s/registry/chart/values.yaml b/k8s/registry/chart/values.yaml new file mode 100644 index 0000000..6828fb1 --- /dev/null +++ b/k8s/registry/chart/values.yaml @@ -0,0 +1,159 @@ +# zot as a stelae registry: real registry software over an object store, with +# blob reads redirected to the store so payload bytes never cross the pod. +# +# These defaults are instance-agnostic — no cluster, bucket, hostname or +# credential is assumed. Everything an instance must supply is empty here and +# `required` in the templates, so a missing value fails at render rather than +# deploying a half-configured registry. This domain's instance supplies them +# from `solution/stelae/ops/` (values file + secrets manifest + runbook). +# +# Values that look arbitrary carry their reason. Several are load-bearing — +# `redirectBlobURL`, the JSON-string sizes, `rootDirectory: "/"`, the HTTP +# timeouts — and the evidence for them is in +# `solution/stelae/kb/zot-instance-disconnects/`. + +nameOverride: "" +fullnameOverride: "" + +image: + # The upstream release, which is what a fresh instance should run. + # + # Caveat with teeth: zot v2.1.17–v2.1.20 answer a blob HEAD's Content-Type + # by walking the repo's manifests out of the object store — O(tags) per + # request, which at ~300 tags measured 143 s per HEAD and made publishing + # impossible. Upstream fixed it in commit fe0679da (#4350), unreleased as + # of 2026-08-29. An instance publishing at any scale must either run + # v2.1.21+ once it ships, or override this with a patched build (ours is + # ../zot-build/, wired up in solution/stelae/ops/values.registry-m2.yaml). + repository: ghcr.io/project-zot/zot-linux-arm64 + tag: v2.1.20 + digest: sha256:56230c5a589eb55acc57afc34307f6ea1b2efe5cf8e0057ccca64099ba837ff6 + pullPolicy: IfNotPresent + # Upstream images name the binary by platform; a custom build may not. + binary: /usr/local/bin/zot-linux-arm64 + +# One replica, Recreate: chunked upload sessions and the metaDB live on an +# RWO volume, so two pods must never overlap. +replicaCount: 1 + +# Instance placement. Empty means "anywhere the scheduler likes", which is +# rarely right on a tainted cluster. +nodeSelector: {} +tolerations: [] + +resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi + +persistence: + enabled: true + size: 10Gi + # Empty string uses the cluster default StorageClass. + storageClass: "" + # zot rebuilds its metaDB by parsing every manifest in the bucket unless it + # finds its fast-restart stamp here. On ephemeral storage that parse runs on + # every start and grows with the repository — it is what crash-looped this + # registry on its previous substrate, against a platform start deadline. + # Persist it and boot is seconds. + mountPath: /var/lib/zot + +service: + type: ClusterIP + port: 5000 + +# Public reads — two mutually exclusive shapes, both off by default. +# Publishers use neither: they reach the ClusterIP Service in-cluster over +# plain HTTP. Blob payloads use neither: reads are 307s to presigned +# object-store URLs, so what crosses the public path is auth, manifests and +# redirects. + +# Shape 1: a cluster ingress controller + cert-manager certificate. +ingress: + enabled: false + className: "" + # Optional: cert-manager issues the certificate named in tlsSecretName. + clusterIssuer: "" + host: "" + tlsSecretName: "" + annotations: {} + +# Shape 2: a second Service of type LoadBalancer, rendered only when +# enabled. The chart is dialect-neutral: whatever makes the cloud terminate +# TLS and pick subnets is the instance's annotations, not the chart's +# business. The ClusterIP Service stays untouched either way, so enabling +# this never exposes the publishers' plain-HTTP port by accident. +loadBalancer: + enabled: false + port: 443 + annotations: {} + +# The object store behind the registry. No default is possible. +s3: + endpoint: "" + bucket: "" + region: auto + # The S3 driver applies rootdirectory twice — "/zot" yields keys at + # zot/zot/..., an empty value is rejected — so "/" is the only value that + # gives one un-prefixed layout at the bucket root. + rootDirectory: "/" + # Both are JSON strings on purpose: as numbers, zot refuses to start. + chunkSize: "10485760" + multipartCopyThresholdSize: "5368709120" + +storage: + # A blob GET answered with a 307 to a presigned URL, so payload bytes never + # enter the pod. Not a tweak: proxied reads meter every restore through the + # cluster and give up the store's egress terms. If a blob GET ever answers + # 200 instead of 307, the deployment is wrong. + redirectBlobURL: true + fastRestart: true + # Dedupe needs a cache driver, which for a remote store means a remote + # dependency; it is an optimization, not a correctness property. gc is off + # where every blob is referenced by its own tag and nothing is collectable. + dedupe: false + remoteCache: false + gc: false + commit: false + +http: + port: "5000" + # 10 m, not zot's 60 s default: a monolithic layer PUT has to stream and + # commit to the store inside this window, and at 60 s mid-size layers behind + # a busy commit queue were killed and retried forever. + readTimeout: 10m + writeTimeout: 10m + +auth: + # Reach zot's accessControl policies, and must match the htpasswd entries + # in the Secret below — divergence authenticates a user and grants nothing. + publisherUsername: "" + readonlyUsername: "" + # Referenced, never templated: htpasswd needs bcrypt hashing no template can + # do, and object-store keys are credentials. The store Secret's keys reach + # the S3 driver through the AWS credential chain, so they must be named + # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY. + htpasswdSecretName: stelae-registry-htpasswd + r2SecretName: stelae-registry-r2 + +log: + level: info + +# TCP probes because every HTTP route is behind htpasswd. The startup budget +# is generous for one reason: the first boot on an empty volume runs the full +# metaDB parse over the bucket. Every later boot finds the stamp. +probes: + startup: + periodSeconds: 10 + failureThreshold: 90 + readiness: + periodSeconds: 10 + liveness: + periodSeconds: 30 + +podAnnotations: {} +podSecurityContext: {} +securityContext: {} diff --git a/k8s/registry/zot-build/4350-drop-index-walk.patch b/k8s/registry/zot-build/4350-drop-index-walk.patch new file mode 100644 index 0000000..4ae6eaf --- /dev/null +++ b/k8s/registry/zot-build/4350-drop-index-walk.patch @@ -0,0 +1,568 @@ +From fe0679dab2a9e8e307da893a384f212841d75840 Mon Sep 17 00:00:00 2001 +From: datadot +Date: Tue, 25 Aug 2026 07:54:52 +0100 +Subject: [PATCH] perf: drop the index walk on blob Content-Type (#4350) + +* perf: drop the index walk on blob Content-Type + +HEAD/GET on a blob resolved the response Content-Type through +GetBlobDescriptorFromRepo, which reads index.json and then fetches +every manifest in the repo until it finds the digest. That is +O(manifests) storage reads per request, and on a remote driver it +dominates the response: roughly 0.145s per manifest, so minutes on a +repo with several hundred tags (#4344). + +Content-Type is not specified for blob pulls. The dist-spec requires +it for manifest pulls only, and the helper already fell back to +application/octet-stream whenever the lookup failed, so the walk only +ever upgraded a header that nothing is entitled to depend on. + +Blob responses now always advertise application/octet-stream, which +stays non-empty and parseable for consumers such as +stargz-snapshotter. Manifest media-type resolution is untouched. This +reverts the blob half of #4325; the tests that pinned the descriptor +type now assert the constant and additionally fail if the blob path +reads the index at all. + +Signed-off-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> + +* docs: document octet-stream on blob responses + +Both blob endpoints now always answer with application/octet-stream, so +the swagger annotations follow: the GET endpoint declared the gzipped +layer media type, and the HEAD endpoint declared json while its handler +sets the same octet-stream header the GET one does. + +buildMultipartPartHeader still described resolving a descriptor media +type, which is the lookup this branch removes. + +swagger/ regenerated with make swagger. + +Signed-off-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> + +* docs: address review on blob Content-Type docs + +Drops the repeated comment above the index-read counters: the assertion +message already explains why the reads are counted (@vrajashkr). + +Rewrites the section and fixture documentation, which still described +deriving the response type from the blob's descriptor. The fixture stays, +because the tests use it to prove the blob path never reads the index, +but it no longer reads as if that lookup were live (@copilot). + +Restores @Produce json on the blob HEAD endpoint and adds +application/octet-stream alongside it, rather than replacing it: the +endpoint answers errors with JSON and successes with headers only +(@vrajashkr). swagger/ regenerated. + +Signed-off-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> + +* docs: document json on the blob GET produces list + +The blob GET endpoint answers errors with JSON through WriteJSON, so it +produces both types, the same as HEAD. Success responses are unchanged +and still carry application/octet-stream. + +swagger/ regenerated. + +Signed-off-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> + +* docs: trim the blob Content-Type test preamble + +Drops the sentence on how the response type used to be derived. The +tests document what the code does now. + +Signed-off-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> + +--------- + +Signed-off-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> +Co-authored-by: Paul Kennedy <9027062+datadot@users.noreply.github.com> +--- + pkg/api/routes.go | 55 +++----------- + pkg/api/routes_test.go | 165 +++++++++++++++++++++-------------------- + swagger/docs.go | 6 +- + swagger/swagger.json | 6 +- + swagger/swagger.yaml | 4 +- + 5 files changed, 107 insertions(+), 129 deletions(-) + +diff --git a/pkg/api/routes.go b/pkg/api/routes.go +index d08e0ece17..806c850c36 100644 +--- a/pkg/api/routes.go ++++ b/pkg/api/routes.go +@@ -13,7 +13,6 @@ import ( + "errors" + "fmt" + "io" +- "mime" + "mime/multipart" + "net/http" + "net/textproto" +@@ -1051,41 +1050,12 @@ func canMount(userAc *reqCtx.UserAccessControl, imgStore storageTypes.ImageStore + return canMount, nil + } + +-// resolveBlobResponseMediaType resolves the OCI media type to advertise for a blob via +-// the repo's index/manifests. If the descriptor lookup fails (or the descriptor +-// has no media type), it falls back to application/octet-stream. +-// +-// Use this for Content-Type on HEAD/GET blob responses to satisfy OCI +-// distribution-spec conformance and consumers like stargz-snapshotter that +-// require a non-empty, well-formed media type. +-func resolveBlobResponseMediaType( +- imgStore storageTypes.ImageStore, +- repo string, +- digest godigest.Digest, +- logger log.Logger, +-) string { +- desc, err := storageCommon.GetBlobDescriptorFromRepo(imgStore, repo, digest, logger) +- if err == nil && desc.MediaType != "" { +- // Descriptor media types originate from manifest JSON and are not +- // necessarily validated. Ensure we only emit a header-safe, parseable +- // media type; otherwise fall back to application/octet-stream. +- // +- // ParseMediaType also strips parameters so we only propagate the base +- // type (e.g. "application/vnd.oci.image.layer.v1.tar+gzip"). +- mediaType, _, parseErr := mime.ParseMediaType(desc.MediaType) +- if parseErr == nil && mediaType != "" { +- return mediaType +- } +- } +- +- return constants.BinaryMediaType +-} +- + // CheckBlob godoc + // @Summary Check image blob/layer + // @Description Check an image's blob/layer given a digest + // @Accept json + // @Produce json ++// @Produce application/octet-stream + // @Param name path string true "repository name" + // @Param digest path string true "blob/layer digest" + // @Success 200 {object} api.ImageManifest +@@ -1175,7 +1145,7 @@ func (rh *RouteHandler) CheckBlob(response http.ResponseWriter, request *http.Re + + response.Header().Set("Content-Length", strconv.FormatInt(blen, 10)) + response.Header().Set("Accept-Ranges", "bytes") +- response.Header().Set("Content-Type", resolveBlobResponseMediaType(imgStore, name, digest, rh.c.Log)) ++ response.Header().Set("Content-Type", constants.BinaryMediaType) + response.Header().Set(constants.DistContentDigestKey, digest.String()) + response.WriteHeader(http.StatusOK) + } +@@ -1344,10 +1314,10 @@ func buildMultipartPartHeader(rng httpRange, mediaType string, size int64) textp + partHeader := textproto.MIMEHeader{} + partHeader.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", rng.start, rng.end, size)) + +- // RFC 9110 §15.3.7 lets us omit per-part Content-Type, but when the +- // caller has resolved a real media type (descriptor lookup succeeded +- // or fell back to application/octet-stream) we propagate it so OCI +- // clients don't have to re-derive it from the index for every part. ++ // RFC 9110 §15.3.7 lets us omit per-part Content-Type, but when the caller ++ // has a media type for the blob we propagate it to each part so clients do ++ // not have to derive one themselves. Blob responses always carry ++ // application/octet-stream. + if mediaType != "" { + partHeader.Set("Content-Type", mediaType) + } +@@ -1478,7 +1448,8 @@ func normalizeBlobRedirectURL(rawURL string) (string, bool) { + // @Summary Get image blob/layer + // @Description Get an image's blob/layer given a digest + // @Accept json +-// @Produce application/vnd.oci.image.layer.v1.tar+gzip ++// @Produce json ++// @Produce application/octet-stream + // @Param name path string true "repository name" + // @Param digest path string true "blob/layer digest" + // @Header 200 {string} Docker-Content-Digest "Manifest digest of the content" +@@ -1573,9 +1544,7 @@ func (rh *RouteHandler) GetBlob(response http.ResponseWriter, request *http.Requ + return + } + +- // Resolve the response Content-Type from the blob's OCI descriptor (if +- // any), with a fallback to application/octet-stream. +- mediaType := resolveBlobResponseMediaType(imgStore, name, digest, rh.c.Log) ++ mediaType := constants.BinaryMediaType + + ranges, err := parseRangeHeader(contentRange, bsize) + if err != nil { +@@ -1641,11 +1610,7 @@ func (rh *RouteHandler) GetBlob(response http.ResponseWriter, request *http.Requ + + var blen int64 + +- // Resolve the response Content-Type from the blob's OCI descriptor +- // (if any), with a fallback to application/octet-stream. This lookup +- // may require an additional repo index/manifest walk before we read +- // the blob, but preserves a more specific Content-Type when available. +- mediaType := resolveBlobResponseMediaType(imgStore, name, digest, rh.c.Log) ++ mediaType := constants.BinaryMediaType + + repo, blen, err := imgStore.GetBlob(name, digest, mediaType) + if err != nil { +diff --git a/pkg/api/routes_test.go b/pkg/api/routes_test.go +index 0f79c781bc..94a693e71f 100644 +--- a/pkg/api/routes_test.go ++++ b/pkg/api/routes_test.go +@@ -2339,17 +2339,14 @@ func TestWriteDataFromReader(t *testing.T) { + }) + } + +-// Descriptor-aware Content-Type tests for blob HEAD/GET. ++// Content-Type tests for blob HEAD/GET. + // +-// The blob endpoints derive the response Content-Type from the OCI +-// descriptor associated with the blob (via the repo's index/manifest +-// chain), and fall back to application/octet-stream when no such +-// descriptor is available. These tests use mock image stores to drive +-// both branches independently of the on-disk storage layer. ++// Blob responses always advertise application/octet-stream. These tests use mock ++// image stores that serve an index and manifest chain, so they can assert both the ++// constant header and that the blob path never reads the index. + + // descriptorTestDigests returns deterministic layer, manifest, and +-// config digests (in that order) used by the descriptor-aware +-// Content-Type tests. ++// config digests (in that order) used by the blob Content-Type tests. + func descriptorTestDigests() (godigest.Digest, godigest.Digest, godigest.Digest) { + return godigest.FromString("layer"), godigest.FromString("manifest"), godigest.FromString("config") + } +@@ -2369,9 +2366,9 @@ func newBlobTestRouteHandler(t *testing.T, store mocks.MockedImageStore) *api.Ro + return api.NewRouteHandler(ctlr) + } + +-// descriptorFixture builds a minimal index -> manifest -> layer chain +-// that resolves the layer digest from descriptorTestDigests to +-// MediaTypeImageLayerGzip. ++// descriptorFixture builds a minimal index -> manifest -> layer chain that would ++// resolve the layer digest from descriptorTestDigests to MediaTypeImageLayerGzip. ++// It exists so the tests can prove the blob path leaves it untouched. + func descriptorFixture(t *testing.T) ([]byte, []byte) { + t.Helper() + +@@ -2416,10 +2413,11 @@ func descriptorFixture(t *testing.T) ([]byte, []byte) { + return indexJSON, manifestJSON + } + +-// descriptorStore returns a mock store backed by descriptorFixture. +-// Looking up the layer digest from descriptorTestDigests resolves to a +-// layer with media type MediaTypeImageLayerGzip via the index walk; +-// other digests fall through to the binary fallback. ++// descriptorStore returns a mock store backed by descriptorFixture, whose index ++// and manifest would resolve the layer digest from descriptorTestDigests to ++// MediaTypeImageLayerGzip. Blob responses no longer consult it: the tests below ++// keep the fixture so they can assert that the blob path never reads it, since ++// that lookup was O(manifests) per request (project-zot/zot#4344). + func descriptorStore(t *testing.T) mocks.MockedImageStore { + t.Helper() + +@@ -2446,8 +2444,21 @@ func descriptorStore(t *testing.T) mocks.MockedImageStore { + } + } + +-func TestCheckBlobUsesDescriptorContentType(t *testing.T) { ++func TestCheckBlobAlwaysBinaryContentType(t *testing.T) { + store := descriptorStore(t) ++ ++ var indexReads atomic.Int32 ++ ++ store.GetIndexContentFn = func(repo string) ([]byte, error) { ++ indexReads.Add(1) ++ ++ return []byte("{}"), nil ++ } ++ store.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { ++ indexReads.Add(1) ++ ++ return []byte("{}"), nil ++ } + store.CheckBlobFn = func(ctx context.Context, repo string, digest godigest.Digest) (bool, int64, error) { + return true, 42, nil + } +@@ -2475,9 +2486,11 @@ func TestCheckBlobUsesDescriptorContentType(t *testing.T) { + defer resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, resp.Header.Get("Content-Type")) ++ assert.Equal(t, constants.BinaryMediaType, resp.Header.Get("Content-Type")) + assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges")) + assert.Equal(t, layerDigest.String(), resp.Header.Get(constants.DistContentDigestKey)) ++ ++ assert.Zero(t, indexReads.Load(), "blob path must not walk the repo index") + } + + func TestCheckBlobFallsBackToBinaryContentType(t *testing.T) { +@@ -2517,70 +2530,30 @@ func TestCheckBlobFallsBackToBinaryContentType(t *testing.T) { + assert.Equal(t, constants.BinaryMediaType, resp.Header.Get("Content-Type")) + } + +-func TestGetBlobUsesDescriptorContentType(t *testing.T) { ++func TestGetBlobAlwaysBinaryContentType(t *testing.T) { + store := descriptorStore(t) +- store.GetBlobFn = func(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { +- // The mediaType argument forwarded to the storage layer is a +- // hint and is currently ignored; we still feed it the resolved +- // value so the surface stays consistent. +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, mediaType) + +- return io.NopCloser(strings.NewReader("blob")), 4, nil +- } +- +- handler := newBlobTestRouteHandler(t, store) +- +- layerDigest, _, _ := descriptorTestDigests() ++ var indexReads atomic.Int32 + +- req := httptest.NewRequestWithContext( +- context.Background(), +- http.MethodGet, +- "http://example.com/v2/test/blobs/sha256:test", +- http.NoBody, +- ) +- // Wildcard / mixed Accept must not leak into the response. +- req.Header.Set("Accept", "application/vnd.oci.image.layer.v1.tar+gzip, */*") +- req = mux.SetURLVars(req, map[string]string{ +- "name": "test", +- "digest": layerDigest.String(), +- }) +- +- rec := httptest.NewRecorder() +- handler.GetBlob(rec, req) +- +- resp := rec.Result() +- defer resp.Body.Close() ++ store.GetIndexContentFn = func(repo string) ([]byte, error) { ++ indexReads.Add(1) + +- require.Equal(t, http.StatusOK, resp.StatusCode) +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, resp.Header.Get("Content-Type")) +-} ++ return []byte("{}"), nil ++ } ++ store.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { ++ indexReads.Add(1) + +-func TestGetBlobFallsBackOnInvalidDescriptorContentType(t *testing.T) { +- // Descriptor media types are user-supplied and may be invalid as HTTP +- // header values. resolveBlobResponseMediaType must sanitize/validate +- // and fall back to application/octet-stream on parse failure. +- store := descriptorStore(t) ++ return []byte("{}"), nil ++ } + store.GetBlobFn = func(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { ++ // The mediaType argument forwarded to the storage layer is a ++ // hint and is currently ignored; we still feed it the resolved ++ // value so the surface stays consistent. + assert.Equal(t, constants.BinaryMediaType, mediaType) + + return io.NopCloser(strings.NewReader("blob")), 4, nil + } + +- // Force descriptor lookup success but with an invalid media type string. +- store.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { +- _, manifestJSON := descriptorFixture(t) +- +- var manifest ispec.Manifest +- require.NoError(t, json.Unmarshal(manifestJSON, &manifest)) +- require.Len(t, manifest.Layers, 1) +- manifest.Layers[0].MediaType = "bad\r\nvalue" +- +- out, err := json.Marshal(manifest) +- require.NoError(t, err) +- +- return out, nil +- } +- + handler := newBlobTestRouteHandler(t, store) + + layerDigest, _, _ := descriptorTestDigests() +@@ -2591,6 +2564,8 @@ func TestGetBlobFallsBackOnInvalidDescriptorContentType(t *testing.T) { + "http://example.com/v2/test/blobs/sha256:test", + http.NoBody, + ) ++ // Wildcard / mixed Accept must not leak into the response. ++ req.Header.Set("Accept", "application/vnd.oci.image.layer.v1.tar+gzip, */*") + req = mux.SetURLVars(req, map[string]string{ + "name": "test", + "digest": layerDigest.String(), +@@ -2604,6 +2579,8 @@ func TestGetBlobFallsBackOnInvalidDescriptorContentType(t *testing.T) { + + require.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, constants.BinaryMediaType, resp.Header.Get("Content-Type")) ++ ++ assert.Zero(t, indexReads.Load(), "blob path must not walk the repo index") + } + + func TestGetBlobFallsBackToBinaryContentType(t *testing.T) { +@@ -2645,8 +2622,21 @@ func TestGetBlobFallsBackToBinaryContentType(t *testing.T) { + assert.Equal(t, constants.BinaryMediaType, resp.Header.Get("Content-Type")) + } + +-func TestGetBlobPartialUsesDescriptorContentType(t *testing.T) { ++func TestGetBlobPartialAlwaysBinaryContentType(t *testing.T) { + store := descriptorStore(t) ++ ++ var indexReads atomic.Int32 ++ ++ store.GetIndexContentFn = func(repo string) ([]byte, error) { ++ indexReads.Add(1) ++ ++ return []byte("{}"), nil ++ } ++ store.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { ++ indexReads.Add(1) ++ ++ return []byte("{}"), nil ++ } + store.GetBlobPartialFn = func( + repo string, + digest godigest.Digest, +@@ -2654,7 +2644,7 @@ func TestGetBlobPartialUsesDescriptorContentType(t *testing.T) { + from, + to int64, + ) (io.ReadCloser, int64, int64, error) { +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, mediaType) ++ assert.Equal(t, constants.BinaryMediaType, mediaType) + assert.Equal(t, int64(0), from) + assert.Equal(t, int64(1), to) + +@@ -2684,9 +2674,11 @@ func TestGetBlobPartialUsesDescriptorContentType(t *testing.T) { + defer resp.Body.Close() + + require.Equal(t, http.StatusPartialContent, resp.StatusCode) +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, resp.Header.Get("Content-Type")) ++ assert.Equal(t, constants.BinaryMediaType, resp.Header.Get("Content-Type")) + assert.Equal(t, "bytes 0-1/4", resp.Header.Get("Content-Range")) + assert.Equal(t, layerDigest.String(), resp.Header.Get(constants.DistContentDigestKey)) ++ ++ assert.Zero(t, indexReads.Load(), "blob path must not walk the repo index") + } + + func TestGetBlobPartialFallsBackToBinaryContentType(t *testing.T) { +@@ -2735,13 +2727,26 @@ func TestGetBlobPartialFallsBackToBinaryContentType(t *testing.T) { + assert.Equal(t, constants.BinaryMediaType, resp.Header.Get("Content-Type")) + } + +-// TestGetBlobMultipartPartHasDescriptorContentType verifies that each +-// part of a multipart/byteranges response carries the descriptor- +-// derived Content-Type alongside the per-part Content-Range. +-func TestGetBlobMultipartPartHasDescriptorContentType(t *testing.T) { ++// TestGetBlobMultipartPartBinaryContentType verifies that each part of a ++// multipart/byteranges response carries application/octet-stream ++// alongside the per-part Content-Range. ++func TestGetBlobMultipartPartBinaryContentType(t *testing.T) { + const blobBody = "0123456789" + + store := descriptorStore(t) ++ ++ var indexReads atomic.Int32 ++ ++ store.GetIndexContentFn = func(repo string) ([]byte, error) { ++ indexReads.Add(1) ++ ++ return []byte("{}"), nil ++ } ++ store.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { ++ indexReads.Add(1) ++ ++ return []byte("{}"), nil ++ } + store.CheckBlobFn = func(ctx context.Context, repo string, digest godigest.Digest) (bool, int64, error) { + return true, int64(len(blobBody)), nil + } +@@ -2752,7 +2757,7 @@ func TestGetBlobMultipartPartHasDescriptorContentType(t *testing.T) { + from, + to int64, + ) (io.ReadCloser, int64, int64, error) { +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, mediaType) ++ assert.Equal(t, constants.BinaryMediaType, mediaType) + + return io.NopCloser(strings.NewReader(blobBody[from : to+1])), to - from + 1, int64(len(blobBody)), nil + } +@@ -2801,7 +2806,7 @@ func TestGetBlobMultipartPartHasDescriptorContentType(t *testing.T) { + require.NoError(t, err, "read part %d", i) + + assert.Equal(t, want.contentRange, part.Header.Get("Content-Range"), "part %d content-range", i) +- assert.Equal(t, ispec.MediaTypeImageLayerGzip, part.Header.Get("Content-Type"), ++ assert.Equal(t, constants.BinaryMediaType, part.Header.Get("Content-Type"), + "part %d content-type", i) + + body, err := io.ReadAll(part) +@@ -2813,6 +2818,8 @@ func TestGetBlobMultipartPartHasDescriptorContentType(t *testing.T) { + require.ErrorIs(t, err, io.EOF) + + assert.Equal(t, layerDigest.String(), resp.Header.Get(constants.DistContentDigestKey)) ++ ++ assert.Zero(t, indexReads.Load(), "blob path must not walk the repo index") + } + + // Streaming-multipart tests for the lazy-fan-out path. +diff --git a/swagger/docs.go b/swagger/docs.go +index c419ee326a..08a64a7624 100644 +--- a/swagger/docs.go ++++ b/swagger/docs.go +@@ -633,7 +633,8 @@ const docTemplate = `{ + "application/json" + ], + "produces": [ +- "application/vnd.oci.image.layer.v1.tar+gzip" ++ "application/json", ++ "application/octet-stream" + ], + "summary": "Get image blob/layer", + "parameters": [ +@@ -698,7 +699,8 @@ const docTemplate = `{ + "application/json" + ], + "produces": [ +- "application/json" ++ "application/json", ++ "application/octet-stream" + ], + "summary": "Check image blob/layer", + "parameters": [ +diff --git a/swagger/swagger.json b/swagger/swagger.json +index 5d13646ce2..3132eac27b 100644 +--- a/swagger/swagger.json ++++ b/swagger/swagger.json +@@ -625,7 +625,8 @@ + "application/json" + ], + "produces": [ +- "application/vnd.oci.image.layer.v1.tar+gzip" ++ "application/json", ++ "application/octet-stream" + ], + "summary": "Get image blob/layer", + "parameters": [ +@@ -690,7 +691,8 @@ + "application/json" + ], + "produces": [ +- "application/json" ++ "application/json", ++ "application/octet-stream" + ], + "summary": "Check image blob/layer", + "parameters": [ +diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml +index 8f011420f4..5a03de6251 100644 +--- a/swagger/swagger.yaml ++++ b/swagger/swagger.yaml +@@ -468,7 +468,8 @@ paths: + required: true + type: string + produces: +- - application/vnd.oci.image.layer.v1.tar+gzip ++ - application/json ++ - application/octet-stream + responses: + "200": + description: OK +@@ -492,6 +493,7 @@ paths: + type: string + produces: + - application/json ++ - application/octet-stream + responses: + "200": + description: OK diff --git a/k8s/registry/zot-build/Dockerfile b/k8s/registry/zot-build/Dockerfile new file mode 100644 index 0000000..24b51c5 --- /dev/null +++ b/k8s/registry/zot-build/Dockerfile @@ -0,0 +1,48 @@ +# zot v2.1.20 + upstream fe0679da ("perf: drop the index walk on blob +# Content-Type", #4350), which is merged upstream but not yet released. +# +# Why this image exists at all: v2.1.17..v2.1.20 resolve a blob HEAD's +# Content-Type by reading the repo's manifests from S3 one by one until a +# descriptor matches. On cardano/mainnet (~300+ epoch tags, one manifest +# each) that is 20-150 s per HEAD, growing with every published epoch — +# kb/zot-instance-disconnects/. No released version both has +# redirectBlobURL (v2.1.17+) and lacks the walk, so this carries exactly +# one vendored upstream patch until v2.1.21 ships; delete the whole +# directory then and re-pin the official image. +# +# Build & push (founder, Apple Silicon builds arm64 natively): +# cd solution/stelae/codebase/k8s/registry/zot-build +# docker build --platform linux/arm64 -t ghcr.io/txpipe/zot-linux-arm64:v2.1.20-pr4350 . +# docker push ghcr.io/txpipe/zot-linux-arm64:v2.1.20-pr4350 +# docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/txpipe/zot-linux-arm64:v2.1.20-pr4350 +# then pin the printed digest in ../deployment.yaml. + +FROM --platform=linux/arm64 golang:1.26-trixie AS build +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /src +RUN git clone --depth 1 --branch v2.1.20 https://github.com/project-zot/zot.git . +COPY 4350-drop-index-walk.patch /tmp/4350.patch +# Only the handler change; the patch's test and swagger hunks are not needed +# in a binary build. +RUN git apply --include=pkg/api/routes.go /tmp/4350.patch \ + && grep -q "resolveBlobResponseMediaType" pkg/api/routes.go || true +# Mirrors the Makefile's `binary` target (no BUILD_LABELS), minus the +# build-metadata indirection; the ldflags only stamp version strings. +RUN env CGO_ENABLED=0 GOEXPERIMENT=jsonv2 GOOS=linux GOARCH=arm64 \ + go build -o bin/zot -trimpath \ + -ldflags "-X zotregistry.dev/zot/v2/pkg/buildinfo.ReleaseTag=v2.1.20-pr4350 \ + -X zotregistry.dev/zot/v2/pkg/buildinfo.Commit=v2.1.20+fe0679da \ + -X zotregistry.dev/zot/v2/pkg/buildinfo.BinaryType=dist \ + -X zotregistry.dev/zot/v2/pkg/buildinfo.GoVersion=go1.26 \ + -s -w" ./cmd/zot + +FROM --platform=linux/arm64 debian:bookworm-slim +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /src/bin/zot /usr/local/bin/zot +EXPOSE 5000 +ENTRYPOINT ["/usr/local/bin/zot"] +CMD ["serve", "/etc/zot/config.json"] From 4a6138c32f087322fad9aebf2e467aead94b0880 Mon Sep 17 00:00:00 2001 From: Santiago Date: Tue, 8 Sep 2026 17:57:29 -0300 Subject: [PATCH 2/2] fix: address deployment review findings --- k8s/dolos-publisher/.driver-backup.patch | 1184 ------------------ k8s/dolos-publisher/.gitignore | 2 - k8s/dolos-publisher/README.md | 32 +- k8s/dolos-publisher/chart/values.yaml | 4 +- k8s/registry/README.md | 47 +- k8s/registry/chart/templates/_helpers.tpl | 6 +- k8s/registry/chart/templates/deployment.yaml | 2 +- k8s/registry/chart/values.yaml | 28 +- k8s/registry/zot-build/Dockerfile | 6 +- 9 files changed, 52 insertions(+), 1259 deletions(-) delete mode 100644 k8s/dolos-publisher/.driver-backup.patch delete mode 100644 k8s/dolos-publisher/.gitignore diff --git a/k8s/dolos-publisher/.driver-backup.patch b/k8s/dolos-publisher/.driver-backup.patch deleted file mode 100644 index 194cc4e..0000000 --- a/k8s/dolos-publisher/.driver-backup.patch +++ /dev/null @@ -1,1184 +0,0 @@ -diff --git a/src/bin/dolos/bootstrap/mithril.rs b/src/bin/dolos/bootstrap/mithril.rs -index 0a2886c9..64c401ad 100644 ---- a/src/bin/dolos/bootstrap/mithril.rs -+++ b/src/bin/dolos/bootstrap/mithril.rs -@@ -15,35 +15,35 @@ use dolos::prelude::*; - #[derive(Debug, clap::Args, Clone)] - pub struct Args { - #[arg(long, default_value = "./snapshot")] -- download_dir: String, -+ pub(crate) download_dir: String, - - /// Skip the Mithril certificate validation - #[arg(long, action)] -- skip_validation: bool, -+ pub(crate) skip_validation: bool, - - /// Assume the snapshot is already available in the download dir - #[arg(long, action)] -- skip_download: bool, -+ pub(crate) skip_download: bool, - - /// Retain downloaded snapshot instead of deleting it - #[arg(long, action)] -- retain_snapshot: bool, -+ pub(crate) retain_snapshot: bool, - - /// Number of blocks to process in each chunk, more is faster but uses more - /// memory - #[arg(long, default_value = "500")] -- chunk_size: usize, -+ pub(crate) chunk_size: usize, - - #[arg(long)] -- start_from: Option, -+ pub(crate) start_from: Option, - - /// Start downloading from this immutable file number (inclusive) - #[arg(long)] -- download_start: Option, -+ pub(crate) download_start: Option, - - /// Download up to this immutable file number (inclusive) - #[arg(long)] -- download_end: Option, -+ pub(crate) download_end: Option, - } - - impl Default for Args { -@@ -170,7 +170,7 @@ impl mithril_client::feedback::FeedbackReceiver for MithrilFeedback { - } - - /// Scan the immutable directory for the highest immutable file number present. --fn highest_existing_immutable(immutable_dir: &Path) -> Option { -+pub(crate) fn highest_existing_immutable(immutable_dir: &Path) -> Option { - let entries = std::fs::read_dir(immutable_dir).ok()?; - let mut max: Option = None; - for entry in entries.flatten() { -@@ -239,17 +239,39 @@ fn plan_download(args: &Args, immutable_dir: &Path, last_immutable: u64) -> Down - } - } - --async fn fetch_snapshot( -+/// One aggregator client configuration, shared by the fetch and the beacon -+/// query so the two cannot drift apart on discovery or key handling. -+fn client_builder(config: &MithrilConfig) -> ClientBuilder { -+ ClientBuilder::new(AggregatorDiscoveryType::Url(config.aggregator.clone())) -+ .set_genesis_verification_key(mithril_client::GenesisVerificationKey::JsonHex( -+ config.genesis_key.clone(), -+ )) -+} -+ -+/// The highest immutable file number any aggregator snapshot covers. -+/// -+/// What `snapshot backfill` sizes its next download window against, and how it -+/// knows the aggregator has nothing past the files already on disk. -+pub(crate) async fn latest_immutable_file(config: &MithrilConfig) -> MithrilResult { -+ let client = client_builder(config).build()?; -+ -+ let snapshots = client.cardano_database_v2().list().await?; -+ -+ snapshots -+ .iter() -+ .map(|snapshot| snapshot.beacon.immutable_file_number) -+ .max() -+ .ok_or(MithrilError::msg("no snapshot available")) -+} -+ -+pub(crate) async fn fetch_snapshot( - args: &Args, - config: &MithrilConfig, - feedback: &Feedback, - ) -> MithrilResult<()> { - let feedback = MithrilFeedback::new(feedback); - -- let client = ClientBuilder::new(AggregatorDiscoveryType::Url(config.aggregator.clone())) -- .set_genesis_verification_key(mithril_client::GenesisVerificationKey::JsonHex( -- config.genesis_key.clone(), -- )) -+ let client = client_builder(config) - .add_feedback_receiver(Arc::new(feedback)) - .build()?; - -diff --git a/src/bin/dolos/bootstrap/mod.rs b/src/bin/dolos/bootstrap/mod.rs -index 78a60fab..4ef0bf28 100644 ---- a/src/bin/dolos/bootstrap/mod.rs -+++ b/src/bin/dolos/bootstrap/mod.rs -@@ -8,7 +8,7 @@ use tracing::info; - use crate::feedback::Feedback; - use dolos_core::{StateStore, WalStore}; - --mod mithril; -+pub(crate) mod mithril; - mod ranged; - mod relay; - mod snapshot; -diff --git a/src/bin/dolos/common.rs b/src/bin/dolos/common.rs -index 8e74c9bf..5969ac75 100644 ---- a/src/bin/dolos/common.rs -+++ b/src/bin/dolos/common.rs -@@ -148,6 +148,19 @@ pub fn stele_scratch_dir(config: &StorageConfig, chosen: Option<&std::path::Path - } - - pub fn setup_domain(config: &RootConfig) -> miette::Result { -+ setup_domain_with_stop_epoch(config, None) -+} -+ -+/// The same domain [`setup_domain`] assembles, with `chain.stop_epoch` forced. -+/// -+/// For callers that replay to a chosen epoch boundary over the live stores — -+/// `snapshot backfill` — the way `doctor rebuild-state` forces it on the -+/// domain it hand-builds. A `Some` here overrides whatever the configuration -+/// says; `None` leaves it alone. -+pub fn setup_domain_with_stop_epoch( -+ config: &RootConfig, -+ stop_epoch: Option, -+) -> miette::Result { - let stores = open_data_stores(config).map_err(|e| match e { - Error::WalError(WalError::IncompatibleVersion { found, expected }) => miette::miette!( - help = format!( -@@ -162,7 +175,11 @@ pub fn setup_domain(config: &RootConfig) -> miette::Result { - let (tip_broadcast, _) = tokio::sync::broadcast::channel(100); - let chain = config.chain.clone(); - -- let ChainConfig::Cardano(chain_config) = chain; -+ let ChainConfig::Cardano(mut chain_config) = chain; -+ -+ if stop_epoch.is_some() { -+ chain_config.stop_epoch = stop_epoch; -+ } - - let chain = dolos_cardano::CardanoLogic::initialize::( - chain_config, -@@ -333,7 +350,7 @@ pub fn open_genesis_files(config: &GenesisConfig) -> miette::Result { - - #[inline] - #[cfg(unix)] --async fn wait_for_exit_signal() { -+pub(crate) async fn wait_for_exit_signal() { - let mut sigterm = - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap(); - -@@ -349,7 +366,7 @@ async fn wait_for_exit_signal() { - - #[inline] - #[cfg(windows)] --async fn wait_for_exit_signal() { -+pub(crate) async fn wait_for_exit_signal() { - tokio::signal::ctrl_c().await.unwrap() - } - -diff --git a/src/bin/dolos/snapshot/backfill.rs b/src/bin/dolos/snapshot/backfill.rs -new file mode 100644 -index 00000000..e16a37f4 ---- /dev/null -+++ b/src/bin/dolos/snapshot/backfill.rs -@@ -0,0 +1,846 @@ -+//! `dolos snapshot backfill` — replay mithril history one epoch at a time, -+//! publishing a stele at each boundary. -+//! -+//! The publisher driver: one restart-safe loop that acquires immutable files -+//! from mithril in bounded windows, replays them to the next epoch boundary -+//! under `stop_epoch`, publishes the resulting sequence into an OCI repository -+//! in-process, prunes behind itself, and repeats until the aggregator has -+//! nothing further. A premature rerun — cron firing before a new epoch is -+//! available — finds the repository up to date and exits zero without writing. -+//! -+//! Downloads resume from the directory's own contents when any immutable -+//! files are present. When none are — a cold container start, whose disk -+//! keeps nothing and whose store was just restored from the registry — the -+//! start is derived from the cursor's chunk file instead, a margin early, -+//! so a restart costs one window rather than the chain so far. -+//! -+//! The reader never opens the highest downloaded file — pallas pops it as -+//! "not really immutable" — so at the aggregator tip the replay stands at -+//! most one chunk (~21600 slots, about six hours) behind the mithril beacon. -+//! That lag is steady state, not loss: the next run picks the chunk up once -+//! the beacon moves past it. -+//! -+//! Each iteration *opens* by publishing the sequence the cursor already stands -+//! at, then extends the replay by one epoch. Publishing on entry rather than -+//! right after the boundary import is what makes every crash window resumable: -+//! a rerun that finds the boundary reached but unpublished publishes it before -+//! moving on, instead of replaying past it and leaving a gap no later stele -+//! could close. -+//! -+//! This module is orchestration only: it composes the mithril fetch, the -+//! import lifecycle, and the publish path `snapshot publish --repo` uses, and -+//! changes none of them. -+ -+use std::path::{Path, PathBuf}; -+use std::sync::Arc; -+ -+use clap::Parser; -+use dolos_core::config::RootConfig; -+use dolos_core::{Domain as _, DomainError, ImportExt as _, StateStore as _, WalStore as _}; -+use dolos_snapshot::{export, registry::Repository}; -+use indicatif::ProgressBar; -+use itertools::Itertools as _; -+use miette::{bail, Context as _, IntoDiagnostic as _}; -+use tokio_util::sync::CancellationToken; -+use tracing::info; -+ -+use crate::feedback::Feedback; -+use dolos::adapters::DomainAdapter; -+ -+/// Blocks handed to `import_blocks` per batch. -+const IMPORT_CHUNK: usize = 100; -+ -+/// Files of margin kept around the cursor's own immutable file, on both of -+/// the convention's uses: cleanup spares this many files behind the consumed -+/// threshold, and a download start derived from the cursor backs up this -+/// many files. Early is the cheap direction — a re-downloaded file's blocks -+/// at or before the cursor are skipped on import — while late would leave -+/// the immutable reader without the cursor's own chunk. -+const IMMUTABLE_FILE_MARGIN: u64 = 2; -+ -+/// Slots per immutable chunk file — a node packaging convention, not a -+/// protocol invariant. Used to pick files safe to delete and, on a cold -+/// start whose download dir is empty, to derive where downloading resumes; -+/// never to plan how far a replay goes. Both uses carry -+/// [`IMMUTABLE_FILE_MARGIN`], and a derivation that still lands past the -+/// cursor's chunk is caught by the stalled-window check rather than trusted. -+const SLOTS_PER_IMMUTABLE_FILE: u64 = 21_600; -+ -+/// Where the mithril window lands when the operator names nowhere: beside the -+/// stores, so the bytes stay on the data mount. -+const DOWNLOAD_DIR: &str = "mithril"; -+ -+const INTERRUPTED: &str = -+ "interrupted by a shutdown signal; the stores are consistent and a rerun resumes here"; -+ -+#[derive(Debug, Parser)] -+pub struct Args { -+ /// OCI repository to publish into, e.g. -+ /// `oci://ghcr.io/txpipe/dolos-mainnet` -+ #[arg(long, value_name = "OCI_URL")] -+ repo: Repository, -+ -+ /// talk to the repository over plaintext HTTP rather than HTTPS; for a -+ /// registry on a loopback address or a mirror inside a cluster, and for -+ /// nothing reachable from outside one -+ #[arg(long, action)] -+ insecure: bool, -+ -+ /// directory to stage layers in while they are uploaded; defaults to -+ /// `/scratch` -+ #[arg(long, value_name = "DIR")] -+ scratch_dir: Option, -+ -+ /// directory the mithril immutable files are downloaded into; defaults to -+ /// `/mithril` -+ #[arg(long, value_name = "DIR")] -+ download_dir: Option, -+ -+ /// immutable files fetched per download round -+ #[arg(long, default_value = "40")] -+ window: u64, -+ -+ /// stop after publishing this sequence; for smoke tests -+ #[arg(long, value_name = "N")] -+ until_epoch: Option, -+ -+ /// skip the mithril digest and merkle validation; the certificate chain -+ /// is still verified. local smoke tests only -+ #[arg(long, action)] -+ skip_validation: bool, -+} -+ -+/// What an iteration's opening publish decided about the run. -+enum Step { -+ /// Replay toward `target`'s boundary. `prune` says whether history behind -+ /// the cursor may be dropped first — true only once the publish step has -+ /// run, because pruning at tip T is safe exactly when everything below T -+ /// is already in the repository. -+ Extend { target: u64, prune: bool }, -+ /// `--until-epoch` is published; the run is over. -+ Done, -+} -+ -+/// How an epoch's replay ended. -+enum Advance { -+ /// `stop_epoch` fired: the cursor stands on the target epoch's first -+ /// block. -+ Boundary { cursor_slot: u64 }, -+ /// The local files ran out and the aggregator has nothing newer. -+ MithrilExhausted, -+ /// A shutdown signal arrived; everything imported so far is committed. -+ Cancelled, -+} -+ -+/// One import pass over the files on disk. -+enum Import { -+ Boundary, -+ /// The files ran out before the boundary. Deliberately silent about how -+ /// many blocks the pass imported: on a sparse chain zero is an ordinary -+ /// answer, so nothing downstream may treat it as evidence. -+ Exhausted, -+ Cancelled, -+} -+ -+/// The epoch the next replay stops at: one past the cursor's, or 1 from a -+/// fresh store. -+fn target_epoch(cursor_epoch: Option) -> u64 { -+ cursor_epoch.map_or(1, |epoch| epoch + 1) -+} -+ -+/// The file a download round resumes from, or `None` for the beginning. -+/// -+/// The directory's own contents stay authoritative when any files are -+/// present — that is what re-fetches a possibly truncated highest file. An -+/// empty directory with a cursor is a cold container start: the store was -+/// restored from the registry onto a disk that keeps nothing, and resuming -+/// from file zero would re-download the whole chain, so the start is derived -+/// from the cursor's own chunk file instead, a margin early. -+fn resume_file(highest: Option, cursor_slot: Option) -> Option { -+ highest.or_else(|| { -+ cursor_slot -+ .map(|slot| (slot / SLOTS_PER_IMMUTABLE_FILE).saturating_sub(IMMUTABLE_FILE_MARGIN)) -+ }) -+} -+ -+/// The next download round, as `(download_start, download_end)`, or `None` -+/// when the aggregator has nothing past what is on disk. -+/// -+/// The resume file is deliberately re-fetched rather than skipped: an -+/// interrupted download may have left it truncated, and it has not been -+/// verified yet. -+fn next_window(resume: Option, window: u64, beacon: u64) -> Option<(Option, u64)> { -+ match resume { -+ Some(resume) if beacon <= resume => None, -+ Some(resume) => Some((Some(resume), beacon.min(resume + window))), -+ None => Some((None, beacon.min(window))), -+ } -+} -+ -+/// Whether a download round actually landed new files. -+/// -+/// The stall test, and it is asked of file numbers rather than block counts -+/// on purpose: a window of legitimately empty chunks advances the files while -+/// importing nothing, and that is an ordinary round on a sparse chain. A -+/// fetch that left the highest file where it was is the hard error. `Option` -+/// ordering puts an absent highest below every present one, so the first -+/// window into an empty directory counts as an advance. -+fn fetch_advanced(before: Option, after: Option) -> bool { -+ after > before -+} -+ -+/// Immutable files strictly below this number sit wholly behind the cursor, -+/// margin included, and are safe to delete. -+fn consumed_below(cursor_slot: u64) -> u64 { -+ (cursor_slot / SLOTS_PER_IMMUTABLE_FILE).saturating_sub(IMMUTABLE_FILE_MARGIN) -+} -+ -+/// What the immutable directory holds, for a diagnostic. -+fn dir_contents(immutable_dir: &Path) -> String { -+ let Ok(entries) = std::fs::read_dir(immutable_dir) else { -+ return "an unreadable immutable dir".to_owned(); -+ }; -+ -+ let mut numbers: Vec = entries -+ .flatten() -+ .filter_map(|entry| { -+ let name = entry.file_name(); -+ let name = name.to_string_lossy(); -+ name.split('.').next().and_then(|s| s.parse().ok()) -+ }) -+ .collect(); -+ -+ numbers.sort_unstable(); -+ numbers.dedup(); -+ -+ match (numbers.first(), numbers.last()) { -+ (Some(first), Some(last)) => { -+ format!("files {first:05}..={last:05} ({} of them)", numbers.len()) -+ } -+ _ => "no immutable files".to_owned(), -+ } -+} -+ -+/// Delete the numbered immutable files the replay has consumed. -+fn cleanup_consumed(immutable_dir: &Path, cursor_slot: u64) -> miette::Result<()> { -+ let threshold = consumed_below(cursor_slot); -+ -+ if threshold == 0 { -+ return Ok(()); -+ } -+ -+ let Ok(entries) = std::fs::read_dir(immutable_dir) else { -+ return Ok(()); -+ }; -+ -+ let mut removed = 0u64; -+ -+ for entry in entries.flatten() { -+ let name = entry.file_name(); -+ let name = name.to_string_lossy(); -+ -+ let Some(number) = name.split('.').next().and_then(|s| s.parse::().ok()) else { -+ continue; -+ }; -+ -+ if number < threshold { -+ std::fs::remove_file(entry.path()) -+ .into_diagnostic() -+ .with_context(|| format!("removing the consumed immutable file {name}"))?; -+ -+ removed += 1; -+ } -+ } -+ -+ if removed > 0 { -+ info!(removed, threshold, "removed consumed immutable files"); -+ } -+ -+ Ok(()) -+} -+ -+/// SIGTERM/SIGINT as a token the synchronous loop polls between chunks. -+/// -+/// The driver has no ambient tokio runtime for `hook_exit_token`, so the -+/// signal wait gets a dedicated thread with a current-thread runtime of its -+/// own. -+fn spawn_exit_watcher() -> miette::Result { -+ let cancel = CancellationToken::new(); -+ let hooked = cancel.clone(); -+ -+ std::thread::Builder::new() -+ .name("exit-signal".to_owned()) -+ .spawn(move || { -+ let runtime = tokio::runtime::Builder::new_current_thread() -+ .enable_all() -+ .build() -+ .expect("building the signal-wait runtime"); -+ -+ runtime.block_on(async { -+ crate::common::wait_for_exit_signal().await; -+ tracing::warn!("shutdown requested; stopping at the next chunk"); -+ hooked.cancel(); -+ }); -+ }) -+ .into_diagnostic() -+ .context("spawning the signal-wait thread")?; -+ -+ Ok(cancel) -+} -+ -+fn dir_argument(dir: &Path) -> miette::Result { -+ dir.to_str() -+ .map(str::to_owned) -+ .ok_or_else(|| miette::miette!("the download dir {} is not valid UTF-8", dir.display())) -+} -+ -+/// Everything an iteration needs and the CLI already settled. -+struct Driver<'a> { -+ config: &'a RootConfig, -+ args: &'a Args, -+ feedback: &'a Feedback, -+ /// For the async mithril calls only. The registry client owns a -+ /// current-thread runtime of its own and must never run inside this one, -+ /// so every publish stays on the plain thread. -+ runtime: tokio::runtime::Runtime, -+ cancel: CancellationToken, -+ download_dir: PathBuf, -+ immutable_dir: PathBuf, -+} -+ -+impl Driver<'_> { -+ /// Publish the sequence the cursor stands at, and name the next target. -+ /// -+ /// Also reseeds the WAL from the state cursor before anything else opens -+ /// the domain: `import_blocks` skips the WAL by design, so a run that -+ /// died mid-import left the state ahead of it, and the next domain open -+ /// would refuse with `InconsistentState`. -+ fn publish_pending(&self) -> miette::Result { -+ let stores = crate::common::open_data_stores(self.config) -+ .into_diagnostic() -+ .context("opening the data stores")?; -+ -+ let cursor = stores -+ .state -+ .read_cursor() -+ .into_diagnostic() -+ .context("reading the state cursor")?; -+ -+ let Some(cursor) = cursor else { -+ return Ok(Step::Extend { -+ target: target_epoch(None), -+ prune: false, -+ }); -+ }; -+ -+ if cursor.is_fully_defined() { -+ stores -+ .wal -+ .reset_to(&cursor) -+ .into_diagnostic() -+ .context("seeding the WAL from the state cursor")?; -+ } -+ -+ let summary = dolos_cardano::eras::load_chain_summary_from_state(&stores.state) -+ .map_err(|err| miette::miette!("loading the chain summary: {err:?}"))?; -+ -+ let (epoch, _) = summary.slot_epoch(cursor.slot()); -+ -+ // Nothing publishable yet: a sequence-0 stele would be epoch 0's -+ // mid-epoch sliver, which no consumer chains from. -+ if epoch == 0 { -+ return Ok(Step::Extend { -+ target: target_epoch(Some(epoch)), -+ prune: false, -+ }); -+ } -+ -+ let genesis = crate::common::open_genesis_files(&self.config.genesis)?; -+ -+ let plan = export::plan( -+ &stores.state, -+ u64::from(genesis.network_magic()), -+ super::retained_epochs(self.config)?, -+ ) -+ .into_diagnostic() -+ .context("planning the publish")?; -+ -+ super::report_plan(&plan)?; -+ -+ let publish = super::publish::RepositoryPublish { -+ repo: &self.args.repo, -+ insecure: self.args.insecure, -+ scratch_dir: self.args.scratch_dir.as_deref(), -+ rebuild: false, -+ dry_run: false, -+ require_new: false, -+ }; -+ -+ super::publish::to_repository(self.config, &publish, &plan, &stores, self.feedback)?; -+ -+ if self -+ .args -+ .until_epoch -+ .is_some_and(|until| plan.sequence >= until) -+ { -+ println!( -+ "sequence {} published; stopping at --until-epoch", -+ plan.sequence -+ ); -+ -+ return Ok(Step::Done); -+ } -+ -+ Ok(Step::Extend { -+ target: target_epoch(Some(epoch)), -+ prune: true, -+ }) -+ } -+ -+ /// Replay toward `target`'s boundary inside a domain that stops there. -+ fn extend(&self, target: u64, prune: bool) -> miette::Result { -+ let domain = crate::common::setup_domain_with_stop_epoch(self.config, Some(target))?; -+ -+ let result = self.advance_domain(&domain, prune); -+ -+ // Shut down even when the replay failed: fjall in particular has -+ // background work to flush before the handle drops. -+ let shutdown = domain.shutdown(); -+ -+ let advance = result?; -+ shutdown.map_err(|e| miette::miette!("shutting down the domain: {e}"))?; -+ -+ Ok(advance) -+ } -+ -+ /// Import what is on disk, fetching windows from mithril whenever the -+ /// files run out, until the boundary, the aggregator's tip, or a signal. -+ fn advance_domain(&self, domain: &DomainAdapter, prune: bool) -> miette::Result { -+ let mithril = self -+ .config -+ .mithril -+ .as_ref() -+ .ok_or_else(|| miette::miette!("missing mithril config"))?; -+ -+ // After the publish and before the next epoch goes in, never between -+ // a boundary and its publish: pruning at tip T drops history below -+ // `T - max_history` only, and every later publish reads blocks at or -+ // above the T it was standing at when its predecessor was published. -+ if prune { -+ let rounds = domain -+ .drain_housekeeping(None) -+ .map_err(|e| miette::miette!("{e}")) -+ .context("pruning excess history")?; -+ -+ info!(rounds, "housekeeping drained"); -+ } -+ -+ let progress = self.feedback.slot_progress_bar(); -+ progress.set_message("replaying immutable blocks"); -+ -+ let outcome = loop { -+ if self.cancel.is_cancelled() { -+ break Advance::Cancelled; -+ } -+ -+ match self.import_available(domain, &progress)? { -+ Import::Boundary => { -+ let cursor_slot = domain -+ .state -+ .read_cursor() -+ .into_diagnostic() -+ .context("reading the state cursor at the boundary")? -+ .map(|cursor| cursor.slot()) -+ .unwrap_or_default(); -+ -+ break Advance::Boundary { cursor_slot }; -+ } -+ Import::Cancelled => break Advance::Cancelled, -+ // Zero blocks imported is not a stall: on a sparse chain a -+ // whole window of chunks can legitimately be empty, and the -+ // replay simply needs those slot ranges walked. Keep -+ // fetching; the stall check below speaks in file numbers. -+ Import::Exhausted => {} -+ } -+ -+ let Some(beacon) = self -+ .runtime -+ .block_on(async { -+ tokio::select! { -+ beacon = crate::bootstrap::mithril::latest_immutable_file(mithril) => { -+ beacon.map(Some) -+ } -+ _ = self.cancel.cancelled() => Ok(None), -+ } -+ }) -+ .map_err(|err| miette::miette!(err.to_string())) -+ .context("listing mithril snapshots")? -+ else { -+ break Advance::Cancelled; -+ }; -+ -+ let highest = -+ crate::bootstrap::mithril::highest_existing_immutable(&self.immutable_dir); -+ -+ let cursor_slot = domain -+ .state -+ .read_cursor() -+ .into_diagnostic() -+ .context("reading the state cursor")? -+ .map(|cursor| cursor.slot()); -+ -+ let resume = resume_file(highest, cursor_slot); -+ -+ let Some((start, end)) = next_window(resume, self.args.window, beacon) else { -+ break Advance::MithrilExhausted; -+ }; -+ -+ info!( -+ ?start, -+ end, beacon, "fetching an immutable window from mithril" -+ ); -+ -+ let fetch = crate::bootstrap::mithril::Args { -+ download_dir: dir_argument(&self.download_dir)?, -+ skip_validation: self.args.skip_validation, -+ download_start: start, -+ download_end: Some(end), -+ ..Default::default() -+ }; -+ -+ let fetched = self -+ .runtime -+ .block_on(async { -+ tokio::select! { -+ fetched = crate::bootstrap::mithril::fetch_snapshot( -+ &fetch, -+ mithril, -+ self.feedback, -+ ) => fetched.map(Some), -+ _ = self.cancel.cancelled() => Ok(None), -+ } -+ }) -+ .map_err(|err| miette::miette!(err.to_string())) -+ .context("fetching a mithril immutable window")?; -+ -+ if fetched.is_none() { -+ break Advance::Cancelled; -+ } -+ -+ // The one stall that is a hard error, and it is measured in file -+ // numbers, never blocks: a fetch that left the highest file -+ // where it was returned nothing new — a misconfigured range or -+ // a download failure — and every later round would only repeat -+ // it. -+ let after = crate::bootstrap::mithril::highest_existing_immutable(&self.immutable_dir); -+ -+ if !fetch_advanced(highest, after) { -+ let cursor_slot = cursor_slot.unwrap_or_default(); -+ -+ bail!( -+ "the fetched immutable window {:05}..={end:05} did not advance the \ -+ downloaded files (highest was {highest:?}, still {after:?}); mithril \ -+ returned nothing new for the state cursor at slot {cursor_slot} — the \ -+ immutable dir holds {}", -+ start.unwrap_or(0), -+ dir_contents(&self.immutable_dir), -+ ); -+ } -+ }; -+ -+ // Whatever ended the replay, the chunks it committed are in the state -+ // and the WAL must agree before the next domain open. -+ self.seed_wal(domain)?; -+ -+ progress.abandon_with_message("replay round complete"); -+ -+ Ok(outcome) -+ } -+ -+ /// Import everything on disk past the cursor, in chunks. -+ fn import_available( -+ &self, -+ domain: &DomainAdapter, -+ progress: &ProgressBar, -+ ) -> miette::Result { -+ use pallas::network::miniprotocols::Point; -+ -+ // Before the first download the immutable dir does not exist at all; -+ // that is the caller's cue to fetch, not an error. -+ if !self.immutable_dir.is_dir() { -+ return Ok(Import::Exhausted); -+ } -+ -+ // Nothing to walk yet, same cue. Deliberately *not* `get_tip`: on a -+ // sparse chain the second-highest chunk can be empty, and `get_tip` -+ // reads only that one chunk and answers `None` for the whole db — -+ // with full unimported chunks sitting right there. The walk from the -+ // cursor is the only reader that tells the truth here. -+ if crate::bootstrap::mithril::highest_existing_immutable(&self.immutable_dir).is_none() { -+ return Ok(Import::Exhausted); -+ } -+ -+ let cursor = domain -+ .state -+ .read_cursor() -+ .into_diagnostic() -+ .context("reading the state cursor")?; -+ -+ let point: Point = cursor -+ .map(|c| c.try_into().unwrap()) -+ .unwrap_or(Point::Origin); -+ -+ let mut iter = pallas::interop::hardano::storage::immutable::read_blocks_from_point( -+ &self.immutable_dir, -+ point.clone(), -+ ) -+ .map_err(|err| miette::miette!(err.to_string())) -+ .context("iterating the local immutable db")?; -+ -+ // unless we're starting from the origin of the chain, the iterator -+ // stands on the last block already imported; skip it rather than -+ // import it twice -+ if point != Point::Origin { -+ iter.next(); -+ } -+ -+ for batch in iter.chunks(IMPORT_CHUNK).into_iter() { -+ let batch: Vec<_> = batch -+ .try_collect() -+ .into_diagnostic() -+ .context("reading block data")?; -+ -+ let batch: Vec<_> = batch.into_iter().map(Arc::new).collect(); -+ -+ match domain.import_blocks(batch) { -+ Ok(last) => progress.set_position(last), -+ Err(DomainError::StopEpochReached) => return Ok(Import::Boundary), -+ Err(e) => { -+ return Err(miette::miette!("{e}")) -+ .context("importing an immutable block chunk") -+ } -+ } -+ -+ if self.cancel.is_cancelled() { -+ return Ok(Import::Cancelled); -+ } -+ } -+ -+ // A yield of nothing is not a verdict: the walk exhausts silently at -+ // the retained edge, empty chunks contribute no blocks, and a chunk -+ // read error truncates the iterator the same way. Whether anything -+ // more is coming is the fetch loop's question, answered in file -+ // numbers, never in block counts. -+ Ok(Import::Exhausted) -+ } -+ -+ /// Reseed the WAL from the state cursor, so the next domain open finds -+ /// the two agreeing. -+ fn seed_wal(&self, domain: &DomainAdapter) -> miette::Result<()> { -+ let cursor = domain -+ .state -+ .read_cursor() -+ .into_diagnostic() -+ .context("reading the state cursor")?; -+ -+ let Some(cursor) = cursor else { -+ return Ok(()); -+ }; -+ -+ if !cursor.is_fully_defined() { -+ bail!( -+ "state cursor at slot {} has no block hash, cannot seed the WAL", -+ cursor.slot(), -+ ); -+ } -+ -+ domain -+ .wal -+ .reset_to(&cursor) -+ .into_diagnostic() -+ .context("seeding the WAL from the state cursor") -+ } -+} -+ -+pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Result<()> { -+ crate::common::setup_tracing(&config.logging, &config.telemetry)?; -+ -+ if args.window == 0 { -+ bail!("--window must be at least 1"); -+ } -+ -+ if config.mithril.is_none() { -+ bail!("missing mithril config"); -+ } -+ -+ let download_dir = args -+ .download_dir -+ .clone() -+ .unwrap_or_else(|| config.storage.path.join(DOWNLOAD_DIR)); -+ -+ std::fs::create_dir_all(&download_dir) -+ .into_diagnostic() -+ .with_context(|| format!("creating the download dir {}", download_dir.display()))?; -+ -+ let driver = Driver { -+ config, -+ args, -+ feedback, -+ runtime: tokio::runtime::Runtime::new() -+ .into_diagnostic() -+ .context("creating the tokio runtime for mithril downloads")?, -+ cancel: spawn_exit_watcher()?, -+ immutable_dir: download_dir.join("immutable"), -+ download_dir, -+ }; -+ -+ loop { -+ if driver.cancel.is_cancelled() { -+ bail!(INTERRUPTED); -+ } -+ -+ let (target, prune) = match driver.publish_pending()? { -+ Step::Done => return Ok(()), -+ Step::Extend { target, prune } => (target, prune), -+ }; -+ -+ info!(target, "replaying toward the next epoch boundary"); -+ -+ match driver.extend(target, prune)? { -+ Advance::Boundary { cursor_slot } => { -+ cleanup_consumed(&driver.immutable_dir, cursor_slot)?; -+ } -+ Advance::MithrilExhausted => { -+ println!("the repository is up to date with mithril; nothing left to backfill"); -+ return Ok(()); -+ } -+ Advance::Cancelled => bail!(INTERRUPTED), -+ } -+ } -+} -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ -+ #[test] -+ fn the_first_target_is_epoch_one_and_every_later_one_follows_the_cursor() { -+ assert_eq!(target_epoch(None), 1); -+ assert_eq!(target_epoch(Some(0)), 1); -+ assert_eq!(target_epoch(Some(499)), 500); -+ } -+ -+ #[test] -+ fn an_empty_download_dir_resumes_from_the_cursors_own_file() { -+ // empty dir with a cursor: the cursor's chunk file, a margin early -+ assert_eq!( -+ resume_file(None, Some(SLOTS_PER_IMMUTABLE_FILE * 6000)), -+ Some(5998), -+ ); -+ -+ // mid-file slots land in the same file before the margin applies -+ assert_eq!( -+ resume_file(None, Some(SLOTS_PER_IMMUTABLE_FILE * 6000 + 5)), -+ Some(5998), -+ ); -+ -+ // the margin floors at the first file -+ assert_eq!(resume_file(None, Some(SLOTS_PER_IMMUTABLE_FILE)), Some(0)); -+ assert_eq!(resume_file(None, Some(0)), Some(0)); -+ -+ // empty dir, fresh store: the beginning -+ assert_eq!(resume_file(None, None), None); -+ assert_eq!( -+ next_window(resume_file(None, None), 40, 1000), -+ Some((None, 40)) -+ ); -+ -+ // files on disk stay authoritative, wherever the cursor is -+ assert_eq!( -+ resume_file(Some(120), Some(SLOTS_PER_IMMUTABLE_FILE * 6000)), -+ Some(120), -+ ); -+ } -+ -+ #[test] -+ fn only_a_fetch_that_leaves_the_files_where_they_were_is_a_stall() { -+ // the first window into an empty dir is an advance -+ assert!(fetch_advanced(None, Some(0))); -+ -+ // new files landed — even when every one of them is an empty chunk -+ // and the import that follows adds zero blocks, the loop goes on -+ assert!(fetch_advanced(Some(5), Some(6))); -+ -+ // nothing new on disk after a fetch: the hard error -+ assert!(!fetch_advanced(Some(5), Some(5))); -+ assert!(!fetch_advanced(Some(5), None)); -+ assert!(!fetch_advanced(None, None)); -+ } -+ -+ #[test] -+ fn windows_advance_from_the_highest_existing_file() { -+ // a fresh dir starts at the beginning, one window deep -+ assert_eq!(next_window(None, 40, 1000), Some((None, 40))); -+ -+ // a short chain clamps to the beacon -+ assert_eq!(next_window(None, 40, 7), Some((None, 7))); -+ -+ // resuming re-fetches the highest file: it may be truncated -+ assert_eq!(next_window(Some(100), 40, 1000), Some((Some(100), 140))); -+ -+ // the last window clamps to the beacon -+ assert_eq!(next_window(Some(990), 40, 1000), Some((Some(990), 1000))); -+ -+ // nothing newer than what is on disk -+ assert_eq!(next_window(Some(1000), 40, 1000), None); -+ assert_eq!(next_window(Some(1001), 40, 1000), None); -+ } -+ -+ #[test] -+ fn cleanup_keeps_a_margin_behind_the_cursor() { -+ assert_eq!(consumed_below(0), 0); -+ assert_eq!(consumed_below(SLOTS_PER_IMMUTABLE_FILE * 2), 0); -+ assert_eq!(consumed_below(SLOTS_PER_IMMUTABLE_FILE * 3), 1); -+ assert_eq!(consumed_below(SLOTS_PER_IMMUTABLE_FILE * 10 + 5), 8); -+ } -+ -+ #[test] -+ fn cleanup_removes_only_consumed_numbered_files() { -+ let dir = tempfile::tempdir().unwrap(); -+ -+ for n in 0..6u64 { -+ for ext in ["chunk", "primary", "secondary"] { -+ std::fs::write(dir.path().join(format!("{n:05}.{ext}")), []).unwrap(); -+ } -+ } -+ -+ std::fs::write(dir.path().join("lock"), []).unwrap(); -+ -+ // threshold 3: files 0..=2 consumed, 3..=5 and non-numeric names stay -+ cleanup_consumed(dir.path(), SLOTS_PER_IMMUTABLE_FILE * 5).unwrap(); -+ -+ let mut remaining: Vec = std::fs::read_dir(dir.path()) -+ .unwrap() -+ .flatten() -+ .map(|entry| entry.file_name().to_string_lossy().into_owned()) -+ .collect(); -+ -+ remaining.sort(); -+ -+ assert_eq!( -+ remaining, -+ [ -+ "00003.chunk", -+ "00003.primary", -+ "00003.secondary", -+ "00004.chunk", -+ "00004.primary", -+ "00004.secondary", -+ "00005.chunk", -+ "00005.primary", -+ "00005.secondary", -+ "lock", -+ ], -+ ); -+ } -+} -diff --git a/src/bin/dolos/snapshot/mod.rs b/src/bin/dolos/snapshot/mod.rs -index c86d87fe..f0fda476 100644 ---- a/src/bin/dolos/snapshot/mod.rs -+++ b/src/bin/dolos/snapshot/mod.rs -@@ -27,6 +27,8 @@ use miette::{Context as _, IntoDiagnostic as _}; - - use crate::feedback::Feedback; - -+#[cfg(feature = "mithril")] -+mod backfill; - mod digest; - mod inspect; - mod publish; -@@ -37,6 +39,11 @@ pub enum Command { - /// writes a stele to a local directory or an OCI repository - Publish(publish::Args), - -+ /// replays mithril history one epoch at a time, publishing a stele at -+ /// each boundary into an OCI repository -+ #[cfg(feature = "mithril")] -+ Backfill(backfill::Args), -+ - /// computes a stele's inscription and identity without writing one - Digest(digest::Args), - -@@ -61,6 +68,8 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res - // that waits on a store walk and a network, and the other three are - // over in the time it takes to print what they found. - Command::Publish(x) => publish::run(config, x, feedback), -+ #[cfg(feature = "mithril")] -+ Command::Backfill(x) => backfill::run(config, x, feedback), - Command::Digest(x) => digest::run(config, x), - Command::Verify(x) => verify::run(config, x), - Command::Inspect(x) => inspect::run(config, x), -diff --git a/src/bin/dolos/snapshot/publish.rs b/src/bin/dolos/snapshot/publish.rs -index 32f8c1df..4f653395 100644 ---- a/src/bin/dolos/snapshot/publish.rs -+++ b/src/bin/dolos/snapshot/publish.rs -@@ -89,13 +89,50 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res - super::report_plan(&plan)?; - - match (&args.repo, &args.output_dir) { -- (Some(repo), _) => to_repository(config, args, repo, &plan, &stores, feedback), -+ (Some(repo), _) => { -+ let publish = RepositoryPublish { -+ repo, -+ insecure: args.insecure, -+ scratch_dir: args.scratch_dir.as_deref(), -+ rebuild: args.rebuild, -+ dry_run: args.dry_run, -+ require_new: args.require_new, -+ }; -+ -+ to_repository(config, &publish, &plan, &stores, feedback) -+ } - (None, Some(dir)) => to_directory(args, dir, &plan, &stores, feedback), - // The required `destination` group already refuses this. - (None, None) => unreachable!("one of --output-dir and --repo is required"), - } - } - -+/// The repository arm's settings, freed of the CLI that spelled them. -+/// -+/// Factored so `snapshot backfill` publishes through exactly the code path -+/// `snapshot publish --repo` does — same standing check, same preflight, same -+/// chained predecessor, same report — rather than a second telling of it that -+/// would drift. -+pub(super) struct RepositoryPublish<'a> { -+ /// The repository to publish into. -+ pub repo: &'a Repository, -+ -+ /// Talk plaintext HTTP rather than HTTPS. -+ pub insecure: bool, -+ -+ /// Where to stage layers; `None` takes `/scratch`. -+ pub scratch_dir: Option<&'a std::path::Path>, -+ -+ /// Build every layer instead of carrying forward published ones. -+ pub rebuild: bool, -+ -+ /// Report what would be written and stop. -+ pub dry_run: bool, -+ -+ /// Fail when the repository is already at this node's sequence. -+ pub require_new: bool, -+} -+ - fn to_directory( - args: &Args, - dir: &std::path::Path, -@@ -145,14 +182,15 @@ fn to_directory( - /// The report is what a publisher wants to check rather than trust: how much of - /// this stele was inherited rather than built, and how much of it moved. Both - /// are numbers the code counted, not an inference from a duration. --fn to_repository( -+pub(super) fn to_repository( - config: &RootConfig, -- args: &Args, -- repo: &Repository, -+ publish: &RepositoryPublish, - plan: &export::Plan, - stores: &crate::common::Stores, - feedback: &Feedback, - ) -> miette::Result<()> { -+ let repo = publish.repo; -+ - // A publisher's credentials come from `STELAE_REGISTRY_USER` / - // `STELAE_REGISTRY_PASSWORD`, which override anything configured. The - // configured user is still the fallback: it is read-only, so authenticating -@@ -160,9 +198,9 @@ fn to_repository( - // is the honest place for "these credentials cannot publish" to be said. - let auth = crate::common::stele_registry_auth(&config.stelae)?; - -- let scratch = crate::common::stele_scratch_dir(&config.storage, args.scratch_dir.as_deref()); -+ let scratch = crate::common::stele_scratch_dir(&config.storage, publish.scratch_dir); - -- let registry = registry::open(repo, args.insecure, auth, scratch) -+ let registry = registry::open(repo, publish.insecure, auth, scratch) - .into_diagnostic() - .context("opening the repository")?; - -@@ -172,11 +210,11 @@ fn to_repository( - // along with everything else. - let publishing = registry::Publishing::new(®istry) - .recording_in(&config.storage.path) -- .rebuilding(args.rebuild); -+ .rebuilding(publish.rebuild); - - // Before anything is built, and before the dry run too: a publisher asking - // what a publish would do wants the same answer the publish gives. -- if !standing(®istry, plan, args)? { -+ if !standing(®istry, plan, publish.require_new)? { - return Ok(()); - } - -@@ -190,7 +228,7 @@ fn to_repository( - .into_diagnostic() - .context("sizing the staging directory")?; - -- if args.dry_run { -+ if publish.dry_run { - // `None` here and `None` at the `publish` below are one decision: a dry - // run describes the publish that follows it, so the two calls are - // handed the same digest records or the number is about something else. -@@ -273,7 +311,7 @@ fn to_repository( - /// to invent. What is new is that the message names the distance alongside - /// both sequences, so "the publisher has been down for a day" and "the - /// publisher has been down for a month" do not read the same. --fn standing(registry: &Registry, plan: &export::Plan, args: &Args) -> miette::Result { -+fn standing(registry: &Registry, plan: &export::Plan, require_new: bool) -> miette::Result { - let standing = registry::standing(registry, plan) - .into_diagnostic() - .context("reading the repository's latest stele")?; -@@ -294,7 +332,7 @@ fn standing(registry: &Registry, plan: &export::Plan, args: &Args) -> miette::Re - plan.sequence, - ); - -- if args.require_new { -+ if require_new { - return Err(miette::miette!("{message}")); - } - diff --git a/k8s/dolos-publisher/.gitignore b/k8s/dolos-publisher/.gitignore deleted file mode 100644 index 97fe62e..0000000 --- a/k8s/dolos-publisher/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Nothing local lives here: the registry pair is the third Secret in -# ../../../ops/secrets.local.yaml, ignored there. diff --git a/k8s/dolos-publisher/README.md b/k8s/dolos-publisher/README.md index 24025b9..4049cdd 100644 --- a/k8s/dolos-publisher/README.md +++ b/k8s/dolos-publisher/README.md @@ -1,16 +1,9 @@ ---- -provenance: authored -owner: org/founder -tags: [] ---- # Dolos publisher — deployment unit The publisher is one Kubernetes Job per network, rendered from the Helm -chart in [`chart/`](chart/) with a values file per network in -[`solution/stelae/ops/`](../../../ops/README.md), and deployed by the same -two commands as the registry. The -[`stelae-registry-ops`](../../../skills/registry-ops/SKILL.md) skill -holds those commands with the re-run, first-run and monitoring procedures. +chart in [`chart/`](chart/) with a values file per network maintained by the +operator. The operator's runbook owns deployment, re-run, first-run, and +monitoring procedures. Each Job runs the official `ghcr.io/txpipe/dolos` image on the Demeter m2 EKS cluster: it restores its network's latest stele (or starts from genesis), replays to the next epoch boundary, publishes, prunes, repeats, @@ -81,19 +74,10 @@ HTTP), `publisherSecretName`, `backoffLimit`, Local check, no cluster needed: ```bash -helm lint solution/stelae/codebase/k8s/dolos-publisher/chart \ - --values solution/stelae/ops/values.publisher-preprod.yaml -helm template stelae-publisher-preprod solution/stelae/codebase/k8s/dolos-publisher/chart \ - --namespace stelae-publisher --values solution/stelae/ops/values.publisher-preprod.yaml +helm lint k8s/dolos-publisher/chart --values "$PUBLISHER_VALUES" +helm template stelae-publisher-preprod k8s/dolos-publisher/chart \ + --namespace stelae-publisher --values "$PUBLISHER_VALUES" ``` -Its predecessors are in git history: the raw manifests this chart replaced -(one Job and ConfigMap per network under `k8s/`, applied by hand, retired -under [publisher-chart](../../../../../archive/plans/dolos-stelae-publication-ops-publisher-chart.md)), -and before them a Cloudflare Worker with a container-backed Durable Object. -The registry moved to the cluster -(`decisions/0037-stelae-registry-on-eks.md`), the Jobs followed, the -Worker's cron was stood down (`decisions/0038-preprod-stele-publication.md`), -and the infrastructure was deleted on 2026-09-06 under -[registry-decommission](../../../../../plans/stelae-registry-decommission.md), -tier 5. +Its predecessors are in git history: raw manifests applied by hand, and before +them a Cloudflare Worker with a container-backed Durable Object. diff --git a/k8s/dolos-publisher/chart/values.yaml b/k8s/dolos-publisher/chart/values.yaml index bb92b88..4a642ef 100644 --- a/k8s/dolos-publisher/chart/values.yaml +++ b/k8s/dolos-publisher/chart/values.yaml @@ -6,8 +6,8 @@ # These defaults are instance-agnostic — no network, registry, image pin or # credential is assumed. Everything a network must supply is empty here and # `required` in the templates, so a missing value fails at render rather -# than starting a half-configured publisher. This domain's instances are -# The operator supplies one values file per network. +# than starting a half-configured publisher. The operator supplies one values +# file per network. # # Two values are deliberately without a default. `concurrency` was measured # per network against the registry it writes to. `dolosToml` carries the diff --git a/k8s/registry/README.md b/k8s/registry/README.md index ce68a2a..56d4c87 100644 --- a/k8s/registry/README.md +++ b/k8s/registry/README.md @@ -1,27 +1,17 @@ ---- -provenance: authored -owner: org/founder -tags: [] ---- # Stelae registry The registry software that serves stele repositories: **zot over an S3 object store**, packaged so that no cluster, bucket, hostname or credential -is assumed. This directory is the instance-agnostic half. This domain's own -instance — m2 EKS over R2, `oci.stelae.store` — is -[`solution/stelae/ops/`](../../../ops/) -and is operated through the -[`stelae-registry-ops`](../../../skills/registry-ops/SKILL.md) skill. +is assumed. This directory is the instance-agnostic deployment unit; +deployment values, Secret manifests, and operational procedures belong in the +operator's infrastructure repository. Why zot: real registry software over an object store pays ~44 ms per request where the stock `cloudflare/serverless-registry` Worker paid ~3 s per `PATCH` -and capped a whole mainnet publish at 2.40 MB/s -([`kb/registry-alternatives/`](../../../kb/registry-alternatives/), decision -[0034](../../../../../decisions/0034-stelae-registry-front-end.md)). Why a cluster: +and capped a whole mainnet publish at 2.40 MB/s. Why a cluster: the first zot deployment, in a Cloudflare Container behind a Worker, killed its own write path under sustained upload and could not be probed when it -broke ([`kb/zot-instance-disconnects/`](../../../kb/zot-instance-disconnects/), -decision [0037](../../../../../decisions/0037-stelae-registry-on-eks.md)). +broke. ``` publisher ── in-cluster, plain HTTP ──▶ zot ──▶ object store (S3 API) @@ -31,7 +21,7 @@ client ──── public name, TLS ─────────▶ zot ── | path | holds | |---|---| | `chart/` | the Helm chart: one zot replica, `Recreate`, a PVC for the metaDB, a ClusterIP Service for the write path, and two mutually exclusive public-read shapes (ingress or LoadBalancer), both off by default | -| `zot-build/` | an interim zot image, v2.1.20 plus upstream's unreleased blob-`HEAD` walk fix (#4350); retired at v2.1.21 by [stelae-zot-upstream-repin](../../../../../plans/stelae-zot-upstream-repin.md) | +| `zot-build/` | an interim zot image, v2.1.20 plus upstream's unreleased blob-`HEAD` walk fix (#4350); retire it when an official release includes that fix | ## The chart @@ -71,9 +61,19 @@ driver through the AWS credential chain. The usernames live in both the values and the htpasswd lines and only convention keeps them matched; divergence authenticates a user and grants nothing. -Render and install are the instance's business — for this domain, the two -commands in the ops skill. `templates/NOTES.txt` prints the smoke test: a -`401` on `/v2/`, a sub-second blob `HEAD`, and a `307` on a blob `GET`. +Kubernetes does not restart a pod when a referenced Secret changes. After +rotating the store credentials, apply the Secret and explicitly restart the +registry Deployment before revoking the old credentials: + +```bash +kubectl --context "$CTX" apply --server-side --force-conflicts -f "$SECRETS_FILE" +kubectl --context "$CTX" --namespace stelae-publisher \ + rollout restart deployment/stelae-registry +``` + +Render and install are the operator's responsibility. `templates/NOTES.txt` +prints the smoke test: a `401` on `/v2/`, a sub-second blob `HEAD`, and a +`307` on a blob `GET`. ## The interim image @@ -92,15 +92,12 @@ on the reference digest, patched takes 0.8 s. Two Cloudflare front ends preceded this: the stock `serverless-registry` Worker over the `stelae-registry` bucket, and the first zot deployment in a Cloudflare Container behind a Worker router. Both were decommissioned under -[stelae-registry-decommission](../../../../../plans/stelae-registry-decommission.md). -The attempts that localized the Container fault are recorded in -[`kb/zot-instance-disconnects/`](../../../kb/zot-instance-disconnects/); the deploy -unit itself lives in git history at the 2026-08-29 commit. +the operator's infrastructure plan; the deployment unit lives in git history. The steles themselves were moved once, on 2026-08-28, from the retired Worker's R2 layout into zot's by a control-plane copy tool that lived here as `migrate/`. It was removed on 2026-09-06: its source layout no longer exists and everything it carried over has since been re-published from scratch, so nothing remained for it to convert or verify. The account of -that job — cost, method, byte-for-byte verification — is -[`kb/registry-alternatives/migration.md`](../../../kb/registry-alternatives/migration.md). +that job — cost, method, and byte-for-byte verification — remains in the +operator's infrastructure records. diff --git a/k8s/registry/chart/templates/_helpers.tpl b/k8s/registry/chart/templates/_helpers.tpl index 4a6466a..2a02616 100644 --- a/k8s/registry/chart/templates/_helpers.tpl +++ b/k8s/registry/chart/templates/_helpers.tpl @@ -38,9 +38,11 @@ app: {{ include "stelae-registry.fullname" . }} {{- define "stelae-registry.image" -}} {{- $img := .Values.image -}} +{{- $repository := required "image.repository is required" $img.repository -}} +{{- $tag := required "image.tag is required" $img.tag -}} {{- if $img.digest -}} -{{- printf "%s:%s@%s" $img.repository $img.tag $img.digest -}} +{{- printf "%s:%s@%s" $repository $tag $img.digest -}} {{- else -}} -{{- printf "%s:%s" $img.repository $img.tag -}} +{{- printf "%s:%s" $repository $tag -}} {{- end -}} {{- end -}} diff --git a/k8s/registry/chart/templates/deployment.yaml b/k8s/registry/chart/templates/deployment.yaml index 10d1efd..8764abf 100644 --- a/k8s/registry/chart/templates/deployment.yaml +++ b/k8s/registry/chart/templates/deployment.yaml @@ -39,7 +39,7 @@ spec: - name: zot image: {{ include "stelae-registry.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - command: [{{ .Values.image.binary | quote }}, "serve", "/etc/zot/config.json"] + command: [{{ required "image.binary is required" .Values.image.binary | quote }}, "serve", "/etc/zot/config.json"] ports: - containerPort: {{ .Values.http.port | int }} name: http diff --git a/k8s/registry/chart/values.yaml b/k8s/registry/chart/values.yaml index 6828fb1..b85969d 100644 --- a/k8s/registry/chart/values.yaml +++ b/k8s/registry/chart/values.yaml @@ -4,33 +4,29 @@ # These defaults are instance-agnostic — no cluster, bucket, hostname or # credential is assumed. Everything an instance must supply is empty here and # `required` in the templates, so a missing value fails at render rather than -# deploying a half-configured registry. This domain's instance supplies them -# from `solution/stelae/ops/` (values file + secrets manifest + runbook). +# deploying a half-configured registry. The operator supplies them from its +# infrastructure repository (values file + secrets manifest + runbook). # -# Values that look arbitrary carry their reason. Several are load-bearing — -# `redirectBlobURL`, the JSON-string sizes, `rootDirectory: "/"`, the HTTP -# timeouts — and the evidence for them is in -# `solution/stelae/kb/zot-instance-disconnects/`. +# Values that look arbitrary carry their reason. Several are load-bearing: +# `redirectBlobURL`, the JSON-string sizes, `rootDirectory: "/"`, and the HTTP +# timeouts. nameOverride: "" fullnameOverride: "" image: - # The upstream release, which is what a fresh instance should run. - # # Caveat with teeth: zot v2.1.17–v2.1.20 answer a blob HEAD's Content-Type # by walking the repo's manifests out of the object store — O(tags) per # request, which at ~300 tags measured 143 s per HEAD and made publishing - # impossible. Upstream fixed it in commit fe0679da (#4350), unreleased as - # of 2026-08-29. An instance publishing at any scale must either run - # v2.1.21+ once it ships, or override this with a patched build (ours is - # ../zot-build/, wired up in solution/stelae/ops/values.registry-m2.yaml). - repository: ghcr.io/project-zot/zot-linux-arm64 - tag: v2.1.20 - digest: sha256:56230c5a589eb55acc57afc34307f6ea1b2efe5cf8e0057ccca64099ba837ff6 + # impossible. Upstream fixed it in commit fe0679da (#4350). The operator + # must choose an official release containing that fix or a patched build + # such as ../zot-build/, and match the image architecture to placement. + repository: "" + tag: "" + digest: "" pullPolicy: IfNotPresent # Upstream images name the binary by platform; a custom build may not. - binary: /usr/local/bin/zot-linux-arm64 + binary: "" # One replica, Recreate: chunked upload sessions and the metaDB live on an # RWO volume, so two pods must never overlap. diff --git a/k8s/registry/zot-build/Dockerfile b/k8s/registry/zot-build/Dockerfile index 24b51c5..f8b5228 100644 --- a/k8s/registry/zot-build/Dockerfile +++ b/k8s/registry/zot-build/Dockerfile @@ -11,11 +11,11 @@ # directory then and re-pin the official image. # # Build & push (founder, Apple Silicon builds arm64 natively): -# cd solution/stelae/codebase/k8s/registry/zot-build +# cd k8s/registry/zot-build # docker build --platform linux/arm64 -t ghcr.io/txpipe/zot-linux-arm64:v2.1.20-pr4350 . # docker push ghcr.io/txpipe/zot-linux-arm64:v2.1.20-pr4350 # docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/txpipe/zot-linux-arm64:v2.1.20-pr4350 -# then pin the printed digest in ../deployment.yaml. +# then pin the printed digest in the operator's registry values (`image.digest`). FROM --platform=linux/arm64 golang:1.26-trixie AS build RUN apt-get update \ @@ -27,7 +27,7 @@ COPY 4350-drop-index-walk.patch /tmp/4350.patch # Only the handler change; the patch's test and swagger hunks are not needed # in a binary build. RUN git apply --include=pkg/api/routes.go /tmp/4350.patch \ - && grep -q "resolveBlobResponseMediaType" pkg/api/routes.go || true + && grep -q "resolveBlobResponseMediaType" pkg/api/routes.go # Mirrors the Makefile's `binary` target (no BUILD_LABELS), minus the # build-metadata indirection; the ldflags only stamp version strings. RUN env CGO_ENABLED=0 GOEXPERIMENT=jsonv2 GOOS=linux GOARCH=arm64 \