Skip to content

store: drop unreachable Paged SH and the BDZ1 codec #682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
reardencode merged 1 commit into from
Sep 22, 2026
Merged

store: drop unreachable Paged SH and the BDZ1 codec #682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/rbitcoin-query/src/catchup.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,8 @@ impl Query {

/// Cold bulk-load durable scripthash tables (tip entry).
///
/// Direct IBD defers SH. Tip: one Class A pass into unsorted per-shard
/// files, then in-place unique-sort + seal.
/// Direct IBD defers SH. Tip: two Class A scans (identity-map keys, then
/// fuse-hit postings) into `scripthash.unsorted`, then pack and seal.
///
/// **`RBITCOIN_SH_FORCE_REBUILD=1`:** wipe SH head/runs/SEAL/HWM, then
/// full unsorted Class A collect + pack (not a catch-up tail).
Expand Down
134 changes: 0 additions & 134 deletions crates/rbitcoin-store/src/bdz.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,14 +144,6 @@ impl BdzMphf {
self.modulus
}

#[cfg(test)]
pub fn g_bytes(&self) -> usize {
match &self.g {
GStore::Ram(g) => g.len() * 4,
GStore::Fd { n_bytes, .. } => *n_bytes as usize,
}
}

pub fn g_bytes_resident(&self) -> usize {
match &self.g {
GStore::Ram(g) => g.len() * 4,
Expand Down Expand Up @@ -385,26 +377,6 @@ impl BdzMphf {
Err(StoreError::Corrupt("bdz mphf: graph did not peel"))
}

#[cfg(test)]
pub fn write_to(&self, path: &Path) -> Result<(), StoreError> {
const MAGIC: &[u8; 4] = b"BDZ1";
const HEADER_LEN: u64 = 24;
let GStore::Ram(g) = &self.g else {
return Err(StoreError::Corrupt("bdz mphf: write requires RAM g"));
};
let mut buf = Vec::with_capacity(HEADER_LEN as usize + g.len() * 4);
buf.extend_from_slice(MAGIC);
buf.extend_from_slice(&VERSION.to_le_bytes());
buf.extend_from_slice(&self.n.to_le_bytes());
buf.extend_from_slice(&self.m.to_le_bytes());
buf.extend_from_slice(&self.seed.to_le_bytes());
for &x in g.iter() {
buf.extend_from_slice(&x.to_le_bytes());
}
std::fs::write(path, &buf).map_err(|e| StoreError::io(path, e))?;
Ok(())
}

pub fn write_packed_to(&self, path: &Path) -> Result<(), StoreError> {
let GStore::Ram(g) = &self.g else {
return Err(StoreError::Corrupt("bdz mphf: write requires RAM g"));
Expand All @@ -425,56 +397,6 @@ impl BdzMphf {
})
}

#[cfg(test)]
pub fn read_from(path: &Path) -> Result<Self, StoreError> {
const MAGIC: &[u8; 4] = b"BDZ1";
const HEADER_LEN: u64 = 24;
let file = File::open(path).map_err(|e| StoreError::io(path, e))?;
let mut hdr = [0u8; HEADER_LEN as usize];
pread_exact(&file, path, 0, &mut hdr)?;
if &hdr[0..4] != MAGIC {
return Err(StoreError::Corrupt("bdz mphf: bad magic"));
}
let ver = u32::from_le_bytes(hdr[4..8].try_into().unwrap());
if ver != VERSION {
return Err(StoreError::Corrupt("bdz mphf: bad version"));
}
let n = u32::from_le_bytes(hdr[8..12].try_into().unwrap());
let m = u32::from_le_bytes(hdr[12..16].try_into().unwrap());
let seed = u64::from_le_bytes(hdr[16..24].try_into().unwrap());
if n == 0 {
return Ok(Self {
n: 0,
m: 0,
seed: 0,
modulus: 0,
g: GStore::Ram(Box::new([])),
compact: None,
});
}
let n_words = if n == 1 { 1 } else { m };
let g_bytes = n_words as u64 * 4;
let meta = file.metadata().map_err(|e| StoreError::io(path, e))?;
if meta.len() < HEADER_LEN + g_bytes {
return Err(StoreError::Corrupt("bdz mphf: g length"));
}
Ok(Self {
n,
m: n_words,
seed,
modulus: n,
g: GStore::Fd {
file,
path: path.to_path_buf(),
off: HEADER_LEN,
n_bytes: g_bytes,
g_bits: G_BITS_WORDS,
page_preads: AtomicU64::new(0),
},
compact: None,
})
}

pub fn read_packed_from(path: &Path) -> Result<Self, StoreError> {
let file = File::open(path).map_err(|e| StoreError::io(path, e))?;
let mut hdr = [0u8; HEADER_LEN2 as usize];
Expand Down Expand Up @@ -1399,62 +1321,6 @@ mod tests {
assert_eq!(one.index(99).unwrap(), 0);
}

#[test]
fn bdz_roundtrip_file() {
let dir = std::env::temp_dir().join(format!(
"rbitcoin-bdz-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let keys: Vec<u64> = (0..200u64).map(|i| i * 17 + 3).collect();
let f = BdzMphf::build(&keys).unwrap();
let p = dir.join("t.mphf");
f.write_to(&p).unwrap();
assert_eq!(&std::fs::read(&p).unwrap()[0..4], b"BDZ1");
let g = BdzMphf::read_from(&p).unwrap();
for &k in &keys {
assert_eq!(f.index(k).unwrap(), g.index(k).unwrap());
}
assert_eq!(g.g_bytes_resident(), 0, "open must not retain the g array");
assert_eq!(g.g_bytes(), f.g_bytes());
let miss = g.index(0xDEAD_BEEF_u64).unwrap();
assert!(miss < keys.len() as u32);
let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn bdz_open_matches_ram_index_without_g_heap() {
let dir = std::env::temp_dir().join(format!(
"rbitcoin-bdz-fd-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let keys: Vec<u64> = (0..10_000u64)
.map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(7))
.collect();
let ram = BdzMphf::build(&keys).unwrap();
assert!(ram.g_bytes_resident() > 0);
let p = dir.join("t.mphf");
ram.write_to(&p).unwrap();
let fd = BdzMphf::read_from(&p).unwrap();
assert_eq!(fd.g_bytes_resident(), 0);
for &k in &keys {
assert_eq!(ram.index(k).unwrap(), fd.index(k).unwrap());
}
let miss_k = 0xDEAD_BEEF_u64;
assert_eq!(ram.index(miss_k).unwrap(), fd.index(miss_k).unwrap());
assert!(fd.index(miss_k).unwrap() < keys.len() as u32);
let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn assigned_peel_index_is_rel_minus_one() {
let keys = [10u64, 20, 30, 40];
Expand Down
67 changes: 4 additions & 63 deletions crates/rbitcoin-store/src/scripthash.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1234,20 +1234,17 @@ impl ScriptHashTable {
scripthash: &[u8; 32],
) -> Result<Option<(ShHeadValue, KeyHome)>, StoreError> {
if let Some(v) = self.ingest.lock().unwrap().get(scripthash)? {
let v = self.fill_paged_first(scripthash, v, KeyHome::Ingest)?;
return Ok(Some((v, KeyHome::Ingest)));
}
let hk = head_key_from_full(scripthash);
for h in self.sealed_ovf.lock().unwrap().iter().rev() {
if let Some(v) = h.get(&hk)? {
let v = self.fill_paged_first(scripthash, v, KeyHome::SealedOvf)?;
return Ok(Some((v, KeyHome::SealedOvf)));
}
}
if let Some(l1) = self.ovf_l1.lock().unwrap().as_ref() {
if l1.fuse.contains(mix_key16(&hk)) {
if let Some(v) = l1.head.get(&hk)? {
let v = self.fill_paged_first(scripthash, v, KeyHome::SealedOvf)?;
return Ok(Some((v, KeyHome::SealedOvf)));
}
}
Expand All @@ -1258,7 +1255,6 @@ impl ScriptHashTable {
let g = slot.read().unwrap();
if let Some(h) = g.as_ref() {
if let Some(v) = h.get(&hk)? {
let v = self.fill_paged_first(scripthash, v, KeyHome::Main)?;
return Ok(Some((v, KeyHome::Main)));
}
}
Expand All @@ -1267,25 +1263,6 @@ impl ScriptHashTable {
Ok(None)
}

fn fill_paged_first(
&self,
key: &[u8; 32],
val: ShHeadValue,
home: KeyHome,
) -> Result<ShHeadValue, StoreError> {
match val {
ShHeadValue::Paged {
first_page: 0,
last_page,
} if last_page != 0 => {
let first =
paged_first_from_last(self.body_for(key, home), last_page, &self.page_ios)?;
Ok(ShHeadValue::paged(first, last_page))
}
other => Ok(other),
}
}

fn has_sorted_main(&self) -> bool {
self.sorted_main_on
.load(std::sync::atomic::Ordering::Acquire)
Expand Down Expand Up @@ -1438,13 +1415,6 @@ impl ScriptHashTable {
.collect())
}

fn collect_page_chain(&self, body: &TableFile, first_page: u64) -> Result<Vec<Fk>, StoreError> {
if first_page == 0 {
return Ok(Vec::new());
}
collect_page_chain_linked(body, first_page, &self.page_ios)
}

#[cfg(test)]
pub(crate) fn take_page_ios(&self) -> u64 {
self.page_ios.swap(0, Ordering::Relaxed)
Expand All @@ -1469,7 +1439,7 @@ impl ScriptHashTable {
let ents = self.read_slab(body, *class, *off)?;
Ok(ents.last().copied())
}
ShHeadValue::Paged { last_page, .. } | ShHeadValue::Extent { last_page } => {
ShHeadValue::Extent { last_page } => {
let mut page = [0u8; SH_PAGE_SIZE];
body.read_at(*last_page, &mut page)?;
sh_page_last_fk(&page)
Expand Down Expand Up @@ -1917,17 +1887,6 @@ impl ScriptHashTable {
}
Ok(got)
}
ShHeadValue::Paged {
first_page,
last_page,
} => {
let first = if *first_page != 0 {
*first_page
} else {
paged_first_from_last(body, *last_page, &self.page_ios)?
};
self.collect_page_chain(body, first)
}
ShHeadValue::Extent { last_page } => {
collect_extent_then_tail(body, *last_page, &self.page_ios)
}
Expand Down Expand Up @@ -2017,14 +1976,6 @@ impl ScriptHashTable {
}
Ok(new_val)
}
ShHeadValue::Paged {
first_page,
last_page,
} => {
let last =
self.append_fks_to_pages(body, alloc, *first_page, *last_page, new_ents)?;
Ok(ShHeadValue::paged(*first_page, last))
}
ShHeadValue::Extent { last_page } => {
let first = paged_first_from_last(body, *last_page, &self.page_ios)?;
let last = self.append_fks_to_pages(body, alloc, first, *last_page, new_ents)?;
Expand Down Expand Up @@ -2280,16 +2231,6 @@ impl ScriptHashTable {
old: &ShHeadValue,
) -> Result<(), StoreError> {
match old {
ShHeadValue::Paged { first_page, .. } => {
let mut off = *first_page;
while off != 0 {
let mut page = [0u8; SH_PAGE_SIZE];
body.read_at(off, &mut page)?;
let next = sh_page_next(&page)?;
self.free_slab(body, alloc, SH_PAGE_SLAB_CLASS, off)?;
off = next;
}
}
ShHeadValue::Extent { last_page } => {
let mut off = paged_first_from_last(body, *last_page, &self.page_ios)?;
while off != 0 {
Expand Down Expand Up @@ -3523,7 +3464,7 @@ fn write_alloc_header(body: &TableFile, state: &AllocState) -> Result<(), StoreE
/// Read SHAL alloc page. Returns `(state, on_disk_version)`.
///
/// **v1** (schema-13 slabs) and **v2** (schema-14 page chains) share the same
/// header field layout. Callers upgrade empty v1 → v2 or refuse durable v1.
/// header field layout. An empty older header is reset; a durable pre-v3 body refuses.
fn read_alloc_header(body: &TableFile) -> Result<(AllocState, u16), StoreError> {
let mut buf = vec![0u8; SH_ALLOC_HEADER_LEN];
let avail = body
Expand All @@ -3532,13 +3473,13 @@ fn read_alloc_header(body: &TableFile) -> Result<(AllocState, u16), StoreError>
.min(SH_ALLOC_HEADER_LEN as u64) as usize;
if avail < 24 {
return Err(StoreError::Corrupt(
"scripthash body missing alloc header (expected hybrid SHAL; migrate v3 stores)",
"scripthash body missing alloc header (expected SHAL)",
));
}
body.read_at(FILE_HEADER_LEN as u64, &mut buf[..avail])?;
if buf[0..4] != SH_ALLOC_MAGIC {
return Err(StoreError::Corrupt(
"scripthash body not hybrid (no SHAL magic; run migrate)",
"scripthash body not hybrid (no SHAL magic)",
));
}
let ver = u16::from_le_bytes([buf[4], buf[5]]);
Expand Down
Loading
Loading