From 5594855be5a24c85728899e3c20b060ac62a9976 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 21:09:40 +0000 Subject: [PATCH] feat(bench): Add a forced decision instrument that inverts one shortcut at a time The ledger and the residuals label a decision at its own node. Whether being wrong there cost the root anything is a different question, and most such errors cost nothing: a re-search recovers them, or the root's move survives them. `arche forced` searches each root of a suite under the default, samples the shortcut decisions it takes, and searches the root again for each sampled decision with that one decision inverted, printing what the root answered both times. Four kinds can be inverted: the reverse futility margin answering a node, the null move's cut, a late quiet skipped (by the model at depth four and up, or a shallow rule below), and a reduced scout's fail low trusted. A decision is addressed by its kind, a position key and the deciding node's depth; a move decision is keyed by the position the move leaves, as the ledger keys its rows. The inversion applies wherever the search meets the address. `kinds` and `from ` narrow the sampling, since shallow decisions are most of them. The hooks sit where each decision is taken, behind a bare is_some with the body cold and out of line, as the sampler's are. While armed, the move loop asks the shallow rules move by move as it does under the ledger, and reduced scouts are staged so their features reach the row. The model's score is exposed for the row where the gate reads one. The default search is unchanged: the bench is 5965973, and callgrind over `bench 5` reads 195,641,600 instructions against master's 195,202,622 (+0.22%), with the same nodes. The tests hold a recording arm's move, score and nodes to an unarmed search's, an address never met to no visit and no change, every kept decision to at least one visit, one row an address, the kinds and depth filters, and the model's score to the depths the gate reads it at. A session test runs the binary. Bench: 5965973 Elo: not measured Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NgTGRjqAYAXR7VCxCRqhjh --- README.md | 2 +- arche-core/src/engine.rs | 219 +++++++++++- arche-core/src/forced.rs | 642 ++++++++++++++++++++++++++++++++++++ arche-core/src/late_move.rs | 19 ++ arche-core/src/lib.rs | 1 + arche-core/src/recorder.rs | 6 +- docs/ARCHITECTURE.md | 4 + docs/INSTRUMENTS.md | 82 ++++- src/instruments.rs | 155 ++++++++- tests/forced_session.rs | 52 +++ tests/uci_session.rs | 4 + 11 files changed, 1169 insertions(+), 17 deletions(-) create mode 100644 arche-core/src/forced.rs create mode 100644 tests/forced_session.rs diff --git a/README.md b/README.md index af79b4f..784dca3 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ The program starts in UCI mode immediately. `arche --help` lists the arguments t anything else. `bench` searches a fixed set of positions and prints what each search counted, for measuring a change to the search or the speed of a machine, and is a UCI command as well; [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) describes it. `residuals`, `cutoffs`, `reductions`, -`effort` and `terms` measure the search and the evaluation, and +`effort`, `forced` and `terms` measure the search and the evaluation, and [docs/INSTRUMENTS.md](docs/INSTRUMENTS.md) describes them. Binaries for linux, macos and windows are attached to each diff --git a/arche-core/src/engine.rs b/arche-core/src/engine.rs index ee34584..73e5a53 100644 --- a/arche-core/src/engine.rs +++ b/arche-core/src/engine.rs @@ -5,6 +5,7 @@ use crate::board::{Board, MOVE_LIST_INLINE, MoveList, Unplayable}; use crate::census; use crate::effort; use crate::eval; +use crate::forced; use crate::ghi::GhiCounters; use crate::late_move; use crate::limits::Limits; @@ -1193,6 +1194,10 @@ pub struct AlphaBeta { /// population its reservoir samples. Bumped only when the reservoir is /// armed. effort_depths: effort::Depths, + /// The forced decision instrument's arm, or none, on the sampler's + /// terms: read behind a bare check where each decision it can invert + /// is taken, and nowhere else. + forced: Option>, } /// What a search can be armed to record. Implemented here rather than @@ -1266,6 +1271,7 @@ impl AlphaBeta { ledger: None, effort: None, effort_depths: effort::Depths::default(), + forced: None, } } @@ -1284,6 +1290,167 @@ impl AlphaBeta { T::slot(self).take() } + /// Arm the forced decision instrument, to sample the decisions taken + /// or to invert one. + pub(crate) fn arm_forced(&mut self, arm: forced::Arm) { + self.forced = Some(Box::new(arm)); + } + + /// The arm back, with what it sampled or counted. + pub(crate) fn disarm_forced(&mut self) -> Option { + self.forced.take().map(|arm| *arm) + } + + /// A node decision just taken, the margin's or the pass's, offered to + /// the forced decision instrument. True when it is the decision being + /// inverted, which the caller then does not take. + // cold and out of line behind a bare is_some, for `sample`'s reason + #[cold] + #[inline(never)] + fn forced_node( + &mut self, + kind: forced::Kind, + depth: u8, + eval: Score, + alpha: Score, + beta: Score, + ) -> bool { + let address = forced::Address { + kind, + key: self.board.key, + depth, + }; + let board = &self.board; + let Some(arm) = self.forced.as_mut() else { + return false; + }; + if arm.inverts(address) { + return true; + } + arm.offer(address, |root, at| forced::Event { + address, + root, + at, + features: None, + eval_beta: i32::from(eval) - i32::from(beta), + alpha_gap: i32::from(alpha) - i32::from(eval), + attention: None, + fen: board.to_fen(), + }); + false + } + + /// What answered a node whose margin was inverted, kept from the first + /// such visit. + #[cold] + #[inline(never)] + fn forced_answered(&mut self, answered: forced::Answered) { + if let Some(arm) = self.forced.as_mut() { + arm.answered.get_or_insert(answered); + } + } + + /// A late quiet a rule just passed over, offered to the forced decision + /// instrument. True when it is the decision being inverted. The move is + /// made and unmade to read the key of the position it leaves, which is + /// the address; one that turns out illegal denied nothing. + #[cold] + #[inline(never)] + fn forced_skip( + &mut self, + node: &Node, + rules: &mut late_move::Rules, + moves: &[Play], + m: &Play, + ) -> bool { + let features = late_move::features(&self.deciding(), node, rules, moves, m); + if !self.board.make_move(m) { + return false; + } + let key = self.board.key; + self.board.undo_move(); + let address = forced::Address { + kind: forced::Kind::Skip, + key, + depth: node.depth, + }; + let board = &self.board; + let Some(arm) = self.forced.as_mut() else { + return false; + }; + if arm.inverts(address) { + return true; + } + arm.offer(address, |root, at| { + let eval = i64::from(crate::eval::eval(board)); + let (eval_beta, alpha_gap) = + (eval - i64::from(node.beta), i64::from(node.alpha) - eval); + forced::Event { + address, + root, + at, + features: Some(features), + eval_beta: eval_beta as i32, + alpha_gap: alpha_gap as i32, + attention: late_move::attention(node.depth, &features, eval_beta, alpha_gap), + fen: board.to_fen(), + } + }); + false + } + + /// A reduced scout that came back at or below alpha, offered to the + /// forced decision instrument before it answers for its move. True when + /// it is the decision being inverted. The board is the position the + /// move left, which is the address, and the node's own evaluation is + /// read by stepping the move back, for kept events alone. + #[cold] + #[inline(never)] + fn forced_scout( + &mut self, + staged: Option<&reduction::Staged>, + depth: u8, + alpha: Score, + beta: Score, + ) -> bool { + let address = forced::Address { + kind: forced::Kind::TrustedScout, + key: self.board.key, + depth, + }; + let board = &mut self.board; + let Some(arm) = self.forced.as_mut() else { + return false; + }; + if arm.inverts(address) { + return true; + } + let Some(staged) = staged else { + return false; + }; + arm.offer(address, |root, at| { + board.undo_move(); + let eval = i64::from(crate::eval::eval(board)); + let fen = board.to_fen(); + assert!( + board.make_move(&staged.play), + "the scouted move was made once already" + ); + let (eval_beta, alpha_gap) = (eval - i64::from(beta), i64::from(alpha) - eval); + forced::Event { + address, + root, + at, + features: Some(staged.features), + eval_beta: eval_beta as i32, + alpha_gap: alpha_gap as i32, + attention: late_move::attention(depth, &staged.features, eval_beta, alpha_gap), + fen, + } + }); + false + } + /// What the node knew about a move at the gate, gathered for a /// ledger row: the staged half of a scouted event, and the whole of /// a skipped one. @@ -1954,6 +2121,9 @@ impl AlphaBeta { None => self.eval(), }; *eval_memo = Some(eval); + // set only by the forced decision instrument, which then wants to + // know what answered the node instead + let mut margin_inverted = false; // the margin proves `eval - margin` as a lower bound, and fail soft // returns that. Clean: a static eval consulted no path @@ -1965,10 +2135,16 @@ impl AlphaBeta { self.sample(Shortcut::ShadowFutility, depth, floor, alpha, beta, eval); } if floor >= beta { - if self.sampler.is_some() { - self.sample(Shortcut::ReverseFutility, depth, floor, alpha, beta, eval); + if self.forced.is_some() + && self.forced_node(forced::Kind::ReverseFutility, depth, eval, alpha, beta) + { + margin_inverted = true; + } else { + if self.sampler.is_some() { + self.sample(Shortcut::ReverseFutility, depth, floor, alpha, beta, eval); + } + return Ok(Some(Value::clean(floor))); } - return Ok(Some(Value::clean(floor))); } } @@ -1986,7 +2162,10 @@ impl AlphaBeta { // undo before an abort can propagate self.board.undo_null_move(); let value = -result?; - if value.score >= beta { + if value.score >= beta + && !(self.forced.is_some() + && self.forced_node(forced::Kind::NullMove, depth, eval, alpha, beta)) + { // a mate found through a pass is not a mate, since the pass // is not a legal move, so the score is held under the window // a caller reads mates in @@ -1994,10 +2173,16 @@ impl AlphaBeta { if self.sampler.is_some() { self.sample(Shortcut::NullMove, depth, score, alpha, beta, eval); } + if margin_inverted { + self.forced_answered(forced::Answered::NullMove); + } return Ok(Some(Value::with_taint(score, value.tainted))); } taint.absorb(value); } + if margin_inverted { + self.forced_answered(forced::Answered::Moves); + } Ok(None) } @@ -2089,11 +2274,14 @@ impl AlphaBeta { entered_at, ); } - if scout.score <= alpha { + if scout.score <= alpha + && !(self.forced.is_some() && self.forced_scout(staged, depth, alpha, beta)) + { return Ok(scout); } // the scout's fail high asked for the searches below, so they - // carry its taint + // carry its taint, as does a trusted fail low the forced + // decision instrument inverted tainted = scout.tainted; } let probe = -self.alpha_beta( @@ -2205,7 +2393,10 @@ impl AlphaBeta { let front = quiets.front; if i == front { if let Some(ply) = quiets.ply { - if self.ledger.is_some() { + // the forced decision instrument asks the shallow rules + // move by move as the ledger does, since it has to see + // every skip it may invert + if self.ledger.is_some() || self.forced.is_some() { self.ordering.order_quiets( &self.board, &mut moves[front..], @@ -2286,6 +2477,12 @@ impl AlphaBeta { return Decision::First; } if rules.skips(&self.deciding(), node, m) { + if self.forced.is_some() && self.forced_skip(node, rules, moves, m) { + return Decision::Search { + reduction: 0, + staged: None, + }; + } self.record_skip(node, rules, moves, m); return Decision::Skip; } @@ -2297,6 +2494,12 @@ impl AlphaBeta { } match late_move::decide_admitted(&self.deciding(), node, rules, moves, m) { late_move::Verdict::Skip => { + if self.forced.is_some() && self.forced_skip(node, rules, moves, m) { + return Decision::Search { + reduction: 0, + staged: None, + }; + } self.record_skip(node, rules, moves, m); Decision::Skip } @@ -2413,7 +2616,7 @@ impl AlphaBeta { m: &Play, reduction: u8, ) -> Option { - if reduction > 0 && self.ledger.is_some() { + if reduction > 0 && (self.ledger.is_some() || self.forced.is_some()) { Some(self.staged_reduction(m, node, rules, moves)) } else { None diff --git a/arche-core/src/forced.rs b/arche-core/src/forced.rs new file mode 100644 index 0000000..4294d61 --- /dev/null +++ b/arche-core/src/forced.rs @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2022-2026 Andrew Wright + +//! The forced decision instrument: what one shortcut decision cost the +//! root. +//! +//! The ledger and the residuals label a decision at its own node, by +//! replaying it under the reference. Whether being wrong there cost the +//! root anything is a different question, and most such errors cost +//! nothing: a re-search recovers them, or the root's move survives them. +//! This searches each root once under the default, sampling the decisions +//! it takes, and then once more for each sampled decision with that one +//! decision inverted, and reports what the root answered both times. +//! +//! A decision is addressed by its kind, a position key and a depth (see +//! `Address`), and the inversion applies wherever the search meets that +//! address: every visit, in every iteration of the deepening. The two +//! searches agree until the first such visit, since each starts from a +//! fresh engine and table with no clock. + +use crate::bench::{self, Position}; +use crate::board::Board; +use crate::engine::{AlphaBeta, Engine, SearchOutcome, SearchParameters}; +use crate::late_move::Features; +use crate::misc::Score; +use crate::play::Play; +use crate::recorder::{self, Sampler}; +use std::fmt; + +/// About one decision in every this many, unless the command says +/// otherwise. Every kept decision costs a search of its root, so the rate +/// is far sparser than the recorders' whose rows cost nothing. +pub const DEFAULT_EVERY: u32 = 100_000; + +/// What can be inverted. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Kind { + /// The margin answered the node from its static evaluation. Inverted, + /// the node does not answer from the margin, and the null move then + /// gets its turn, as it would with the margin off. + ReverseFutility, + /// The pass cleared beta and answered the node. Inverted, the node goes + /// on to its moves. + NullMove, + /// A late quiet was passed over, by the model at depth four and up or + /// by either shallow rule below. Inverted, it is searched unreduced. + Skip, + /// A reduced scout came back at or below alpha and answered for its + /// move. Inverted, the move goes on to the probe and the proof as if + /// the scout had failed high. + TrustedScout, +} + +impl Kind { + pub const ALL: [Kind; 4] = [ + Kind::ReverseFutility, + Kind::NullMove, + Kind::Skip, + Kind::TrustedScout, + ]; + + pub fn word(self) -> &'static str { + match self { + Kind::ReverseFutility => "reverse_futility", + Kind::NullMove => "null_move", + Kind::Skip => "skip", + Kind::TrustedScout => "trusted_scout", + } + } + + pub fn of_word(word: &str) -> Option { + Kind::ALL.into_iter().find(|kind| kind.word() == word) + } + + /// Spread into the sampling key, so the kinds at one position and + /// depth are sampled apart. + fn spread(self) -> u64 { + (self as u64 + 1).wrapping_mul(0xd6e8_feb8_6659_fd93) + } + + fn bit(self) -> u8 { + 1 << self as u8 + } +} + +/// Which kinds a run samples, as a set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Kinds(u8); + +impl Kinds { + pub const ALL: Kinds = Kinds(0b1111); + + pub fn of(kinds: &[Kind]) -> Kinds { + Kinds(kinds.iter().fold(0, |set, kind| set | kind.bit())) + } + + pub fn holds(self, kind: Kind) -> bool { + self.0 & kind.bit() != 0 + } + + /// The kinds named, comma separated, in the declared order. + pub fn words(self) -> String { + Kind::ALL + .into_iter() + .filter(|kind| self.holds(*kind)) + .map(Kind::word) + .collect::>() + .join(",") + } +} + +/// One decision, wherever the search meets it. +/// +/// A node decision is keyed by the node's position. A move decision is +/// keyed by the position the move leaves, as the ledger keys its rows, so +/// for a given parent the move is part of the address without being +/// stored. Two parents that reach one child at one depth share an address, +/// and a forced run then inverts both; the row's columns describe the +/// first. The depth is the deciding node's, the check extension included. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Address { + pub kind: Kind, + pub key: u64, + pub depth: u8, +} + +impl Address { + pub(crate) fn sample_key(self) -> u64 { + recorder::sample_key( + self.key ^ self.kind.spread(), + recorder::FORCED_LANE, + self.depth, + ) + } +} + +/// What answered a node once its decision was inverted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Answered { + /// The null move, where the margin was inverted. + NullMove, + /// The node's moves. + Moves, +} + +impl Answered { + fn word(self) -> &'static str { + match self { + Answered::NullMove => "null_move", + Answered::Moves => "moves", + } + } +} + +/// One decision taken under the default, kept by the sampler. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Event { + pub address: Address, + /// The root's place in the suite. + pub root: usize, + /// The decision's place among those offered over the root's search, + /// so the first visit to an address can be told from its revisits. Not + /// the node count, which every iteration and re-search starts again. + pub at: u64, + /// The move's features at the decision, for a move decision. + pub(crate) features: Option, + pub eval_beta: i32, + pub alpha_gap: i32, + /// The model's score, where the gate reads one. + pub attention: Option, + /// The deciding node's position. + pub fen: String, +} + +/// What an engine carries while the instrument is armed: the reservoir the +/// default run samples into, or the address a forced run inverts. +#[derive(Debug)] +pub(crate) struct Arm { + pub(crate) sampler: Option>, + pub(crate) kinds: Kinds, + /// The shallowest depth a decision is sampled at. + pub(crate) from: u8, + pub(crate) root: usize, + /// Decisions offered so far, which numbers them. + offered: u64, + pub(crate) invert: Option
, + /// How often the search met the inverted address. + pub(crate) visits: u64, + /// For a node decision, what answered the node at the first inverted + /// visit. + pub(crate) answered: Option, +} + +impl Arm { + pub(crate) fn recording(sampler: Sampler, kinds: Kinds, from: u8, root: usize) -> Self { + Arm { + sampler: Some(sampler), + kinds, + from, + root, + offered: 0, + invert: None, + visits: 0, + answered: None, + } + } + + pub(crate) fn inverting(address: Address) -> Self { + Arm { + sampler: None, + kinds: Kinds(0), + from: 0, + root: 0, + offered: 0, + invert: Some(address), + visits: 0, + answered: None, + } + } + + /// Whether this address is the one inverted, counting the visit. A + /// pass not taken leaves the node only its moves; what answers a node + /// whose margin is not taken is the search's to say. + pub(crate) fn inverts(&mut self, address: Address) -> bool { + if self.invert == Some(address) { + self.visits += 1; + if address.kind == Kind::NullMove { + self.answered.get_or_insert(Answered::Moves); + } + true + } else { + false + } + } + + /// Offer a decision taken to the reservoir, when one is armed and the + /// run samples its kind at its depth. `describe` is handed the root's + /// place and the decision's number. + pub(crate) fn offer(&mut self, address: Address, describe: impl FnOnce(usize, u64) -> Event) { + let (root, at) = (self.root, self.offered); + self.offered += 1; + if !self.kinds.holds(address.kind) || address.depth < self.from { + return; + } + if let Some(sampler) = self.sampler.as_mut() { + sampler.event(address.sample_key(), || describe(root, at)); + } + } +} + +/// What one search of a root answered. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Answer { + pub best: Play, + pub score: Score, + pub nodes: u64, +} + +/// One root searched with the arm given, to the depth. The table is the +/// bench's and there is no clock, so two searches of a root agree until +/// the arm makes them differ. +fn search(position: &Position, depth: u8, arm: Arm) -> (Answer, Arm) { + let board = Board::from_fen(&position.fen) + .unwrap_or_else(|e| panic!("forced position {} does not parse: {}", position.id, e)); + let mut engine = AlphaBeta::with_table_bytes(board, bench::TABLE_BYTES); + engine.arm_forced(arm); + let outcome = + engine.iterative_deepening_search(SearchParameters::to_depth(depth), |_, _, _, _| {}); + let arm = engine + .disarm_forced() + .expect("the arm just handed to the engine comes back"); + let result = match outcome { + SearchOutcome::Complete(result, _) => result, + other => panic!("forced position {} answered {:?}", position.id, other), + }; + ( + Answer { + best: result.best_move, + score: result.score, + nodes: result.nodes, + }, + arm, + ) +} + +/// One kept decision, forced. +#[derive(Clone, Debug)] +pub struct Row { + pub event: Event, + pub on: Answer, + pub forced: Answer, + pub visits: u64, + pub answered: Option, +} + +impl Row { + /// Whether the root named another move. + pub fn flipped(&self) -> bool { + self.on.best != self.forced.best + } +} + +/// What a run kept and what each forced search answered. +#[derive(Clone, Debug)] +pub struct Report { + pub depth: u8, + pub every: u32, + pub cap: usize, + pub suite: Option, + pub kinds: Kinds, + pub from: u8, + pub positions: usize, + /// Every decision offered, kept or not. + pub events: u64, + pub overflowed: u64, + /// Kept records, before the revisits of an address were folded. + pub records: usize, + /// Each root's id and what the default answered there. + pub roots: Vec<(String, Answer)>, + pub rows: Vec, +} + +/// Search every root under the default with the reservoir armed, keep the +/// first visit's record of each address, and search each kept address's +/// root again with that decision inverted. +pub fn run( + positions: &[Position], + suite: Option<&str>, + depth: u8, + every: u32, + cap: usize, + kinds: Kinds, + from: u8, +) -> Report { + let depth = depth.max(1); + let every = every.max(1); + let mut sampler = Sampler::with_cap(every, cap); + let mut answers = Vec::with_capacity(positions.len()); + for (root, position) in positions.iter().enumerate() { + let (answer, arm) = search(position, depth, Arm::recording(sampler, kinds, from, root)); + sampler = arm + .sampler + .expect("the reservoir handed to the engine comes back"); + answers.push(answer); + } + let sampled = sampler.drain(); + let records = sampled.taken.len(); + let mut kept = sampled.taken; + // the first visit's record of each address at a root: a revisit carries + // its own window and features, and the forced search inverts them all + kept.sort_by_key(|event| (event.root, event.address, event.at)); + kept.dedup_by_key(|event| (event.root, event.address)); + kept.sort_by_key(|event| (event.root, event.at)); + let rows = kept + .into_iter() + .map(|event| { + let position = &positions[event.root]; + let (forced, arm) = search(position, depth, Arm::inverting(event.address)); + Row { + on: answers[event.root], + forced, + visits: arm.visits, + answered: arm.answered, + event, + } + }) + .collect(); + Report { + depth, + every, + cap, + suite: suite.map(str::to_string), + kinds, + from, + positions: positions.len(), + events: sampled.events, + overflowed: sampled.overflowed, + records, + roots: positions + .iter() + .map(|position| position.id.clone()) + .zip(answers) + .collect(), + rows, + } +} + +/// A count and a share, or a `-` with nothing under it. +fn count_share(part: usize, of: usize) -> String { + format!("{} {}", part, recorder::share(part, of)) +} + +/// A header and the rows, then under `summary` the tallies for each kind +/// and what the default answered at each root. +/// +/// A row is `kind depth index searched generated history history_max +/// killer tt eval_beta alpha_gap attention answered visits root best_on +/// best_forced score_on score_forced nodes_on nodes_forced fen`, +/// whitespace separated with the deciding node's fen last. `root` is the +/// root's place in the suite, which its line names. A move decision's own +/// columns print `-` on a node decision, and `attention` prints `-` below +/// the depth the gate reads the model at. +impl fmt::Display for Report { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!( + f, + "forced depth {} every {} cap {} epd {} kinds {} from {} positions {} events {} records {} rows {} overflow {}", + self.depth, + self.every, + self.cap, + self.suite.as_deref().unwrap_or("bench"), + self.kinds.words(), + self.from, + self.positions, + self.events, + self.records, + self.rows.len(), + self.overflowed, + )?; + for row in &self.rows { + let e = &row.event; + let dash = || "-".to_string(); + let features = match &e.features { + Some(m) => format!( + "{} {} {} {} {} {} {}", + m.index, + m.index + 1, + m.generated, + m.history, + m.history_max, + u8::from(m.killer), + m.tt.word(), + ), + None => ["-"; 7].join(" "), + }; + writeln!( + f, + "{} {} {} {} {} {} {} {} {} {} {} {} {} {} {} {}", + e.address.kind.word(), + e.address.depth, + features, + e.eval_beta, + e.alpha_gap, + e.attention.map_or_else(dash, |score| score.to_string()), + row.answered.map_or("-", Answered::word), + row.visits, + e.root, + row.on.best, + row.forced.best, + row.on.score, + row.forced.score, + row.on.nodes, + row.forced.nodes, + e.fen, + )?; + } + writeln!(f)?; + writeln!(f, "summary")?; + for kind in Kind::ALL { + if !self.kinds.holds(kind) { + continue; + } + let rows: Vec<&Row> = self + .rows + .iter() + .filter(|row| row.event.address.kind == kind) + .collect(); + let unmet = rows.iter().filter(|row| row.visits == 0).count(); + let flipped = rows.iter().filter(|row| row.flipped()).count(); + writeln!( + f, + "kind {} forced {} unmet {} flipped {}", + kind.word(), + rows.len(), + unmet, + count_share(flipped, rows.len()), + )?; + } + for (root, (id, answer)) in self.roots.iter().enumerate() { + writeln!( + f, + "root {} best {} score {} nodes {} {}", + root, answer.best, answer.score, answer.nodes, id, + )?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::recorder::DEFAULT_CAP; + use crate::recorder::fixtures::{recording_leaves_the_search_where_it_was, suite}; + use pretty_assertions::assert_eq; + + #[test] + fn recording_changes_nothing() { + recording_leaves_the_search_where_it_was( + 6, + |engine| { + engine.arm_forced(Arm::recording( + Sampler::with_cap(1, DEFAULT_CAP), + Kinds::ALL, + 0, + 0, + )) + }, + |engine| { + engine + .disarm_forced() + .and_then(|arm| arm.sampler) + .expect("the reservoir comes back") + .drain() + .taken + .len() + }, + ); + } + + /// The null run: a recording arm answers as an unarmed search does, + /// and inverting an address the search never meets reproduces that + /// answer and counts no visit. + #[test] + fn an_address_never_met_changes_nothing() { + for position in &suite() { + let mut unarmed = AlphaBeta::with_table_bytes( + Board::from_fen(&position.fen).unwrap(), + bench::TABLE_BYTES, + ); + let SearchOutcome::Complete(result, _) = + unarmed.iterative_deepening_search(SearchParameters::to_depth(6), |_, _, _, _| {}) + else { + panic!("{}: an unlimited search did not complete", position.id); + }; + let (plain, _) = search( + position, + 6, + Arm::recording(Sampler::with_cap(1, 0), Kinds::ALL, 0, 0), + ); + assert_eq!( + plain, + Answer { + best: result.best_move, + score: result.score, + nodes: result.nodes, + }, + "{}", + position.id + ); + for kind in Kind::ALL { + let (forced, arm) = search( + position, + 6, + Arm::inverting(Address { + kind, + key: 0, + depth: 3, + }), + ); + assert_eq!(forced, plain, "{} {}", position.id, kind.word()); + assert_eq!(arm.visits, 0, "{} {}", position.id, kind.word()); + assert_eq!(arm.answered, None, "{} {}", position.id, kind.word()); + } + } + } + + /// Every kept decision is met when its root is searched again, each + /// kind that was kept is forced at least once, and each address of a + /// root has one row. A row whose visits read zero would be a decision + /// the forced search never reached, which is the instrument failing + /// rather than a finding. + #[test] + fn every_kept_decision_is_met_when_forced_and_rowed_once() { + let report = run(&suite(), None, 6, 40, 400, Kinds::ALL, 0); + assert!(!report.rows.is_empty(), "nothing was kept"); + let mut addresses: Vec<_> = report + .rows + .iter() + .map(|row| (row.event.root, row.event.address)) + .collect(); + addresses.sort(); + addresses.dedup(); + assert_eq!(addresses.len(), report.rows.len()); + assert!(report.records >= report.rows.len()); + for row in &report.rows { + assert!(row.visits > 0, "{:?} was never met", row.event.address); + match row.event.address.kind { + Kind::ReverseFutility => assert!(row.answered.is_some()), + Kind::NullMove => assert_eq!(row.answered, Some(Answered::Moves)), + Kind::Skip | Kind::TrustedScout => { + assert_eq!(row.answered, None); + assert!(row.event.features.is_some()); + } + } + } + for kind in [Kind::ReverseFutility, Kind::Skip, Kind::TrustedScout] { + assert!( + report.rows.iter().any(|row| row.event.address.kind == kind), + "no {} row", + kind.word() + ); + } + } + + #[test] + fn the_kinds_asked_for_are_the_kinds_sampled() { + let report = run(&suite(), None, 6, 40, 400, Kinds::of(&[Kind::Skip]), 2); + assert!(!report.rows.is_empty()); + assert!( + report + .rows + .iter() + .all(|row| row.event.address.kind == Kind::Skip && row.event.address.depth >= 2) + ); + assert_eq!(Kinds::of(&[Kind::Skip]).words(), "skip"); + assert_eq!( + Kinds::ALL.words(), + "reverse_futility,null_move,skip,trusted_scout" + ); + for kind in Kind::ALL { + assert_eq!(Kind::of_word(kind.word()), Some(kind)); + } + assert_eq!(Kind::of_word("skips"), None); + } + + /// The model's score is printed where the gate reads one, depth four + /// and up, and nowhere else. + #[test] + fn attention_is_read_where_the_gate_reads_it() { + let report = run(&suite(), None, 7, 20, 2000, Kinds::ALL, 0); + let mut deep = 0; + for row in &report.rows { + let e = &row.event; + let gated = e.features.is_some() + && e.address.depth >= crate::late_move::DEEP_REDUCTION_MIN_DEPTH; + assert_eq!(e.attention.is_some(), gated, "{:?}", e.address); + deep += usize::from(gated); + } + assert!(deep > 0, "no row at the gate's depth"); + } +} diff --git a/arche-core/src/late_move.rs b/arche-core/src/late_move.rs index 9290ff2..078fbde 100644 --- a/arche-core/src/late_move.rs +++ b/arche-core/src/late_move.rs @@ -223,6 +223,25 @@ fn attention_score(f: &AttentionFeatures) -> i64 { + ATTENTION_INTERCEPT } +/// The model's score for a late quiet the gate would read it at, from what +/// the node knew there, for the forced decision instrument's rows. None +/// below the depth the gate reads the model at, where no decision of the +/// move ever consulted it. +pub(crate) fn attention(depth: u8, f: &Features, eval_beta: i64, alpha_gap: i64) -> Option { + (depth >= DEEP_REDUCTION_MIN_DEPTH).then(|| { + attention_score(&AttentionFeatures { + depth, + index: f.index, + hist_milli: f.hist_milli(), + killer: f.killer, + tt: f.tt, + eval_beta, + alpha_gap, + generated: f.generated, + }) + }) +} + /// A reduction ledger row scored as `scripts/fit_attention.py` reads it, /// with the searched column as printed rather than derived from the index. /// What holds the printed column to the one the gate read. diff --git a/arche-core/src/lib.rs b/arche-core/src/lib.rs index ca66773..ce26160 100644 --- a/arche-core/src/lib.rs +++ b/arche-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod census; pub mod effort; mod engine; mod eval; +pub mod forced; mod ghi; mod late_move; mod limits; diff --git a/arche-core/src/recorder.rs b/arche-core/src/recorder.rs index 604a2f8..89a0a19 100644 --- a/arche-core/src/recorder.rs +++ b/arche-core/src/recorder.rs @@ -63,14 +63,18 @@ pub(crate) const CENSUS_LANE: u64 = 0xc5b9_128e_66d0_3a47; pub(crate) const LEDGER_LANE: u64 = 0x6d84_3b2f_51c9_07ea; /// Used on both of the effort instrument's sides, which join on the key. pub(crate) const EFFORT_LANE: u64 = 0xf3b7_0c95_a41e_d682; +/// The forced decision instrument's, for the decisions its default run +/// samples. +pub(crate) const FORCED_LANE: u64 = 0x8a5d_e163_2f07_b94c; -pub(crate) const LANES: [u64; 6] = [ +pub(crate) const LANES: [u64; 7] = [ REVERSE_FUTILITY_LANE, NULL_MOVE_LANE, SHADOW_FUTILITY_LANE, CENSUS_LANE, LEDGER_LANE, EFFORT_LANE, + FORCED_LANE, ]; /// The invariant, checked by the compiler: a lane copied from another would diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a8a044e..aabfdff 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -196,6 +196,10 @@ term over every pair of pieces. once with a named switch off, and joins the two runs by the node, so a row says whether each side reached it and what each spent under it. Driven by the `effort` argument. +- **forced.rs**: What one shortcut decision cost the root. It samples the + decisions a default search takes, then searches each root again with one + of them inverted wherever the search meets it, and reports what the root + answered both times. Driven by the `forced` argument. - **tune.rs**: What a position's evaluation is made of. The evaluation is linear in its weights except where material cannot mate, and for the pair term, which a row carries as a number of its own. This writes down the diff --git a/docs/INSTRUMENTS.md b/docs/INSTRUMENTS.md index bc24a4f..8c9de43 100644 --- a/docs/INSTRUMENTS.md +++ b/docs/INSTRUMENTS.md @@ -1,10 +1,10 @@ # Instruments These measurements ask what the engine gave up rather than how large a tree it -walked. `residuals`, `cutoffs`, `reductions` and `effort` are arguments of their -own and measure the search, as does the bench's `audit` word. `terms` measures -the evaluation, and the last section is the offline harness under `scripts/` -that fits and scores the weights `terms` states. +walked. `residuals`, `cutoffs`, `reductions`, `effort` and `forced` are +arguments of their own and measure the search, as does the bench's `audit` +word. `terms` measures the evaluation, and the last section is the offline +harness under `scripts/` that fits and scores the weights `terms` states. [DEVELOPMENT.md](DEVELOPMENT.md) has the bench itself and everything else a change needs before it is committed. Nothing in this file is needed for that. @@ -20,7 +20,7 @@ cap` is refused with `cap: no value`, and so is one named twice: a run takes minutes, and one started at a default nobody typed answers a question that was not asked. -All of these print to standard output; redirect it to keep a run. The four +All of these print to standard output; redirect it to keep a run. The five arguments and `terms` print a row per sample, whitespace separated with the fen last so a row parses left to right, under a header that states what the run used, so it can be rerun from what it printed. @@ -376,6 +376,78 @@ searches under a configuration its caller chose. There is no pinned count for a configuration with a switch off, and there should not be one, since it would need rewriting whenever the search gained a rule. +## What one decision cost the root + +The ledger and the residuals label a decision at its own node. Whether being +wrong there cost the root anything is another question, and most such errors +cost nothing: a re-search recovers them, or the root's move survives them. +`arche forced` asks it: + +``` +target/release/arche forced [depth] [every ] [cap ] [epd ] [kinds [,]] [from ] +``` + +It searches each root of the suite under the default, sampling the shortcut +decisions taken, and then searches the root again once for each sampled +decision with that one decision inverted. Four kinds can be inverted: + +| kind | taken | inverted | +| --- | --- | --- | +| `reverse_futility` | the margin answered the node | the node does not answer from the margin, and the null move then gets its turn | +| `null_move` | the pass cleared beta | the node goes on to its moves | +| `skip` | a late quiet was passed over, by the model at depth four and up or by a shallow rule below | the move is searched unreduced | +| `trusted_scout` | a reduced scout came back at or below alpha | the move goes on to the probe and the proof, as if the scout had failed high | + +`kinds` narrows the sampling to the kinds named, and `from` to decisions at +that depth and deeper. Shallow decisions are most of them, so a run after +the model's decisions asks for `from 4`. + +A decision is addressed by its kind, a position key and the deciding node's +depth. A node decision is keyed by the node's position; a move decision by +the position the move leaves, as the ledger keys its rows. Two parents that +reach one child at one depth therefore share an address, and a forced run +inverts both. The inversion applies wherever the search meets the address: +every visit, in every iteration of the deepening and every aspiration +re-search, so a row is that decision forced wherever it comes up and not one +occurrence of it. Each root starts from a fresh engine and table with no +clock, so the two searches agree until the first such visit. The sampler +keeps revisits of an address as separate records, and the row keeps the +first visit's. + +An inverted skip is searched unreduced, where with the rule off it would +usually have been scouted first. That is the depth the ledger's replay asks +about, but the two still differ: the replay is the reference on a cold table +over the full window, and a forced row is the default in the node's own +window. The moves after an inverted skip read a searched count one higher. +An inverted pass leaves its taint in the node, as a pass that fails does, and +an inverted scout passes its taint on, as a scout that fails high does. + +While the instrument is armed the move loop asks the shallow rules move by +move, as it does under the ledger, since a run that drops a whole run of +quiets at once never meets the skips in it. The tests hold an armed search's +move, score and node count to an unarmed one's. + +Each row is `kind depth index searched generated history history_max killer +tt eval_beta alpha_gap attention answered visits root best_on best_forced +score_on score_forced nodes_on nodes_forced fen`. The move columns are the +ledger's, read at the first visit, and print `-` on a node decision. +`attention` is the model's score where the gate reads one (a late quiet at +depth four and up) and `-` elsewhere; every skip at those depths scores at +or under the pruning threshold by construction, so the column is read within +a kind. `answered` is, for a node decision, what answered the node once it +was inverted. `visits` is how often the forced search met the address, and a +row that reads zero is the instrument failing, not a finding. `root` is the +root's place in the suite, and the fen is the deciding node's. + +The run ends, under `summary`, with a line per kind (rows forced, rows never +met, rows whose root move changed) and a line per root with what the default +answered there and the root's id last. + +Every kept decision costs a search of its root, so the default rate is one +decision in 100,000. What a changed move cost is not here: a row whose root +move did not change lost nothing to the decision, and one whose move did +change needs its two moves valued by a deeper search, offline. + ## What a position's evaluation is made of This one measures the evaluation, and it is the engine's half of the tuner: diff --git a/src/instruments.rs b/src/instruments.rs index 69d2d77..84729ee 100644 --- a/src/instruments.rs +++ b/src/instruments.rs @@ -17,6 +17,7 @@ use arche_core::SearchConfig; use arche_core::bench; use arche_core::census; use arche_core::effort; +use arche_core::forced; use arche_core::recorder; use arche_core::reduction; use arche_core::residual; @@ -37,7 +38,7 @@ pub struct Instrument { /// Every command the binary takes, in usage order. The dispatch and the /// usage both walk it, so a command cannot be listed without being taken. -pub const INSTRUMENTS: [Instrument; 6] = [ +pub const INSTRUMENTS: [Instrument; 7] = [ Instrument { command: &uci::BENCH, read: read_bench, @@ -58,6 +59,10 @@ pub const INSTRUMENTS: [Instrument; 6] = [ command: &EFFORT, read: |params| report(effort_settings(params)?, EffortSettings::run), }, + Instrument { + command: &FORCED, + read: |params| report(forced_settings(params)?, ForcedSettings::run), + }, Instrument { command: &TERMS, read: |params| report(term_settings(params)?, TermSettings::run), @@ -209,6 +214,39 @@ pub const EFFORT: Command = Command { ], }; +pub const FORCED: Command = Command { + name: "forced", + depth: true, + keywords: &[ + Keyword { + word: "every", + value: "", + }, + Keyword { + word: "cap", + value: "", + }, + Keyword { + word: "epd", + value: "", + }, + Keyword { + word: "kinds", + value: "[,]", + }, + Keyword { + word: "from", + value: "", + }, + ], + flags: &[], + summary: &[ + "search the bench's suite, or the one named, sample the", + "shortcut decisions taken, and search each root again with", + "one of them inverted", + ], +}; + pub const TERMS: Command = Command { name: "terms", // the walk reads the board and the quiet test runs a capture search, @@ -456,6 +494,75 @@ impl EffortSettings { } } +/// What a forced argument asked for. `kinds` narrows the decisions sampled +/// to those named, comma separated, and is absent for all four. `from` is +/// the shallowest depth a decision is sampled at, since the shallow ones +/// are most of them and a run may be after the deep. +pub struct ForcedSettings { + pub depth: u8, + pub every: u32, + pub cap: usize, + pub kinds: forced::Kinds, + pub from: u8, + pub epd: Option, + pub positions: Vec, +} + +/// The refusal for a `kinds` that names no kind, listing them. +fn no_such_kind(word: &str) -> String { + let kinds = forced::Kind::ALL.map(forced::Kind::word).join(", "); + format!("kinds: {word} (a kind is one of {kinds})") +} + +/// The kinds `kinds` names. A refusal echoes the whole word, and a kind +/// named twice is refused, since it is a word the reader did not mean. +fn kinds(word: &str) -> Result { + let mut named = Vec::new(); + for part in word.split(',') { + let kind = forced::Kind::of_word(part).ok_or_else(|| no_such_kind(word))?; + if named.contains(&kind) { + return Err(format!("kinds: {word} (a kind named twice)")); + } + named.push(kind); + } + Ok(forced::Kinds::of(&named)) +} + +pub fn forced_settings(params: &Params) -> Result { + let Sampling { depth, every, cap } = sampling(params, &FORCED, forced::DEFAULT_EVERY)?; + let kinds = match params.value("kinds") { + Param::Absent => forced::Kinds::ALL, + Param::Read(word) => kinds(word)?, + Param::Bare => return Err(no_such_kind(NO_VALUE)), + Param::Unreadable(word) => return Err(no_such_kind(word)), + }; + let from = params.parse::("from").or_refuse("from")?.unwrap_or(0); + let (epd, positions) = suite(params)?; + Ok(ForcedSettings { + depth, + every, + cap, + kinds, + from, + epd, + positions, + }) +} + +impl ForcedSettings { + pub fn run(&self) -> forced::Report { + forced::run( + &self.positions, + self.epd.as_deref(), + self.depth, + self.every, + self.cap, + self.kinds, + self.from, + ) + } +} + /// What a terms argument asked for. No depth, rate or cap: a run states /// every quiet position of the suite, because the corpus is what is being /// built. @@ -531,6 +638,8 @@ mod tests { "epd" => SUITE, "taint" => "trust", "off" => "null_move", + "kinds" => "skip", + "from" => "1", _ => panic!("no value to give {keyword}"), } } @@ -947,6 +1056,32 @@ mod tests { } } + #[test] + fn forced_reads_the_kinds_named_and_refuses_the_rest() { + let read = |line: &str| forced_settings(&Params::of(line)).map(|s| s.kinds); + assert_eq!( + read("forced kinds skip,trusted_scout"), + Ok(forced::Kinds::of(&[ + forced::Kind::Skip, + forced::Kind::TrustedScout + ])) + ); + let listed = "(a kind is one of reverse_futility, null_move, skip, trusted_scout)"; + for (line, refused) in [ + ("forced kinds skips", format!("kinds: skips {listed}")), + ( + "forced kinds skip,nul_move", + format!("kinds: skip,nul_move {listed}"), + ), + ( + "forced kinds skip,skip", + "kinds: skip,skip (a kind named twice)".to_string(), + ), + ] { + assert_eq!(read(line).err(), Some(refused), "{line}"); + } + } + /// The four rate defaults are the same number today, so the assertions /// on them cannot tell which one a reader passed. The loop only shows /// that `sampling` uses the rate it is handed. @@ -956,18 +1091,33 @@ mod tests { let cutoffs = cutoff_settings(&Params::of("cutoffs")).unwrap(); let reductions = reduction_settings(&Params::of("reductions")).unwrap(); let effort = effort_settings(&Params::of("effort")).unwrap(); + let forced = forced_settings(&Params::of("forced")).unwrap(); for depth in [ residuals.depth, cutoffs.depth, reductions.depth, effort.depth, + forced.depth, ] { assert_eq!(depth, bench::DEPTH); } - for cap in [residuals.cap, cutoffs.cap, reductions.cap, effort.cap] { + for cap in [ + residuals.cap, + cutoffs.cap, + reductions.cap, + effort.cap, + forced.cap, + ] { assert_eq!(cap, DEFAULT_CAP); } + assert_eq!(forced.every, forced::DEFAULT_EVERY); + assert_eq!(forced.kinds, forced::Kinds::ALL); + assert_eq!(forced.from, 0); + assert_eq!( + forced_settings(&Params::of("forced from 4")).map(|s| s.from), + Ok(4) + ); assert_eq!(residuals.every, residual::DEFAULT_EVERY); assert_eq!(cutoffs.every, census::DEFAULT_EVERY); assert_eq!(reductions.every, reduction::DEFAULT_EVERY); @@ -978,6 +1128,7 @@ mod tests { (&CUTOFFS, "cutoffs", 22), (&REDUCTIONS, "reductions", 33), (&EFFORT, "effort", 44), + (&FORCED, "forced", 55), ] { let read = sampling(&Params::of(word), command, default).expect(word); assert_eq!(read.every, default, "{word} took a rate not its own"); diff --git a/tests/forced_session.rs b/tests/forced_session.rs new file mode 100644 index 0000000..697e6b3 --- /dev/null +++ b/tests/forced_session.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2022-2026 Andrew Wright + +//! What the forced argument prints, run against the real binary. The +//! spawning and the splitting are in `report_command`. + +mod report_command; + +/// Shallow, and at a rate that keeps a few dozen decisions from the bench's +/// suite, each of which costs a search of its root. +const ARGUMENTS: [&str; 4] = ["forced", "5", "every", "3000"]; + +#[test] +fn the_forced_argument_prints_a_header_rows_and_a_line_a_kind_and_a_root() { + let printed = report_command::run(&ARGUMENTS); + assert!( + printed.header.starts_with( + "forced depth 5 every 3000 cap 10000 epd bench \ + kinds reverse_futility,null_move,skip,trusted_scout from 0 positions " + ), + "header: {}", + printed.header + ); + assert!(printed.events() > 0, "header: {}", printed.header); + for row in &printed.rows { + let words: Vec<&str> = row.split(' ').collect(); + // twenty one columns before a fen of six fields + assert_eq!(words.len(), 27, "row: {row}"); + assert!( + ["reverse_futility", "null_move", "skip", "trusted_scout"].contains(&words[0]), + "row: {row}" + ); + // depth, visits, root, the two scores and the two node counts + for at in [1, 13, 14, 17, 18, 19, 20] { + assert!(words[at].parse::().is_ok(), "field {at} of {row}"); + } + // a kept decision the forced search never met is a failed row + assert!(words[13] != "0", "row: {row}"); + } + let kinds = printed + .summary + .iter() + .filter(|line| line.starts_with("kind ")) + .count(); + let roots = printed + .summary + .iter() + .filter(|line| line.starts_with("root ")) + .count(); + assert_eq!(kinds, 4, "{}", printed.all); + assert!(roots > 0, "{}", printed.all); +} diff --git a/tests/uci_session.rs b/tests/uci_session.rs index 870553d..771e863 100644 --- a/tests/uci_session.rs +++ b/tests/uci_session.rs @@ -621,6 +621,10 @@ fn a_setting_that_cannot_be_read_is_refused_on_stderr() { &["effort", "every", "abc"], "unrecognised effort every: abc", ), + ( + &["forced", "every", "abc"], + "unrecognised forced every: abc", + ), // terms has no rate; its suite is the setting that can fail to read ( &["terms", "epd", "no/such/file.epd"],