diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index cd6ceb6f1..3731031c1 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -1,9 +1,10 @@ //! `nodes_scale` (`docs/contrib/dgx-spark.md` §9.9): the node plans on -//! synthetic DGX Sparks, N = 1 to 4, against the cells of §9.8's "where each -//! fires" table as the predictor derives them; a clause whose cells differ -//! from the table names the difference. Three synthetic shapes carry §9.7's -//! model terms for V4.1, GLM-5.3-Flash and Qwen3.8, A low and high; no file -//! is read but where a clause reads the Qwen3.8 file's tensor table. +//! synthetic DGX Sparks, N = 1 to 4, against the expected cells that +//! `tools/flow/nodes_scale.py` derives from §9.7's model and §9.8's +//! criteria apart from the planner (`tests/nodes_scale_cells.tsv`). Three +//! synthetic shapes carry §9.7's model terms for V4.1, GLM-5.3-Flash and +//! Qwen3.8, A low and high; no file is read but where a clause reads the +//! Qwen3.8 file's tensor table. use std::sync::OnceLock; @@ -412,6 +413,380 @@ fn used(plan: &NodePlan<'_>) -> u64 { .count() as u64 } +/// The inputs `tools/flow/nodes_scale.py` derives the expected cells from. +const INPUTS: &str = include_str!("../../tests/nodes_scale_inputs.tsv"); + +/// The expected cells (§9.9 "Expected sets"), as that script writes them. +const CELLS: &str = include_str!("../../tests/nodes_scale_cells.tsv"); + +/// The value of the inputs row `name`. +fn input(name: &str) -> u64 { + INPUTS + .lines() + .filter(|l| !l.starts_with('#')) + .skip(1) + .find_map(|l| { + let mut c = l.split('\t'); + (c.next() == Some(name)).then(|| c.next()) + }) + .flatten() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| panic!("nodes_scale_inputs.tsv: no whole number for {name}")) +} + +/// A derived value whose band holds its criterion's line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + /// The busiest NIC, on criterion 5's half. + Nic, + /// Node `node`'s link share in ppm, on criterion 2's tenth. + Link { node: usize, ppm: u64, tol: u64 }, + /// Node `node`'s gain in ppm, on criterion 7's twice the ruler. + Gain { node: usize, ppm: u64, tol: u64 }, +} + +impl Edge { + fn criterion(self) -> &'static str { + match self { + Edge::Nic => "nic", + Edge::Link { .. } => "link", + Edge::Gain { .. } => "gain", + } + } +} + +/// One derived cell: a cell at a ruler and S₁, what the script derives of +/// its plan, and each value's tolerance, the largest change one id moved +/// between two used nodes makes. +#[derive(Debug)] +struct Derived { + cell: Cell, + ruler: u64, + rate: u64, + used: u64, + /// The idle nodes and why, `node:pool`, `node:link` or `node:gain`, or + /// `-`. + idle: String, + paged: u64, + /// Each node's paged ids, to one id. + paged_nodes: Vec, + step_ns: u64, + step_tol: u64, + nic: u64, + nic_tol: u64, + fires: bool, + /// Node 0's messages in the layer that reaches the most remotes. + msgs: u64, + edges: Vec, +} + +/// Every derived cell. +fn derived() -> &'static [Derived] { + static DERIVED: OnceLock> = OnceLock::new(); + DERIVED.get_or_init(|| { + let mut rows = CELLS.lines().filter(|l| !l.starts_with('#')); + assert_eq!( + rows.next(), + Some( + "model\ta\tn\tlambda_us\truler\ts1\tused\tidle\tpaged\tpaged_nodes\tstep_ns\tstep_tol\tnic\tnic_tol\tfires\tmsgs\tedge" + ), + "nodes_scale_cells.tsv's header" + ); + rows.map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + let [model, a, n, lambda, ruler, rate, used, idle, paged, per, step, step_tol, nic, nic_tol, fires, msgs, edge] = + c[..] + else { + panic!("nodes_scale_cells.tsv: {l:?} is not 17 cells"); + }; + let num = |v: &str| -> u64 { + v.parse() + .unwrap_or_else(|_| panic!("nodes_scale_cells.tsv: {v:?} in {l:?}")) + }; + let name = SHAPES + .iter() + .map(|s| s.name) + .find(|&s| s == model) + .unwrap_or_else(|| panic!("nodes_scale_cells.tsv: no shape {model}")); + let edges = edge + .split(',') + .filter(|&e| e != "-") + .map(|e| { + let f: Vec<&str> = e.split(':').collect(); + match f[..] { + ["nic"] => Edge::Nic, + ["link", node, ppm, tol] => Edge::Link { + node: num(node) as usize, + ppm: num(ppm), + tol: num(tol), + }, + ["gain", node, ppm, tol] => Edge::Gain { + node: num(node) as usize, + ppm: num(ppm), + tol: num(tol), + }, + _ => panic!("nodes_scale_cells.tsv: edge {e:?} in {l:?}"), + } + }) + .collect(); + Derived { + cell: (name, num(n) as usize, num(lambda), a == "high"), + ruler: num(ruler), + rate: num(rate), + used: num(used), + idle: idle.to_string(), + paged: num(paged), + paged_nodes: per.split(',').map(num).collect(), + step_ns: num(step), + step_tol: num(step_tol), + nic: num(nic), + nic_tol: num(nic_tol), + fires: fires == "1", + msgs: num(msgs), + edges, + } + }) + .collect() + }) +} + +/// The derived cell of `cell` at `ruler` and S₁ `rate`. +fn derived_at(cell: Cell, ruler: u64, rate: u64) -> &'static Derived { + derived() + .iter() + .find(|d| d.cell == cell && d.ruler == ruler && d.rate == rate) + .unwrap_or_else(|| panic!("no derived cell {cell:?} at ruler {ruler}, S₁ {rate}")) +} + +/// The side of each derived edge, pinned here: a cell, its ruler and S₁, the +/// edge's criterion, and whether that criterion fires there (criterion 2 or +/// 7 leaves the node idle, criterion 5 names the NIC). +const SIDES: [(Cell, u64, u64, &str, bool); 4] = [ + (("V4.1", 2, 25, false), RULER, 3_000, "nic", true), + (("V4.1", 2, 25, false), 4 * RULER, 3_000, "nic", true), + (("Qwen3.8", 3, 70, false), RULER, 1_000, "link", false), + (("Qwen3.8", 3, 70, false), RULER, 3_000, "link", false), +]; + +/// Holds the derived side of `e` in `d` to its pin in [`SIDES`]; an edge +/// with no pin, or whose derived side is not its pin, panics by name. +fn side(d: &Derived, e: Edge) { + let fires = match e { + Edge::Nic => d.fires, + Edge::Link { node, .. } => d.idle.split(',').any(|i| i == format!("{node}:link")), + Edge::Gain { node, .. } => d.idle.split(',').any(|i| i == format!("{node}:gain")), + }; + let (_, _, _, _, pin) = SIDES + .iter() + .find(|s| s.0 == d.cell && s.1 == d.ruler && s.2 == d.rate && s.3 == e.criterion()) + .unwrap_or_else(|| { + panic!( + "{:?} at ruler {}, S₁ {}: a {} edge with no pinned side", + d.cell, + d.ruler, + d.rate, + e.criterion() + ) + }); + assert_eq!( + fires, + *pin, + "{:?} at ruler {}, S₁ {}: the derived {} side is not the pinned one", + d.cell, + d.ruler, + d.rate, + e.criterion() + ); +} + +/// Whether a link of `link_ns` can pass a tenth of a step of `step_ns`, +/// both rounded down to whole nanoseconds: the link is under `link_ns + 1` +/// and the step at least `step_ns`. +fn past_tenth(link_ns: u64, step_ns: u64) -> bool { + 10 * (link_ns + 1) > step_ns +} + +/// The idle nodes of `plan` as the derived cells name them. +fn idle_of(plan: &NodePlan<'_>) -> String { + let idle: Vec = plan + .idle() + .iter() + .map(|i| { + let why = match i.why { + IdleWhy::Pool { .. } => "pool", + IdleWhy::LinkShare { .. } => "link", + IdleWhy::Gain { .. } => "gain", + }; + format!("{}:{why}", i.node) + }) + .collect(); + if idle.is_empty() { + "-".to_string() + } else { + idle.join(",") + } +} + +/// §9.9 "Expected sets": the cells are `tools/flow/nodes_scale.py`'s, +/// derived from `nodes_scale_inputs.tsv`, whose rows are this gate's own +/// machine, shapes and budgets. In every cell the plan uses the derived +/// nodes and leaves the others idle for the derived criterion, pages the +/// derived ids in all and on each node to one id, and its step and busiest +/// NIC are the derived ones to one id's change. Where a derived value's band +/// holds its line, the side is pinned in [`SIDES`], and a link share is +/// held to the band, read from the node's `NodeIdle` or, where the node is +/// the last used, from the plan; so is a gain, from a node's `NodeIdle`. +#[test] +fn expected_sets() { + for (name, value) in [ + ("read", GB10_READ), + ("nic", GB10_NIC), + ("pool", case_of(("V4.1", 1, 25, false)).budgets[0].pool()), + ("lambda_lo", LAMBDAS[0]), + ("lambda_hi", LAMBDAS[1]), + ("hop", HOP_NS), + ("post", POST_NS), + ("row_value", 4), + ("vocab", VOCAB), + ("ruler", RULER), + ("ruler_wide", 4 * RULER), + ("s1_lo", PROMPT_RATES[0]), + ("s1_hi", PROMPT_RATES[1]), + ] { + assert_eq!(input(name), value, "{name}"); + } + for case in cases() { + let s = case.shape; + let end = if case.high { "hi" } else { "lo" }; + let at = usize::from(case.high); + for (term, value) in [ + ("layers".to_string(), s.layers as u64), + ("n_embd".to_string(), s.n_embd), + ("experts".to_string(), s.experts), + ("top_k".to_string(), s.used), + ("expert".to_string(), s.expert), + (format!("shared_{end}"), s.shared[at]), + (format!("serial_{end}"), s.serial[at]), + (format!("fixed0_{end}"), case.budgets[0].fixed()), + ("fixed1".to_string(), case.budgets[1].fixed()), + ] { + assert_eq!( + input(&format!("{}/{term}", s.name)), + value, + "{}/{term}", + s.name + ); + } + assert_eq!(case.budgets[1].pool(), case.budgets[0].pool()); + } + let mut seen = 0; + let mut pinned = Vec::new(); + for (ruler, rate) in [ + (RULER, PROMPT_RATES[0]), + (RULER, PROMPT_RATES[1]), + (4 * RULER, PROMPT_RATES[1]), + ] { + each_cell(ruler, rate, |cell, plan| { + seen += 1; + let d = derived_at(cell, ruler, rate); + let at = format!("{cell:?} at ruler {ruler}, S₁ {rate}"); + assert_eq!( + (used(plan), idle_of(plan)), + (d.used, d.idle.clone()), + "{at}: the nodes used and idle" + ); + let p = plan.predict().expect("a prediction"); + let id: u64 = plan.terms().layers.iter().map(|l| l.expert).sum(); + let paged: Vec = plan + .owned() + .iter() + .enumerate() + .map(|(j, &o)| { + let held = if o == 0 { + 0 + } else { + let fixed = case_of(cell).budgets[j.min(1)].fixed(); + p.per_node[j] + .pool + .checked_sub(fixed) + .expect("a used node's pool holds its budget") + / id + }; + o.checked_sub(held).expect("a node holds at most its ids") + }) + .collect(); + assert!( + paged.iter().sum::() == d.paged + && paged.len() == d.paged_nodes.len() + && paged + .iter() + .zip(&d.paged_nodes) + .all(|(a, b)| a.abs_diff(*b) <= 1), + "{at}: paged {paged:?}, derived {} as {:?} to one id each", + d.paged, + d.paged_nodes + ); + assert!( + p.step_ns.abs_diff(d.step_ns) <= d.step_tol, + "{at}: step {} ns, derived {} ± {} ns", + p.step_ns, + d.step_ns, + d.step_tol + ); + let busiest = p.per_node.iter().map(|l| l.nic_permille).max().unwrap_or(0); + assert!( + busiest.abs_diff(d.nic) <= d.nic_tol, + "{at}: NIC {busiest} ‰, derived {} ± {} ‰", + d.nic, + d.nic_tol + ); + for &e in &d.edges { + side(d, e); + pinned.push((cell, ruler, rate, e.criterion())); + let (node, want, tol) = match e { + Edge::Nic => continue, + Edge::Link { node, ppm, tol } | Edge::Gain { node, ppm, tol } => { + (node, ppm, tol) + } + }; + let why = plan.idle().iter().find(|i| i.node == node).map(|i| i.why); + let last = used(plan) == node as u64 + 1; + let ppm = match (e, why) { + (Edge::Link { .. }, Some(IdleWhy::LinkShare { link_ns, step_ns })) => { + assert!(past_tenth(link_ns, step_ns), "{at}: {why:?}"); + i128::from(link_ns) * 1_000_000 / i128::from(step_ns) + } + (Edge::Link { .. }, None) if last => { + i128::from(p.link_ns) * 1_000_000 / i128::from(p.step_ns) + } + ( + Edge::Gain { .. }, + Some(IdleWhy::Gain { + without_ns, + with_ns, + }), + ) => { + (i128::from(without_ns) - i128::from(with_ns)) * 1_000_000 + / i128::from(without_ns) + } + (Edge::Gain { .. }, None) => continue, + _ => panic!("{at}: no witness of {e:?} on node {node}, idle {why:?}"), + }; + assert!( + ppm.abs_diff(i128::from(want)) <= u128::from(tol), + "{at}: node {node}'s {} {ppm} ppm, derived {want} ± {tol}", + e.criterion() + ); + } + }); + } + assert_eq!(seen, derived().len()); + let mut sides: Vec<_> = SIDES.iter().map(|s| (s.0, s.1, s.2, s.3)).collect(); + pinned.sort(); + sides.sort(); + assert_eq!(pinned, sides, "a pinned side with no derived edge"); +} + /// Clause 1, `n1_is_today`: one Spark's machine is [`plan_on`] the GB10, /// and its plan is today's: held whole, node 0's bytes are today's pool sum /// ([`PoolNeed`]), or both refuse it by the same bytes; paged, node 0 holds @@ -818,15 +1193,23 @@ fn read_balance() { } /// Clause 4, `link_share`: the chosen plan's link term is at most a tenth of -/// its step, and a node is dropped for it exactly where criterion 2 fires, -/// as §9.8's table names: Qwen3.8 at N = 4, λ 70, A low, which then uses -/// two of the four nodes. The share is read on the exact steps: three ids -/// of 1,000 B at top-k 1 read at 10¹¹ B/s, λ 17,000 ps, split [2, 1], -/// expose 5,666⅔ of 55,666⅔ ps (10.2 %), and three of 19,199 B at -/// 10¹² B/s, λ 32,000 ps, expose 10,666⅔ of 106,661⅔ ps, past a tenth by -/// ½ ps. +/// its step, and a node dropped for it would pass the tenth, read on its +/// whole nanoseconds ([`past_tenth`]). [`expected_sets`] holds the nodes +/// dropped for it to the derived cells where criterion 2 fires; here, a plan +/// that drops one uses two nodes, and Qwen3.8 at N = 4, λ 70, A low is among +/// them. The share is read on the exact steps: three ids of 1,000 B at +/// top-k 1 read at 10¹¹ B/s, λ 17,000 ps, split [2, 1], expose 5,666⅔ of +/// 55,666⅔ ps (10.2 %), and three of 19,199 B at 10¹² B/s, λ 32,000 ps, +/// expose 10,666⅔ of 106,661⅔ ps, past a tenth by ½ ps. #[test] fn link_share() { + let dropped_past = |at: &dyn std::fmt::Debug, plan: &NodePlan<'_>| { + for i in plan.idle() { + if let IdleWhy::LinkShare { link_ns, step_ns } = i.why { + assert!(past_tenth(link_ns, step_ns), "{at:?}: {i:?}"); + } + } + }; let mut fired = Vec::new(); each_cell(RULER, 3_000, |cell, plan| { let step = plan.step_ps().expect("a step"); @@ -835,6 +1218,7 @@ fn link_share() { link.times(10).expect("ten links") <= step, "{cell:?}: link {link:?} of step {step:?}" ); + dropped_past(&cell, plan); if plan .idle() .iter() @@ -844,7 +1228,7 @@ fn link_share() { assert_eq!(used(plan), 2, "{cell:?}"); } }); - assert_eq!(fired, vec![("Qwen3.8", 4, 70, false)]); + assert!(fired.contains(&("Qwen3.8", 4, 70, false)), "{fired:?}"); let model = tiny(3, 1, 1_000); let machine = spark(GB10_POOL, 1); let link = LinkCost { round_trip_ns: 17 }; @@ -884,21 +1268,26 @@ fn link_share() { } }] ); + dropped_past(&"19,199 B ids", &plan); } -/// Node 0's NIC, in permille, on a plan of `owned` ids: a token's mean -/// rows, one a slot on a remote, kept a fraction, against the decode step -/// and a prompt batch at S₁ times the one-node token's bytes over A and the -/// slowest node's FFN, node 0's with Sh. -fn nic_of(shape: Shape, owned: &[u64], step_ns: u64, s1: u64) -> u128 { +/// Node 0's bytes a token on a plan of `owned` ids, times the experts: the +/// mean rows of its slots on the remotes, one a slot, kept a fraction. +fn nic_bytes(shape: Shape, owned: &[u64]) -> u128 { + let layers = shape.layers as u64; + let k = u128::from(shape.used); + let remote: u128 = owned[1..].iter().map(|&o| u128::from(o)).sum(); + u128::from(layers * 4 * shape.n_embd) * k * remote +} + +/// Node 0's NIC share of a prompt batch on such a plan at S₁ `s1`, a NIC of +/// one byte a second: S₁ times the one-node token's bytes over A and the +/// slowest node's FFN, node 0's with Sh, as a numerator and a denominator. +fn nic_prompt(shape: Shape, owned: &[u64], s1: u64) -> (u128, u128) { let (a, sh, e) = shape.bytes(false); let layers = shape.layers as u64; let (sh, id) = (u128::from(sh * layers), u128::from(e * layers)); let (k, ex) = (u128::from(shape.used), u128::from(shape.experts)); - let remote: u128 = owned[1..].iter().map(|&o| u128::from(o)).sum(); - let bytes = u128::from(layers * 4 * shape.n_embd) * k * remote; - let nic = u128::from(GB10_NIC); - let decode = bytes * PS * 1000 / (u128::from(step_ns) * 1000 * nic * ex); let ffn = owned .iter() .enumerate() @@ -906,33 +1295,43 @@ fn nic_of(shape: Shape, owned: &[u64], step_ns: u64, s1: u64) -> u128 { .max() .unwrap_or(0); let one = u128::from(a) + sh + k * id; - let prompt = u128::from(s1) * one * bytes * 1000 / ((u128::from(a) * ex + ffn) * nic); - decode.max(prompt) + ( + u128::from(s1) * one * nic_bytes(shape, owned), + u128::from(a) * ex + ffn, + ) +} + +/// Node 0's NIC, in permille, on such a plan: against the decode step and +/// the prompt batch, the larger. +fn nic_of(shape: Shape, owned: &[u64], step_ns: u64, s1: u64) -> u128 { + let nic = u128::from(GB10_NIC); + let ex = u128::from(shape.experts); + let decode = nic_bytes(shape, owned) * PS * 1000 / (u128::from(step_ns) * 1000 * nic * ex); + let (num, den) = nic_prompt(shape, owned, s1); + decode.max(num * 1000 / (den * nic)) } /// Clause 5, `nic_not_max`: a plan whose NIC passes half a decode step or a -/// prompt batch names the node and its busy share, which is the node's -/// reported one and the busiest, exactly where criterion 5 fires on the -/// chosen plan at its split: node 0 takes the rows of the remotes' share of -/// the slots. At S₁ 3,000 that is V4.1 and GLM at N ≥ 2 but three of -/// V4.1's four cells at N = 2 (all but λ 25, A low, on the line at 500 ‰), -/// and never Qwen3.8, whose node 0 keeps the most ids; none at 1,000. -/// §9.8's table, from an even split, names N ≥ 3 for all three. The prompt -/// counts the split's own shares: a remote that owns every id gains the -/// prompt only Sh. A token's mean bytes stay a fraction: three ids a layer -/// at top-k 1, owned [1, 2], take 8/3 B a token, two thirds of a 4 B row, -/// and node 0's prompt share is 4/7, past the line. The decode line reads -/// the exact step: the same split of ids of 666,666,664 B read at 10¹⁰ B/s, -/// a NIC of 16 B/s and λ 2,000 ps take a step of 333,333,333,333⅓ ps, twice -/// node 0's NIC time of 166,666,666,666⅔ ps, so on the line. Paged, each -/// window's read and fill round down once, together: forty ids of -/// 649,500,000 B at top-k 1 read at 10¹⁹ B/s on two nodes of one id's room -/// each, at a hit rate of 999 ‰ and a miss's fill of 1,000 ps, split -/// [20, 20], take a step of 325.7 ps that reads 324 ps (a read of 64.95 ps -/// and a fill of 19/20 ps a window). A row of 647 values at 8 × 10¹² B/s -/// gives node 0 a NIC time of 161.75 ps, half of 323.5 ps, 2.2 ps under the -/// step and so past the 2 ps a one-layer step can read low: no node is -/// named. +/// prompt batch names the node and its busy share, which is the node's reported +/// one and the busiest, exactly in the derived cells where criterion 5 fires on +/// the chosen plan at its split: node 0 takes the rows of the remotes' share of +/// the slots. On the line the side is the one [`expected_sets`] pins: V4.1 at +/// N = 2, λ 25, A low fires at S₁ 3,000. A share of exactly one half reads +/// 500 ‰ and does not fire, at ≤ and not <, and one just past it reads 500 ‰ +/// and fires. The prompt counts the split's own shares: a remote that owns +/// every id gains the prompt only Sh. A token's mean bytes stay a fraction: +/// three ids a layer at top-k 1, owned [1, 2], take 8/3 B a token, two thirds +/// of a 4 B row, and node 0's prompt share is 4/7, past the line. The decode +/// line reads the exact step: the same split of ids of 666,666,664 B read at +/// 10¹⁰ B/s, a NIC of 16 B/s and λ 2,000 ps take a step of +/// 333,333,333,333⅓ ps, twice node 0's NIC time of 166,666,666,666⅔ ps, so +/// on the line. Paged, each window's read and fill round down once, together: +/// forty ids of 649,500,000 B at top-k 1 read at 10¹⁹ B/s on two nodes of one +/// id's room each, at a hit rate of 999 ‰ and a miss's fill of 1,000 ps, split +/// [20, 20], take a step of 325.7 ps that reads 324 ps (a read of 64.95 ps and a +/// fill of 19/20 ps a window). A row of 647 values at 8 × 10¹² B/s gives node 0 +/// a NIC time of 161.75 ps, half of 323.5 ps, 2.2 ps under the step and so past +/// the 2 ps a one-layer step can read low: no node is named. #[test] fn nic_not_max() { for rate in PROMPT_RATES { @@ -955,9 +1354,7 @@ fn nic_not_max() { } None => assert!(busiest <= 500, "{cell:?}: {:?}", p.per_node), } - let below = - cell.0 == "V4.1" && cell.1 == 2 && cell.2 == 70 || cell == ("V4.1", 2, 25, true); - if rate == 3_000 && cell.1 >= 2 && cell.0 != "Qwen3.8" && !below { + if derived_at(cell, RULER, rate).fires { expected.push(cell); } }); @@ -995,6 +1392,27 @@ fn nic_not_max() { p.per_node[0].nic_permille ); assert_eq!(plan.nic_bound().expect("a prediction"), None); + let (_, den) = nic_prompt(shape, plan.owned(), 1); + let s1 = u64::try_from(den).expect("an S₁"); + let (num, _) = nic_prompt(shape, plan.owned(), s1); + let line = u64::try_from(2 * num / den).expect("a NIC rate"); + let at = |nic: u64| { + let mut nodes = spark_nodes(2, shape.layers, fabric(2, LAMBDAS[0])); + nodes.nic = nic; + let budgets = budgets_on(&model, &mut nodes, &machines); + let whole = Workload { + prompt_rate: s1, + ..whole + }; + let plan = plan_nodes(&nodes, &terms, &budgets, whole).expect("two Sparks"); + let bound = plan.nic_bound().expect("a prediction"); + ( + plan.predict().expect("a prediction").per_node[0].nic_permille, + bound.map(|b| b.node), + ) + }; + assert_eq!(at(line), (500, None), "a NIC at exactly half"); + assert_eq!(at(line - 1), (500, Some(0)), "a NIC just past half"); let id = 1_000_000_000; let model = tiny(3, 1, id); let link = LinkCost { round_trip_ns: 0 }; @@ -1071,7 +1489,8 @@ fn nic_not_max() { /// slot, and their posts take at most 2 % of the layer. Also where k is /// under N − 1: one expert a token of seven, two ids a pool, on four Sparks /// the pools all need, where a token's handoff goes only to the node its -/// slot lands on, and the step and node 0's NIC are that rule's. +/// slot lands on, and the step and node 0's NIC are that rule's. In every +/// cell node 0's count is the derived cell's. #[test] fn messages_o_n() { let check = |cell: Cell, plan: &NodePlan<'_>| { @@ -1090,7 +1509,15 @@ fn messages_o_n() { "{cell:?}: {busiest} messages a layer" ); }; - each_cell(RULER, 3_000, check); + each_cell(RULER, PROMPT_RATES[1], |cell, plan| { + check(cell, plan); + let p = plan.predict().expect("a prediction"); + assert_eq!( + p.per_node[0].msgs_per_layer, + derived_at(cell, RULER, PROMPT_RATES[1]).msgs, + "{cell:?}: node 0's messages" + ); + }); let shape = Shape { name: "k1", experts: 7, @@ -1130,16 +1557,16 @@ fn messages_o_n() { ); } -/// Clause 7, `never_slower`: a node is named idle for its gain exactly in -/// §9.8's criterion 7 cells but Qwen3.8 at N = 4, λ 70, A low, where -/// criterion 2 names it first, and no chosen plan is slower than the N − 1 -/// plan on the same link: a pair for a ring of three, three switched Sparks -/// for four. The ruler is read from the machine: on one whose ruler is 4 %, -/// Qwen3.8 names node 1 idle at N = 2 where it gains under 8 %, at λ 70. -/// The gain is read on the exact steps: three ids of 5,000 B at top-k 2 -/// read at 10¹² B/s take 30,000 ps on one node and 27,777⁷⁄₉ ps on two, a -/// gain of 7.4 %, under a ruler of 5 % twice, and read at 10¹⁴ B/s 300 and -/// 277⁷⁄₉ ps, a gain of 7.41 %, under a ruler of 3.8 % twice. +/// Clause 7, `never_slower`: no chosen plan is slower than the N − 1 plan on +/// the same link: a pair for a ring of three, three switched Sparks for four. +/// [`expected_sets`] holds the nodes named idle for their gain to the derived +/// cells where criterion 7 fires. The ruler is read from the machine: on one +/// whose ruler is 4 %, Qwen3.8 names node 1 idle at N = 2 where it gains +/// under 8 %, at λ 70, and at `RULER` it does not. The gain is read on the +/// exact steps: three ids of 5,000 B at top-k 2 read at 10¹² B/s take +/// 30,000 ps on one node and 27,777⁷⁄₉ ps on two, a gain of 7.4 %, under a +/// ruler of 5 % twice, and read at 10¹⁴ B/s 300 and 277⁷⁄₉ ps, a gain of +/// 7.41 %, under a ruler of 3.8 % twice. #[test] fn never_slower() { let gain = |plan: &NodePlan<'_>| { @@ -1150,7 +1577,7 @@ fn never_slower() { let mut fired = Vec::new(); let mut slower = Vec::new(); each_cell(RULER, 3_000, |cell, plan| { - if gain(plan) { + if cell.0 == "Qwen3.8" && cell.1 == 2 && gain(plan) { fired.push(cell); } let (_, n, lambda, _) = cell; @@ -1172,15 +1599,7 @@ fn never_slower() { } }); let qwen = |n, lambda, high| ("Qwen3.8", n, lambda, high); - assert_eq!( - fired, - vec![ - qwen(4, 25, false), - qwen(4, 25, true), - qwen(3, 70, true), - qwen(4, 70, true), - ] - ); + assert_eq!(fired, vec![]); assert_eq!(slower, vec![]); let mut wide = Vec::new(); each_cell(4 * RULER, 3_000, |cell, plan| { diff --git a/crates/placement/tests/nodes_scale_cells.tsv b/crates/placement/tests/nodes_scale_cells.tsv new file mode 100644 index 000000000..73352603a --- /dev/null +++ b/crates/placement/tests/nodes_scale_cells.tsv @@ -0,0 +1,146 @@ +# nodes_scale's expected cells (docs/contrib/dgx-spark.md §9.9), written by tools/flow/nodes_scale.py --write from nodes_scale_inputs.tsv; do not edit. +model a n lambda_us ruler s1 used idle paged paged_nodes step_ns step_tol nic nic_tol fires msgs edge +V4.1 low 1 25 10 1000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 25 10 1000 2 - 62 11,51 30612002 1750 167 3 0 2 - +V4.1 low 3 25 10 1000 3 - 0 0,0,0 27747601 585 256 3 0 4 - +V4.1 low 4 25 10 1000 4 - 0 0,0,0,0 26265584 1510 312 3 0 6 - +V4.1 low 1 70 10 1000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 70 10 1000 2 - 62 28,34 31422423 339 149 3 0 2 - +V4.1 low 3 70 10 1000 3 - 0 0,0,0 28986815 904 227 3 0 4 - +V4.1 low 4 70 10 1000 4 - 0 0,0,0,0 27660087 2483 277 3 0 6 - +V4.1 low 1 25 10 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 25 10 3000 2 - 62 11,51 30612002 1750 501 5 1 2 nic +V4.1 low 3 25 10 3000 3 - 0 0,0,0 27747601 585 769 6 1 4 - +V4.1 low 4 25 10 3000 4 - 0 0,0,0,0 26265584 1510 936 6 1 6 - +V4.1 low 1 70 10 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 70 10 3000 2 - 62 28,34 31422423 339 446 5 0 2 - +V4.1 low 3 70 10 3000 3 - 0 0,0,0 28986815 904 680 5 1 4 - +V4.1 low 4 70 10 3000 4 - 0 0,0,0,0 27660087 2483 830 6 1 6 - +V4.1 low 1 25 40 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 25 40 3000 2 - 62 11,51 30612002 1750 501 5 1 2 nic +V4.1 low 3 25 40 3000 3 - 0 0,0,0 27747601 585 769 6 1 4 - +V4.1 low 4 25 40 3000 3 3:gain 0 0,0,0,0 27792333 575 766 6 1 4 - +V4.1 low 1 70 40 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 70 40 3000 2 - 62 28,34 31422423 339 446 5 0 2 - +V4.1 low 3 70 40 3000 3 - 0 0,0,0 28986815 904 680 5 1 4 - +V4.1 low 4 70 40 3000 3 3:gain 0 0,0,0,0 29024289 956 677 5 1 4 - +V4.1 high 1 25 10 1000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 25 10 1000 2 - 67 0,67 44522408 12728 163 2 0 2 - +V4.1 high 3 25 10 1000 3 - 0 0,0,0 40991983 1460 264 3 0 4 - +V4.1 high 4 25 10 1000 4 - 0 0,0,0,0 39358988 4479 314 3 0 6 - +V4.1 high 1 70 10 1000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 70 10 1000 2 - 67 0,67 45557256 3135 163 2 0 2 - +V4.1 high 3 70 10 1000 3 - 0 0,0,0 42448939 1081 239 3 0 4 - +V4.1 high 4 70 10 1000 4 - 0 0,0,0,0 40883420 3798 285 3 0 6 - +V4.1 high 1 25 10 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 25 10 3000 2 - 67 0,67 44522408 12728 489 4 0 2 - +V4.1 high 3 25 10 3000 3 - 0 0,0,0 40991983 1460 793 5 1 4 - +V4.1 high 4 25 10 3000 4 - 0 0,0,0,0 39358988 4479 943 5 1 6 - +V4.1 high 1 70 10 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 70 10 3000 2 - 67 0,67 45557256 3135 489 4 0 2 - +V4.1 high 3 70 10 3000 3 - 0 0,0,0 42448939 1081 718 5 1 4 - +V4.1 high 4 70 10 3000 4 - 0 0,0,0,0 40883420 3798 856 5 1 6 - +V4.1 high 1 25 40 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 25 40 3000 2 - 67 0,67 44522408 12728 489 4 0 2 - +V4.1 high 3 25 40 3000 3 - 0 0,0,0 40991983 1460 793 5 1 4 - +V4.1 high 4 25 40 3000 3 3:gain 0 0,0,0,0 41044095 1519 791 5 1 4 - +V4.1 high 1 70 40 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 70 40 3000 2 - 67 0,67 45557256 3135 489 4 0 2 - +V4.1 high 3 70 40 3000 3 - 0 0,0,0 42448939 1081 718 5 1 4 - +V4.1 high 4 70 40 3000 3 3:gain 0 0,0,0,0 42493096 1017 716 5 1 4 - +GLM low 1 25 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 25 10 1000 2 - 0 0,0 55158319 12570 184 3 0 2 - +GLM low 3 25 10 1000 3 - 0 0,0,0 50939235 878 282 3 0 4 - +GLM low 4 25 10 1000 4 - 0 0,0,0,0 48718290 8205 335 3 0 6 - +GLM low 1 70 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 70 10 1000 2 - 0 0,0 56206586 4233 177 3 0 2 - +GLM low 3 70 10 1000 3 - 0 0,0,0 52318199 749 259 3 0 4 - +GLM low 4 70 10 1000 4 - 0 0,0,0,0 50299739 4173 307 3 0 6 - +GLM low 1 25 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 25 10 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM low 3 25 10 3000 3 - 0 0,0,0 50939235 878 847 7 1 4 - +GLM low 4 25 10 3000 4 - 0 0,0,0,0 48718290 8205 1004 7 1 6 - +GLM low 1 70 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 70 10 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM low 3 70 10 3000 3 - 0 0,0,0 52318199 749 776 6 1 4 - +GLM low 4 70 10 3000 4 - 0 0,0,0,0 50299739 4173 920 7 1 6 - +GLM low 1 25 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 25 40 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM low 3 25 40 3000 2 2:gain 0 0,0,0 55158319 12570 551 6 1 2 - +GLM low 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 55180155 12094 551 6 1 2 - +GLM low 1 70 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 70 40 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM low 3 70 40 3000 2 2:gain 0 0,0,0 56206586 4233 531 5 1 2 - +GLM low 4 70 40 3000 2 2:gain,3:gain 0 0,0,0,0 56244746 4184 529 5 1 2 - +GLM high 1 25 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 25 10 1000 2 - 0 0,0 55158319 12570 184 3 0 2 - +GLM high 3 25 10 1000 3 - 0 0,0,0 50939235 878 282 3 0 4 - +GLM high 4 25 10 1000 4 - 0 0,0,0,0 48718290 8205 335 3 0 6 - +GLM high 1 70 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 70 10 1000 2 - 0 0,0 56206586 4233 177 3 0 2 - +GLM high 3 70 10 1000 3 - 0 0,0,0 52318199 749 259 3 0 4 - +GLM high 4 70 10 1000 4 - 0 0,0,0,0 50299739 4173 307 3 0 6 - +GLM high 1 25 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 25 10 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM high 3 25 10 3000 3 - 0 0,0,0 50939235 878 847 7 1 4 - +GLM high 4 25 10 3000 4 - 0 0,0,0,0 48718290 8205 1004 7 1 6 - +GLM high 1 70 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 70 10 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM high 3 70 10 3000 3 - 0 0,0,0 52318199 749 776 6 1 4 - +GLM high 4 70 10 3000 4 - 0 0,0,0,0 50299739 4173 920 7 1 6 - +GLM high 1 25 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 25 40 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM high 3 25 40 3000 2 2:gain 0 0,0,0 55158319 12570 551 6 1 2 - +GLM high 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 55180155 12094 551 6 1 2 - +GLM high 1 70 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 70 40 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM high 3 70 40 3000 2 2:gain 0 0,0,0 56206586 4233 531 5 1 2 - +GLM high 4 70 40 3000 2 2:gain,3:gain 0 0,0,0,0 56244746 4184 529 5 1 2 - +Qwen3.8 low 1 25 10 1000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 25 10 1000 2 - 0 0,0 21798149 223 112 2 0 2 - +Qwen3.8 low 3 25 10 1000 3 - 0 0,0,0 20955294 226 155 2 0 4 - +Qwen3.8 low 4 25 10 1000 3 3:gain 0 0,0,0,0 20997033 258 153 2 0 4 - +Qwen3.8 low 1 70 10 1000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 70 10 1000 2 - 0 0,0 22798306 1442 73 2 0 2 - +Qwen3.8 low 3 70 10 1000 3 - 0 0,0,0 22311790 1629 105 2 0 4 link:2:99956:78 +Qwen3.8 low 4 70 10 1000 2 2:link,3:link 0 0,0,0,0 22823812 1499 72 2 0 2 - +Qwen3.8 low 1 25 10 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 25 10 3000 2 - 0 0,0 21798149 223 335 3 0 2 - +Qwen3.8 low 3 25 10 3000 3 - 0 0,0,0 20955294 226 466 3 0 4 - +Qwen3.8 low 4 25 10 3000 3 3:gain 0 0,0,0,0 20997033 258 460 3 0 4 - +Qwen3.8 low 1 70 10 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 70 10 3000 2 - 0 0,0 22798306 1442 220 3 0 2 - +Qwen3.8 low 3 70 10 3000 3 - 0 0,0,0 22311790 1629 315 3 0 4 link:2:99956:78 +Qwen3.8 low 4 70 10 3000 2 2:link,3:link 0 0,0,0,0 22823812 1499 217 3 0 2 - +Qwen3.8 low 1 25 40 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 25 40 3000 2 - 0 0,0 21798149 223 335 3 0 2 - +Qwen3.8 low 3 25 40 3000 2 2:gain 0 0,0,0 21798149 223 335 3 0 2 - +Qwen3.8 low 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 21835515 259 331 3 0 2 - +Qwen3.8 low 1 70 40 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 70 40 3000 1 1:gain 0 0,0 24239786 2 0 2 0 0 - +Qwen3.8 low 3 70 40 3000 1 1:gain,2:gain 0 0,0,0 24239786 2 0 2 0 0 - +Qwen3.8 low 4 70 40 3000 1 1:gain,2:gain,3:gain 0 0,0,0,0 24239786 2 0 2 0 0 - +Qwen3.8 high 1 25 10 1000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 25 10 1000 2 - 0 0,0 26587556 223 109 2 0 2 - +Qwen3.8 high 3 25 10 1000 3 - 0 0,0,0 25744701 226 150 2 0 4 - +Qwen3.8 high 4 25 10 1000 3 3:gain 0 0,0,0,0 25786439 258 148 2 0 4 - +Qwen3.8 high 1 70 10 1000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 70 10 1000 2 - 0 0,0 27587713 1442 72 2 0 2 - +Qwen3.8 high 3 70 10 1000 2 2:gain 0 0,0,0 27587713 1442 72 2 0 2 - +Qwen3.8 high 4 70 10 1000 2 2:gain,3:gain 0 0,0,0,0 27613219 1499 71 2 0 2 - +Qwen3.8 high 1 25 10 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 25 10 3000 2 - 0 0,0 26587556 223 327 3 0 2 - +Qwen3.8 high 3 25 10 3000 3 - 0 0,0,0 25744701 226 449 3 0 4 - +Qwen3.8 high 4 25 10 3000 3 3:gain 0 0,0,0,0 25786439 258 444 3 0 4 - +Qwen3.8 high 1 70 10 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 70 10 3000 2 - 0 0,0 27587713 1442 217 3 0 2 - +Qwen3.8 high 3 70 10 3000 2 2:gain 0 0,0,0 27587713 1442 217 3 0 2 - +Qwen3.8 high 4 70 10 3000 2 2:gain,3:gain 0 0,0,0,0 27613219 1499 213 3 0 2 - +Qwen3.8 high 1 25 40 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 25 40 3000 2 - 0 0,0 26587556 223 327 3 0 2 - +Qwen3.8 high 3 25 40 3000 2 2:gain 0 0,0,0 26587556 223 327 3 0 2 - +Qwen3.8 high 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 26624922 259 323 3 0 2 - +Qwen3.8 high 1 70 40 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 70 40 3000 1 1:gain 0 0,0 29029193 2 0 2 0 0 - +Qwen3.8 high 3 70 40 3000 1 1:gain,2:gain 0 0,0,0 29029193 2 0 2 0 0 - +Qwen3.8 high 4 70 40 3000 1 1:gain,2:gain,3:gain 0 0,0,0,0 29029193 2 0 2 0 0 - diff --git a/crates/placement/tests/nodes_scale_inputs.tsv b/crates/placement/tests/nodes_scale_inputs.tsv new file mode 100644 index 000000000..b16b4cbd3 --- /dev/null +++ b/crates/placement/tests/nodes_scale_inputs.tsv @@ -0,0 +1,53 @@ +# nodes_scale's inputs (docs/contrib/dgx-spark.md §9.9): the rows tools/flow/nodes_scale.py derives the gate's cells from, and the +# rows the gate holds its own constants and budgets to. A model's rows are /. kind: measured (a recorded run), +# derived (arithmetic on recorded numbers, code or datasheets), assumed (no record). Every value is a whole number. +name value unit kind source +read 208800000000 B/s measured BW, our kernels' pool read on one Spark: §5 and §9.7 +nic 23700000000 B/s derived a Spark's NIC a direction, the SoC side: §9.1 +pool 115284164096 B derived a node's pool: a GB10 the census reads with 115 GB free (§9.7 U), as NodeBudget::pool reads its card plan +lambda_lo 25000 ns derived λ's band, the exposed round trip a layer: §9.7 +lambda_hi 70000 ns derived λ's band: §9.7 +hop 1500 ns assumed the switch's hop at N = 4: §9.7 +post 1000 ns assumed a further post of node 0's pump: §9.7, §9.8 criterion 6 +row_value 4 B derived a value of the handoff and the rows, one f32 +vocab 151552 rows assumed the synthetic shapes' head rows +ruler 10 permille derived the box's 95 % interval at four rounds: AGENTS.md, "Know the ruler" +ruler_wide 40 permille assumed a machine whose ruler is 4 %: §9.9 M7 +s1_lo 1000 tok/s assumed S₁ at the low end of §9.8's pins +s1_hi 3000 tok/s assumed S₁ at the high end of §9.8's pins +V4.1/layers 40 layers derived §9.7 model terms +V4.1/n_embd 5120 values derived §9.7 model terms +V4.1/experts 384 experts derived §9.7 model terms +V4.1/top_k 6 slots derived §9.7 model terms +V4.1/expert 16770000 B derived §9.7 model terms, an expert's bytes +V4.1/shared_lo 800000000 B derived §9.7 Sh, the band's low end +V4.1/shared_hi 1500000000 B derived §9.7 Sh, the band's high end +V4.1/serial_lo 3200000000 B derived §9.7 A, the band's low end +V4.1/serial_hi 5800000000 B derived §9.7 A, the band's high end +V4.1/fixed0_lo 12286314624 B derived node 0's bytes beside the routed ids at A low: NodeBudget::fixed of the shape's card plan, KV 2 GB at 4,096 positions +V4.1/fixed0_hi 15555949824 B derived the same at A high +V4.1/fixed1 1744456576 B derived a remote's bytes beside its routed ids: NodeBudget::fixed of the routed stacks' card plan, no KV +GLM/layers 42 layers derived §9.7 model terms +GLM/n_embd 4096 values derived §9.7 model terms +GLM/experts 288 experts derived §9.7 model terms +GLM/top_k 8 slots derived §9.7 model terms +GLM/expert 15200000 B derived §9.7 model terms, an expert's bytes +GLM/shared_lo 1600000000 B derived §9.7 Sh, the UD-Q4_K_XL header read +GLM/shared_hi 1600000000 B derived §9.7 Sh, the UD-Q4_K_XL header read +GLM/serial_lo 7360000000 B derived §9.7 A, the UD-Q4_K_XL header read +GLM/serial_hi 7360000000 B derived §9.7 A, the UD-Q4_K_XL header read +GLM/fixed0_lo 23768122598 B derived node 0's bytes beside the routed ids: NodeBudget::fixed of the shape's card plan, KV 2 GB at 4,096 positions +GLM/fixed0_hi 23768122598 B derived the same at A high +GLM/fixed1 1747080960 B derived a remote's bytes beside its routed ids: NodeBudget::fixed of the routed stacks' card plan, no KV +Qwen3.8/layers 48 layers derived §9.7 model terms +Qwen3.8/n_embd 2560 values derived §9.7 model terms +Qwen3.8/experts 512 experts derived §9.7 model terms +Qwen3.8/top_k 10 slots derived §9.7 model terms +Qwen3.8/expert 3170000 B derived §9.7 model terms, an expert's bytes +Qwen3.8/shared_lo 140000000 B derived §9.7 Sh +Qwen3.8/shared_hi 140000000 B derived §9.7 Sh +Qwen3.8/serial_lo 3400000000 B derived §9.7 A, the band's low end +Qwen3.8/serial_hi 4400000000 B derived §9.7 A, the band's high end +Qwen3.8/fixed0_lo 8341707616 B derived node 0's bytes beside the routed ids at A low: NodeBudget::fixed of the shape's card plan, KV 2 GB at 4,096 positions +Qwen3.8/fixed0_hi 9348340576 B derived the same at A high +Qwen3.8/fixed1 1698955264 B derived a remote's bytes beside its routed ids: NodeBudget::fixed of the routed stacks' card plan, no KV diff --git a/docs/contrib/dgx-spark.md b/docs/contrib/dgx-spark.md index 42bf11107..ac50e55b7 100644 --- a/docs/contrib/dgx-spark.md +++ b/docs/contrib/dgx-spark.md @@ -749,7 +749,9 @@ now written beside them are not fed back into the tables. - The (a) figures come from a Monte Carlo of the per-layer window: each routed slot lands on node j with the ownership probability the rule picks, and the window is the max over the nodes of `Sh + slots·e` (node 0) or `λ + hop + posts + rows ingress + slots·e` (a remote). Node 0's share is optimized: 0.55–0.72 at N = 2 for Qwen3.8 and 0.07–0.15 at N = 4 - for GLM and V4.1. The script moves into `tools/` beside the gate (9.9). + for GLM and V4.1. The script moves into `tools/` beside the gate (9.9). The gate's cells and the (a) rows at the + gate's inputs below come from `tools/flow/nodes_scale.py` instead, which sums over the count tuples of a token's slots + rather than sampling them (9.9). **Model terms** `[derived]` from each model's tensor table (file sizes: `docs/models.md`): @@ -784,10 +786,19 @@ routed share] → join`. The next layer's attention needs the join, so node 1 id | N | V4.1 (needs about 268 GB resident) | GLM (about 203 GB) | Qwen3.8 (about 114 GB) | |---|---|---|---| -| 1 | no: about 154 GB of routed on NVMe | no: about 88 GB on NVMe | yes, marginal | -| 2 | no: about 43 GB on NVMe ((b): about 19 GB a node) | yes, about 25 GB spare | yes | +| 1 | no: ~~about 154 GB of routed on NVMe~~ node 0's pool has room for 154 of the 384 routed ids at 4,096 positions and 153 at 32,768, so 230 (155.0 GB) and 231 (155.7 GB) do not fit | no: ~~about 88 GB on NVMe~~ 128 of 288 ids on NVMe, 82.4 GB | yes, marginal | +| 2 | no: about 43 GB on NVMe ((b): about 19 GB a node) | yes, ~~about 25 GB spare~~ every id held: the pools have room for 160 and 176 of the 288 ids | yes | | 4 | yes, about 190 GB spare ((b) about 67 GB a stage; (c) adds 6 GB of dense a node) | yes ((c) adds 11.4 GB of dense a node) | yes | +The cells that replace the struck figures are `[derived]` from the files' headers (V4.1 `Q3_K_M`, GLM `UD-Q4_K_XL`) +at `cb5688c9`, with node 0's card plan of the file and a pool a node of the 115 GB that 9.7 assumes, the same at 4,096 +and 32,768 positions where no position count is named. GLM's are the plans of `plan_nodes` +([#95](https://github.com/midagedev/bloomery/pull/95)). The node plans refuse the V4.1 file at every N: node 0's card +plan leaves its engram tables (84.5 GB) on the NVMe tier and reads 5,280 B of them a token, and `Terms::of` refuses +such rows by name (9.9, Terms). V4.1's cell at N = 1 comes from the room node 0's budget (`NodeBudget::of`) leaves for +routed ids: the pool the plan reads from those 115 GB, 115.28 GB, less its fixed bytes (10.86 GB at 4,096 positions, +12.14 GB at 32,768), over 673.9 MB an id. + For Qwen3.8 at N = 2 under (a), node 0 holds about 75 GB (4.5 non-routed + 28.8 PLE + about 39 routed + KV) against about 114 GB on one node, and node 1 about 40 GB. @@ -839,6 +850,26 @@ that binds:** | (b) pipeline | one boundary (16·n_embd B: 82, 66, 41 KB) and a token id | N − 1 boundaries and a token id | spread: each NIC one in, one out | | (c) EP | as (a) with uniform ownership | as (a) | spread: each node at most 4(N−1) a layer | +**(a) at the gate's inputs** `[derived]` by `tools/flow/nodes_scale.py --table` (9.9): BW 208.8 GB/s, the 23.7 GB/s +NIC, a 115 GB pool a node, the gate's synthetic shapes (GLM's A and Sh from the header read), λ from 25 to 70 µs and A +from low to high, one request. N is the Sparks offered; the plan may leave some of them idle (criteria 2 and 7 of 9.8), +and node 0's share is the split's, not an even one. Decode carries no NVMe term, so for a model that pages it is the +pool's time only. The prompt is the Amdahl term above with the slowest node's FFN at the split, and the NIC cap is node +0's rows ingress at the split's remote share. Messages count a handoff and its rows for each remote a token reaches. + +| (a) star at the gate's inputs | N | Nodes used | Decode, one request (tok/s) | Prompt (× S₁) | Node 0's NIC ingress cap (tok/s) | Link bytes a token (MB) | Messages a token | +|---|---|---|---|---|---|---|---| +| V4.1 | 2 | 2 | 22.0 to 32.7 (pool only: pages 62 to 67 ids) | 1.28 to 1.40 | 7,846 to 9,140 | 3.40 to 3.84 | 79 to 80 | +| V4.1 | 4 | 4 | 24.5 to 38.1 | 1.49 to 1.75 | 4,895 to 6,031 | 6.00 to 7.07 | 203 to 218 | +| GLM | 2 | 2 | 17.8 to 18.1 | 1.28 to 1.29 | 7,005 to 7,216 | 3.97 to 4.07 | 84 | +| GLM | 4 | 4 | 19.9 to 20.5 | 1.48 to 1.52 | 4,551 to 4,825 | 6.85 to 7.17 | 237 to 240 | +| Qwen3.8 | 2 | 2 | 36.2 to 45.9 | 1.09 to 1.16 | 10,396 to 15,055 | 2.06 to 2.77 | 94 to 96 | +| Qwen3.8 | 4 | 2 to 3 | 36.2 to 47.6 | 1.09 to 1.22 | 7,968 to 15,269 | 2.03 to 3.93 | 94 to 187 | + +The rest of the decode, prompt and link tables above is not recomputed. The one-node and NVMe rows need ρ and the +drive's rate, (b) a pipeline model, (c) the union R(2), and N requests in flight a batched step; none of them is an +input of the script (section 10). + **One GLM decode layer under (a), one request, N = 2 (midpoints), the resource timeline:** | Resource | Busy | Note | @@ -931,6 +962,22 @@ Nothing fires at S₁ = 1,000. At N = 2 nothing fires except GLM's prompt, which 8,610): the gate's boundary case, at ≤ and not <. The predictor in the tree reads the measured BW and node 0's real split, and differs from this table in criterion 5's column; 9.9 gives its cells. +**Where each fires at the gate's inputs** `[derived]` by `tools/flow/nodes_scale.py --table` (9.9): BW 208.8 GB/s, the +23.7 GB/s NIC, a 115 GB pool a node, GLM's header read, node 0's NIC at the split's own share, and the gate's `RULER`, +10 ‰ (question 1 of [#95](https://github.com/midagedev/bloomery/pull/95) asks whether that is the box's ruler). An N +alone names every λ and A at it. + +| Model | Decode step (ms) at the gate's inputs, N = 1 → 2 → 3 → 4, λ 25 / 70, A low | Criterion 2 fires | Criterion 7 fires | Criterion 5 fires, S₁ 1,000 | Criterion 5 fires, S₁ 3,000 | +|---|---|---|---|---|---| +| V4.1 | 38.4 → 30.6 / 31.4 → 27.7 / 29.0 → 26.3 / 27.7 | never | never | never | N = 2, λ 25, A low; N = 3; N = 4 | +| GLM | 67.4 → 55.2 / 56.2 → 50.9 / 52.3 → 48.7 / 50.3 | never | never | never | N = 2; N = 3; N = 4 | +| Qwen3.8 | 24.2 → 21.8 / 22.8 → 21.0 / 22.3 → 21.0 / 22.8 | N = 4, λ 70, A low | N = 3, λ 70, A high; N = 4, λ 25, A low and high; N = 4, λ 70, A high | never | never | + +Two cells sit on a line within the change one id moved makes. V4.1 at N = 2, λ 25, A low has its busiest NIC at +501 ± 5 ‰ at S₁ 3,000, on the 500 ‰ line, and criterion 5 fires there. Qwen3.8 at N = 3, λ 70, A low has its third +node's link share at 99,956 ± 78 ppm, on the tenth, and the node is used. The cells file names each as an edge in every +row of a ruler and S₁ it is derived at, four rows in all, and the gate pins its side (9.9). + ### 9.9 Reference: the gate, `nodes_scale` `nodes_scale` is the placement half's gate (answer 20), built with the shape of 9.2. It is a set of unit tests in @@ -951,8 +998,18 @@ Where the gate holds a clause differently from its words below, the clause says bytes the split holds them at. A row table left on the NVMe tier is refused by name, since the predictor has no fill term for it. The row width is an input, the model's embedding length. - **Expected sets.** The table of 9.8, pinned from a derivation script that moves into `tools/` beside the gate with its - own self-test, so the pins have one derivation. The script is not in the tree yet. Until it is, the gate pins the - predictor's own cells, which match 9.8's table in criteria 2 and 7 and differ in criterion 5 (clause 5). + own self-test, so the pins have one derivation. ~~The script is not in the tree yet. Until it is, the gate pins the + predictor's own cells, which match 9.8's table in criteria 2 and 7 and differ in criterion 5 (clause 5).~~ The script + is `tools/flow/nodes_scale.py`, with no code or method shared with the planner: node 0's share in real numbers by + 9.2's rule, found by bisection, and each window an expectation over the count tuples of a token's slots, built one + slot at a time. It reads `crates/placement/tests/nodes_scale_inputs.tsv`, one row per input with its kind (measured, + derived or assumed) and source, and writes the 144 cells (a model, N, λ and A, at the gate's `RULER` and S₁ 1,000 and + 3,000 and at a 4 % ruler and S₁ 3,000) to `nodes_scale_cells.tsv` beside it; `--table` prints 9.8's table at the + gate's inputs and 9.7's (a) rows, and `--check` holds the file and both doc tables to them. Each value carries the + largest change one id moved between two used nodes makes, because the planner splits whole ids. Where that band holds + a line (the NIC's half, the link share's tenth, twice the ruler), the cell names an edge, the gate pins its side by + hand, and the planner's value is held to the band besides. `--self-test` runs hand-worked cases, the refusals and + `--check`; `just check-recipes` runs it. - **Clauses:** 1. `n1_is_today`: every N = 1 plan equals today's plan byte for byte. The gate holds it where node 0 holds every id: node 0's bytes are today's pool need, or both plans refuse the model by the same bytes. Paged, it holds node 0 to @@ -965,11 +1022,12 @@ Where the gate holds a clause differently from its words below, the clause says 4. `link_share`: criterion 2 holds on the chosen plan. Qwen3.8 at N = 4, λ 70, A low uses at most 3 nodes. The predictor's plan of that cell uses two of the four: a third switched node exposes 2.270 of 22.351 ms, 10.2 %. 5. `nic_not_max`: criterion 5 either holds or carries the named prompt-mode note, exactly in the cells of the "fires" - column. The gate pins the predictor's cells: at S₁ 3,000, V4.1 and GLM at N ≥ 2, except three of V4.1's four cells - at N = 2, and never Qwen3.8; at 1,000, none. Node 0 takes the rows of the remotes' share as the split sets it, - where 9.8 assumed an even split: GLM at N = 2, λ 25, A low splits its ids [111, 177], and node 0's NIC is busy - 551 ‰. In the cells that fire, the busiest NIC runs from 500 to 941 ‰ for V4.1 and from 531 to 1,007 ‰ for GLM; - Qwen3.8's stays at or below 465 ‰. + column. The gate pins the ~~predictor's~~ derived cells: at S₁ 3,000, V4.1 and GLM at N ≥ 2, except three of V4.1's + four cells at N = 2, and never Qwen3.8; at 1,000, none. The figures that follow are the predictor's; the gate holds + each busiest NIC to the derived one within one id's change. Node 0 takes the rows of the remotes' share as the + split sets it, where 9.8 assumed an even split: GLM at N = 2, λ 25, A low splits its ids [111, 177], and node 0's + NIC is busy 551 ‰. In the cells that fire, the busiest NIC runs from 500 to 941 ‰ for V4.1 and from 531 to 1,007 ‰ + for GLM; Qwen3.8's stays at or below 465 ‰. 6. `messages_o_n`: criterion 6. The busiest node posts 2 × min(k, N − 1) messages a layer, because node 0 sends the handoff only to the remotes that hold one of the token's slots in that layer. 7. `never_slower`: `NodeIdle` names exactly the cells of criterion 7's column, and no chosen plan is slower than its @@ -1015,10 +1073,14 @@ Each open item; the owner is named where one is agreed. An item that a measureme **P5.** - The `[derived]` bands and criteria of 9.7 and 9.8 still use their first inputs. `nodes_scale`'s derivation script, in the placement half, recomputes them (the placement half as proposed in [#13](https://github.com/midagedev/bloomery/issues/13#issuecomment-6134316241), accepted in - answer 20). + answer 20). `tools/flow/nodes_scale.py` (9.9) recomputes 9.8's "where each fires" table and 9.7's (a) rows at the + gate's inputs. 9.7's one-node and NVMe rows, (b), (c) and N requests in flight still use their first inputs: they need + ρ, the NVMe rate, a pipeline model, the union R(2) and a batched step, none of which is an input of the script. +- A fill term in the node plans for rows read from the NVMe tier, so that they can plan the V4.1 file (9.7, 9.9 + Terms). - Where `nodes_scale` ([#95](https://github.com/midagedev/bloomery/pull/95)) holds a clause of 9.9 differently from its - words, for the maintainers to accept or change: until the derivation script lands, it pins the predictor's cells, - which differ from 9.8's table in criterion 5 (clause 5); clause 1 is byte for byte only where node 0 holds every id, + words, for the maintainers to accept or change: it pins the cells `tools/flow/nodes_scale.py` derives, which differ from + 9.8's table as first written in criterion 5 (clause 5); clause 1 is byte for byte only where node 0 holds every id, since today's plan of one Spark pages V4.1 and GLM under a card budget lowered to the NVMe tier's floor (`plan_in_pool`, section 8) and the node plans take node 0's terms before that split; clause 7 compares each plan with the N − 1 plan on the same link, which the planner can choose, not on the N − 1 plan's own fabric. diff --git a/tools/check-recipes.sh b/tools/check-recipes.sh index 0a0e38a51..d1cbb6168 100755 --- a/tools/check-recipes.sh +++ b/tools/check-recipes.sh @@ -473,6 +473,7 @@ blk_pytools() { "tools/check-ptx-names.py --self-test" "tools/check-records.py --self-test" "tools/flow/ds41_prefill.py --self-test" + "tools/flow/nodes_scale.py --self-test" "tools/flow/pplb.py --self-test" "tools/flow/q38_step.py --self-test" "tools/flow/q38width.py --self-test" diff --git a/tools/flow/nodes_scale.py b/tools/flow/nodes_scale.py new file mode 100755 index 000000000..603eea1b7 --- /dev/null +++ b/tools/flow/nodes_scale.py @@ -0,0 +1,751 @@ +#!/usr/bin/env python3 +"""The expected cells of `nodes_scale` (docs/contrib/dgx-spark.md §9.9), derived apart from the planner. + +What it derives. For each synthetic shape the gate plans (§9.7's terms for V4.1, GLM-5.3-Flash and Qwen3.8, A low and +high), on N = 1 to 4 Sparks (a pair, a ring of three, four on a switch), at λ 25 and 70 µs, under the gate's ruler and a +4 % one, and at S₁ 1,000 and 3,000 tok/s: the nodes the plan uses and why it leaves each other node idle (§9.8 criteria +1, 2 and 7), the routed ids the pools cannot hold, in all and per node (criterion 1), the decode step, node 0's messages +in the layer that reaches the most remotes (criterion 6) and the busiest NIC with whether criterion 5 fires. From the +cells it prints §9.8's "where each fires" table and §9.7's (a) rows at the gate's inputs, as the doc carries them. + +How, so that the cells do not repeat the planner's method (crates/placement/src/placement/nodes.rs): +- The split is §9.2's rule in real shares: node 0 takes the share at which its mean window equals a remote's, the + remotes share the rest equally, and each node is held to its pool's room where the pools hold every id, or keeps at + least its room where they do not. Bisection on node 0's share; the planner instead places whole ids one at a time. +- A layer's windows are expectations over the count tuples of a token's k slots, whose probabilities are built one slot + at a time: each slot moves a tuple's probability to the tuples one count higher, times the node's share (§9.7: "each + routed slot lands on node j with the ownership probability"). The planner sums closed-form multinomials instead. +- Floats with the four basic operations only, so every host writes the same file. + +The formulas and their sources: +- A layer (§9.7 (a), "the window is the max over the nodes of `Sh + slots·e` (node 0) or `λ + hop + posts + rows + ingress + slots·e` (a remote)"): node 0 reads the shared bytes and its slots' experts; a remote that a slot reaches + waits for the round trip (λ, plus the hop on a switch), a post for each other remote reached and the rows of the + other remotes' slots through node 0's NIC, then reads its slots' experts; the layer waits for the slowest. The step + is A plus every layer (§9.7 (a): "t = A/BW + max(...)/BW"). +- Criterion 2 (§9.8): a plan's link share is its step less the step of the same nodes split with no link term; a node + is used where the share stays within a tenth of the step. +- Criterion 7 (§9.8): a node past the pools that already hold every id is used where it gains at least twice the ruler. + Criterion 1: a node the pools need is used; a remote whose own bytes pass its pool holds nothing and is idle for its + pool; node 0's bytes past its pool refuse every plan. +- Criterion 5 (§9.8, "busy at most 50 % of the decode step and of the predicted prompt batch"): a node's NIC is busy, a + direction, by the rows of the slots on the remotes, one row of n_embd values a slot (§9.7's "rows ingress"): node 0 + takes them all in, a remote sends its own. The handoffs go the other way, one a remote a token reaches, and never + outnumber the rows, since a remote reached holds at least one slot. The prompt batch runs at + §9.7's Amdahl term at the split: S₁ times the one-node token's bytes, A + Sh + R, over A and the slowest node's FFN + bytes, node 0's with Sh. At an even split with no Sh that is 9.7's `1/(1−φ+φ/N)`. The planner states the same + definition; it is shared on purpose, as the criterion it pins. +- Criterion 6 (§9.8, "at most 2·min(k, N−1) for (a)"): node 0 posts a handoff and takes the rows of each remote, so + the layer that reaches the most remotes carries 2 × min(k, remotes used). Shared with the planner on purpose. + +What it takes as given: the rows of crates/placement/tests/nodes_scale_inputs.tsv, among them each node's bytes beside +the routed ids (fixed0, fixed1), which today's card plan sets and the gate reads back from it, and the shapes' rounding +to whole Q4_K rows of n_embd values (`shape`, the gate's `Shape::bytes`: the shapes are the gate's inputs). + +Tolerances and edges. The planner splits whole ids, so each value carries the largest change one id moved makes: +between node 0 and any single remote either way, and between two remotes, within each node's bounds (`*_tol`; a node's +paged ids carry one id). A cell whose NIC band holds the 50 % line, whose link share's band holds the tenth, or whose +gain's band holds twice the ruler is named in `edge` (`nic`, `link:::`, `gain:::`), and its side is the derivation's value; the gate pins each edge's side by name and holds the planner's value to +the band besides. + + nodes_scale.py print the cells + nodes_scale.py --write write crates/placement/tests/nodes_scale_cells.tsv + nodes_scale.py --check exit 1 when that file differs from the cells, or the doc's tables from --table + nodes_scale.py --table §9.8's "where each fires" table and §9.7's (a) rows at the gate's inputs + nodes_scale.py --self-test hand-worked cases, the refusals, and --check + +Refused by name (exit 2): an inputs row of another cell count, a name twice, a kind outside measured / derived / +assumed, a value that is not a whole number in ASCII digits, a missing or unknown row, an input outside its domain +(top-k outside 1 to the experts, a ruler outside 1 to 1000 permille, a rate, a pool, layers or a row size of 0, a term +under one row), and node 0's bytes past its pool. Exit 64: usage. + +Python 3 standard library only. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +TESTS = os.path.join(HERE, "..", "..", "crates", "placement", "tests") +INPUTS = os.path.join(TESTS, "nodes_scale_inputs.tsv") +CELLS = os.path.join(TESTS, "nodes_scale_cells.tsv") +DOC = os.path.join(HERE, "..", "..", "docs", "contrib", "dgx-spark.md") + +KINDS = ("measured", "derived", "assumed") +MACHINE = ("read", "nic", "pool", "lambda_lo", "lambda_hi", "hop", "post", "row_value", "vocab", "ruler", + "ruler_wide", "s1_lo", "s1_hi") +TERMS = ("layers", "n_embd", "experts", "top_k", "expert", "shared_lo", "shared_hi", "serial_lo", "serial_hi", + "fixed0_lo", "fixed0_hi", "fixed1") +COLUMNS = ("model", "a", "n", "lambda_us", "ruler", "s1", "used", "idle", "paged", "paged_nodes", "step_ns", "step_tol", + "nic", "nic_tol", "fires", "msgs", "edge") +NS = 1e9 +# Each band's slack in whole units (ns, ‰, ppm): the band is truncated (under 1), the value rounded to the nearest (½), +# and the planner's value rounded down from sums of whole picoseconds (under 1 and a fraction): under 3, so 2 whole. +SLACK = 2 + + +class Refused(Exception): + pass + + +def parse(text, path): + """The inputs file's rows as (name to int, models in file order).""" + rows, models, header = {}, [], None + for line in text.splitlines(): + if line.startswith("#") or not line.strip(): + continue + cells = line.split("\t") + if header is None: + header = cells + if header != ["name", "value", "unit", "kind", "source"]: + raise Refused(f"{path}: the header is {header}, not name, value, unit, kind, source") + continue + if len(cells) != 5: + raise Refused(f"{path}: {cells[0]!r} has {len(cells)} cells, the header 5") + name, value, _, kind, _ = cells + if name in rows: + raise Refused(f"{path}: {name} twice") + if kind not in KINDS: + raise Refused(f"{path}: {name}: kind {kind!r} is not one of {', '.join(KINDS)}") + if not (value.isascii() and value.isdigit()): + raise Refused(f"{path}: {name}: {value!r} is not a whole number") + rows[name] = int(value) + model = name.split("/")[0] if "/" in name else None + if model is not None and model not in models: + models.append(model) + if header is None: + raise Refused(f"{path}: no header row") + return rows, models + + +def inputs(text, path): + """The inputs as (machine, models): name to int, and model to term to int, models in file order.""" + rows, models = parse(text, path) + machine = {} + for k in MACHINE: + if k not in rows: + raise Refused(f"{path}: no {k} row") + machine[k] = rows[k] + terms = {} + for m in models: + terms[m] = {} + for t in TERMS: + if f"{m}/{t}" not in rows: + raise Refused(f"{path}: no {m}/{t} row") + terms[m][t] = rows[f"{m}/{t}"] + known = set(MACHINE) | {f"{m}/{t}" for m in models for t in TERMS} + for name in rows: + if name not in known: + raise Refused(f"{path}: {name} is no input") + domain(machine, terms, path) + return machine, terms + + +def domain(mc, terms, path): + """Refuses an input the model is undefined at, before anything is computed from it.""" + for k in ("read", "nic", "pool", "row_value", "s1_lo", "s1_hi"): + if mc[k] == 0: + raise Refused(f"{path}: {k} is 0") + for k in ("ruler", "ruler_wide"): + if not 1 <= mc[k] <= 1000: + raise Refused(f"{path}: {k} {mc[k]} is outside 1 to 1000 permille") + for m, t in terms.items(): + for k in ("layers", "experts"): + if t[k] == 0: + raise Refused(f"{path}: {m}/{k} is 0") + if not 1 <= t["top_k"] <= t["experts"]: + raise Refused(f"{path}: {m}/top_k {t['top_k']} is outside 1 to its {t['experts']} experts") + if t["n_embd"] < 256: + raise Refused(f"{path}: {m}/n_embd {t['n_embd']} is under one Q4_K block of 256 values") + row = t["n_embd"] // 256 * 144 + if t["expert"] < row: + raise Refused(f"{path}: {m}/expert {t['expert']} B is under one row of {row} B") + for end in ("lo", "hi"): + if t[f"serial_{end}"] // row < mc["vocab"]: + raise Refused(f"{path}: {m}/serial_{end} holds fewer rows than the head's {mc['vocab']}") + + +def load(): + try: + with open(INPUTS, encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise Refused(f"{INPUTS}: {e.strerror}") from None + return inputs(text, INPUTS) + + +def shape(mc, t, high): + """The bytes the gate's shape lays out, in whole Q4_K rows of n_embd values: A (its attention, the head and one + embedding row), a layer's Sh and one expert; and the rest of the case.""" + row = t["n_embd"] // 256 * 144 + layers = t["layers"] + end = "hi" if high else "lo" + attn = (t[f"serial_{end}"] // row - mc["vocab"]) // layers + return { + "layers": layers, "experts": t["experts"], "k": t["top_k"], "n_embd": t["n_embd"], + "serial": (attn * layers + mc["vocab"] + 1) * row, + "shared": t[f"shared_{end}"] // row // layers * row, + "expert": t["expert"] // row * row, + "fixed0": t[f"fixed0_{end}"], "fixed1": t["fixed1"], + } + + +def falls(s, k): + """The probability of each count tuple of a token's `k` slots on nodes of shares `s`, one slot at a time.""" + dist = {(0,) * len(s): 1.0} + for _ in range(k): + nxt = {} + for c, p in dist.items(): + for j, x in enumerate(s): + if x == 0.0: + continue + d = c[:j] + (c[j] + 1,) + c[j + 1:] + nxt[d] = nxt.get(d, 0.0) + p * x + dist = nxt + return dist + + +class Link: + """A remote's link terms a layer, in seconds: the round trip, a further post, one slot's rows through a NIC.""" + + def __init__(self, rt, post, row): + self.rt, self.post, self.row = rt, post, row + + +def layer(s, sh, e, k, link): + """A layer at shares `s` (node 0 first), in seconds: the expected slowest window, and each node's mean window.""" + n = len(s) + slowest, mean = 0.0, [0.0] * n + for c, p in falls(s, k).items(): + reached = sum(1 for cj in c[1:] if cj > 0) + remote = sum(c[1:]) + w = [sh + c[0] * e] + for cj in c[1:]: + w.append(link.rt + link.post * (reached - 1) + link.row * (remote - cj) + cj * e if cj > 0 else 0.0) + slowest += p * max(w) + for j in range(n): + mean[j] += p * w[j] + return slowest, mean + + +class Plan: + def __init__(self, s, step, tol): + self.s, self.step, self.tol = s, step, tol + + +class Case: + def __init__(self, mc, sh, n, lam): + self.mc, self.sh = mc, sh + read = mc["read"] + self.A = sh["serial"] / read + self.Sh = sh["shared"] / read + self.e = sh["expert"] / read + self.id = sh["layers"] * sh["expert"] + if sh["fixed0"] > mc["pool"]: + raise Refused(f"node 0's {sh['fixed0']} B beside the routed ids pass its pool of {mc['pool']} B") + self.rooms = ((mc["pool"] - sh["fixed0"]) // self.id, max(0, mc["pool"] - sh["fixed1"]) // self.id) + self.pool_idle = sh["fixed1"] > mc["pool"] + hop = mc["hop"] if n == 4 else 0 + self.link = Link((lam + hop) / NS, mc["post"] / NS, mc["row_value"] * sh["n_embd"] / mc["nic"]) + self.bare = Link(0.0, 0.0, 0.0) + + def room(self, m): + return self.rooms[0] + m * self.rooms[1] + + def limits(self, m): + """Each used node's share's bounds with `m` remotes: up to its room where the pools hold every id, from it + where they do not.""" + E = self.sh["experts"] + r = [min(1.0, self.rooms[0] / E)] + [min(1.0, self.rooms[1] / E)] * m + if self.room(m) >= E: + return [(0.0, x) for x in r] + return [(x, 1.0) for x in r] + + def shares(self, s0, m): + return [s0] + [(1.0 - s0) / m] * m if m else [1.0] + + def balance(self, m, link): + """§9.2's rule: node 0's share where its mean window equals a remote's, within the bounds.""" + if m == 0: + return 1.0 + b = self.limits(m) + lo, hi = max(b[0][0], 1.0 - m * b[1][1]), min(b[0][1], 1.0 - m * b[1][0]) + k = self.sh["k"] + + def diff(s0): + _, mean = layer(self.shares(s0, m), self.Sh, self.e, k, link) + return mean[0] - mean[1] + + if diff(lo) >= 0: + return lo + if diff(hi) <= 0: + return hi + for _ in range(64): + mid = (lo + hi) / 2 + if diff(mid) < 0: + lo = mid + else: + hi = mid + return (lo + hi) / 2 + + def step(self, s, link): + slowest, _ = layer(s, self.Sh, self.e, self.sh["k"], link) + return self.A + self.sh["layers"] * slowest + + def moves(self, s): + """The shares one id away from `s`: one id from any used node to any other, within each node's bounds.""" + d = 1.0 / self.sh["experts"] + b = self.limits(len(s) - 1) + out = [] + for i in range(len(s)): + for j in range(len(s)): + if i == j or s[i] - d < b[i][0] or s[j] + d > b[j][1]: + continue + t = list(s) + t[i] -= d + t[j] += d + out.append(t) + return out + + def plan(self, m, link): + """The split of `m` remotes: its shares, its step and the step's one-id tolerance.""" + s = self.shares(self.balance(m, link), m) + t = self.step(s, link) + return Plan(s, t, max([abs(self.step(x, link) - t) for x in self.moves(s)], default=0.0)) + + def nic(self, s, t, s1): + """The busiest node's NIC busy share, a fraction, of the plan at shares `s` and step `t`.""" + sh = self.sh + k, L, m = sh["k"], sh["layers"], len(s) - 1 + if m == 0: + return 0.0 + rows = [0.0] * (m + 1) + for c, p in falls(s, k).items(): + for j in range(1, m + 1): + rows[0] += p * c[j] + rows[j] += p * c[j] + row = self.mc["row_value"] * sh["n_embd"] + A, Sh, R = sh["serial"], sh["shared"] * L, k * self.id + ffn = max([Sh + s[0] * R] + [x * R for x in s[1:]]) + rate = s1 * (A + Sh + R) / (A + ffn) + busiest = 0.0 + for j in range(m + 1): + b = L * row * rows[j] / self.mc["nic"] + busiest = max(busiest, b / t, b * rate) + return busiest + + +def ppm(x): + return round(x * 1e6) + + +def busiest(c, p, m, s1): + """The plan's busiest NIC share, its one-id tolerance, and whether its band holds the 50 % line.""" + busy = c.nic(p.s, p.step, s1) + tol = max([abs(c.nic(x, c.step(x, c.link), s1) - busy) for x in c.moves(p.s)], default=0.0) + return busy, tol, m > 0 and abs(busy - 0.5) <= tol + + +def cell(mc, sh, n, lam, ruler): + """The plan of shape `sh` on `n` nodes at round trip `lam` ns: the case, m remotes used, idle [(node, why)], the + edges of the node choice, and the plan.""" + c = Case(mc, sh, n, lam) + E = sh["experts"] + m, idle, edges = 0, [], [] + now = c.plan(0, c.link) + for j in range(1, n): + if c.pool_idle: + idle.append((j, "pool")) + continue + nxt = c.plan(m + 1, c.link) + if c.room(m) < E and c.rooms[1] > 0: + m, now = m + 1, nxt + continue + bare = c.plan(m + 1, c.bare) + share = (nxt.step - bare.step) / nxt.step + share_tol = (nxt.tol + bare.tol) / nxt.step + if abs(share - 0.1) <= share_tol: + edges.append(f"link:{j}:{ppm(share)}:{int(share_tol * 1e6) + SLACK}") + if share > 0.1: + idle.append((j, "link")) + continue + gain = (now.step - nxt.step) / now.step + gain_tol = (now.tol + nxt.tol) / now.step + if abs(gain - 2 * ruler / 1000) <= gain_tol: + edges.append(f"gain:{j}:{ppm(gain)}:{int(gain_tol * 1e6) + SLACK}") + if gain < 2 * ruler / 1000: + idle.append((j, "gain")) + continue + m, now = m + 1, nxt + return c, m, idle, edges, now + + +def derive(mc, terms): + """Every cell, in the gate's order: model, A low then high, (ruler, S₁), λ, N.""" + out = [] + pairs = ((mc["ruler"], mc["s1_lo"]), (mc["ruler"], mc["s1_hi"]), (mc["ruler_wide"], mc["s1_hi"])) + for model, t in terms.items(): + for high in (False, True): + sh = shape(mc, t, high) + for ruler, s1 in pairs: + for lam in (mc["lambda_lo"], mc["lambda_hi"]): + for n in range(1, 5): + try: + c, m, idle, edges, p = cell(mc, sh, n, lam, ruler) + except Refused as e: + raise Refused(f"{model} A {'high' if high else 'low'}: {e}") from None + busy, busy_tol, on_line = busiest(c, p, m, s1) + if on_line: + edges.insert(0, "nic") + E = sh["experts"] + paging = c.room(m) < E + rooms = [c.rooms[0]] + [c.rooms[1]] * m + per = [round(x * E - r) if paging else 0 for x, r in zip(p.s, rooms)] + [0] * (n - 1 - m) + out.append({ + "model": model, "a": "high" if high else "low", "n": n, "lambda_us": lam // 1000, + "ruler": ruler, "s1": s1, "used": m + 1, + "idle": ",".join(f"{j}:{w}" for j, w in idle) or "-", + "paged": max(0, E - c.room(m)), "paged_nodes": ",".join(map(str, per)), + "step_ns": round(p.step * NS), "step_tol": int(p.tol * NS) + SLACK, + "nic": round(busy * 1000), "nic_tol": int(busy_tol * 1000) + SLACK, + "fires": int(m > 0 and busy > 0.5), "msgs": 2 * min(sh["k"], m), + "edge": ",".join(edges) or "-", + }) + return out + + +def text(cells): + lines = ["# nodes_scale's expected cells (docs/contrib/dgx-spark.md §9.9), written by tools/flow/nodes_scale.py " + "--write from nodes_scale_inputs.tsv; do not edit.", + "\t".join(COLUMNS)] + for c in cells: + lines.append("\t".join(str(c[k]) for k in COLUMNS)) + return "\n".join(lines) + "\n" + + +def span(values, fmt): + lo, hi = fmt(min(values)), fmt(max(values)) + return lo if lo == hi else f"{lo} to {hi}" + + +def blocks(mc, terms, cells): + """The doc's derived tables, each a list of lines: §9.8's "where each fires" and §9.7's (a) rows.""" + at = {(c["model"], c["a"], c["n"], c["lambda_us"], c["ruler"], c["s1"]): c for c in cells} + lo, hi = mc["lambda_lo"] // 1000, mc["lambda_hi"] // 1000 + r, s1_lo, s1_hi = mc["ruler"], mc["s1_lo"], mc["s1_hi"] + + def where(model, pick, rate=s1_hi): + """The cells `pick` names, by N; an N alone is every λ and A at it.""" + hits = [] + for n in range(1, 5): + each = [(lam, [a for a in ("low", "high") if pick(at[(model, a, n, lam, r, rate)])]) for lam in (lo, hi)] + if all(len(ends) == 2 for _, ends in each): + hits.append(f"N = {n}") + continue + hits += [f"N = {n}, λ {lam}, A {' and '.join(ends)}" for lam, ends in each if ends] + return "; ".join(hits) or "never" + + fires = [f"| Model | Decode step (ms) at the gate's inputs, N = 1 → 2 → 3 → 4, λ {lo} / {hi}, A low " + f"| Criterion 2 fires | Criterion 7 fires | Criterion 5 fires, S₁ {s1_lo:,} | Criterion 5 fires, " + f"S₁ {s1_hi:,} |", "|---|---|---|---|---|---|"] + for model in terms: + steps = [] + for n in range(1, 5): + a, b = (at[(model, "low", n, lam, r, s1_hi)]["step_ns"] / 1e6 for lam in (lo, hi)) + steps.append(f"{a:.1f}" if n == 1 else f"{a:.1f} / {b:.1f}") + fires.append(f"| {model} | {' → '.join(steps)} | {where(model, lambda c: 'link' in c['idle'])} " + f"| {where(model, lambda c: 'gain' in c['idle'])} " + f"| {where(model, lambda c: c['fires'] == 1, rate=s1_lo)} " + f"| {where(model, lambda c: c['fires'] == 1)} |") + + star = ["| (a) star at the gate's inputs | N | Nodes used | Decode, one request (tok/s) | Prompt (× S₁) " + "| Node 0's NIC ingress cap (tok/s) | Link bytes a token (MB) | Messages a token |", + "|---|---|---|---|---|---|---|---|"] + for model, t in terms.items(): + for n in (2, 4): + cs = [at[(model, a, n, lam, r, s1_hi)] for a in ("low", "high") for lam in (lo, hi)] + used, decode, prompt, cap, link, msgs = [], [], [], [], [], [] + for a in ("low", "high"): + sh = shape(mc, t, a == "high") + for lam in (lo, hi): + c, m, _, _, p = cell(mc, sh, n, lam * 1000, r) + k, L = sh["k"], sh["layers"] + A, Sh, R = sh["serial"], sh["shared"] * L, k * c.id + used.append(m + 1) + decode.append(NS / round(p.step * NS)) + prompt.append((A + Sh + R) / (A + max([Sh + p.s[0] * R] + [x * R for x in p.s[1:]]))) + row = mc["row_value"] * sh["n_embd"] + remote = 1.0 - p.s[0] + cap.append(mc["nic"] / (L * k * remote * row) if m else 0.0) + reached = 0.0 + for cc, q in falls(p.s, k).items(): + reached += q * sum(1 for x in cc[1:] if x > 0) + link.append(L * row * (reached + k * remote) / 1e6) + msgs.append(2 * L * reached) + paged = [x["paged"] for x in cs] + note = f" (pool only: pages {span(paged, str)} ids)" if max(paged) else "" + star.append(f"| {model} | {n} | {span(used, str)} | {span(decode, lambda v: f'{v:.1f}')}{note} " + f"| {span(prompt, lambda v: f'{v:.2f}')} | {span(cap, lambda v: f'{v:,.0f}')} " + f"| {span(link, lambda v: f'{v:.2f}')} | {span(msgs, lambda v: f'{v:.0f}')} |") + return [fires, star] + + +def table(mc, terms, cells): + return "\n\n".join("\n".join(b) for b in blocks(mc, terms, cells)) + + +def doc_differs(bs, text): + """The first line of each block of `bs` that the doc `text` does not carry as printed, alone under its header.""" + doc = text.split("\n") + off = [] + for b in bs: + heads = [i for i, line in enumerate(doc) if line == b[0]] + if len(heads) != 1 or doc[heads[0]:heads[0] + len(b)] != b or heads[0] + len(b) < len(doc) and \ + doc[heads[0] + len(b)].startswith("|"): + off.append(b[0]) + return off + + +def check(): + try: + with open(CELLS, encoding="utf-8") as f: + have = f.read() + except OSError as e: + print(f"nodes_scale: {CELLS}: {e.strerror}", file=sys.stderr) + return 1 + machine, terms = load() + cells = derive(machine, terms) + rc = 0 + if have != text(cells): + print(f"nodes_scale: {CELLS} is not the derivation of {INPUTS}: run tools/flow/nodes_scale.py --write", + file=sys.stderr) + rc = 1 + try: + with open(DOC, encoding="utf-8") as f: + doc = f.read() + except OSError as e: + print(f"nodes_scale: {DOC}: {e.strerror}", file=sys.stderr) + return 1 + for head in doc_differs(blocks(machine, terms, cells), doc): + print(f"nodes_scale: {DOC} does not carry the --table block {head!r} as printed", file=sys.stderr) + rc = 1 + return rc + + +def self_test(): + ok = True + + def expect(what, cond): + nonlocal ok + print(f"{'ok ' if cond else 'FAIL'} {what}") + ok = ok and cond + + def near(a, b): + return abs(a - b) <= 1e-12 * max(1.0, abs(b)) + + # Two slots at shares (1/2, 1/2): both on node 0 a quarter, one each a half, both remote a quarter. + d = falls([0.5, 0.5], 2) + expect("two slots on a pair: 1/4, 1/2, 1/4", d == {(2, 0): 0.25, (1, 1): 0.5, (0, 2): 0.25}) + d = falls([0.25, 0.25, 0.25, 0.25], 10) + expect("ten slots on four nodes: 286 tuples, one in all, (10, 0, 0, 0) at 4^-10", + len(d) == 286 and near(sum(d.values()), 1.0) and near(d[(10, 0, 0, 0)], 1 / 1048576)) + # (2, 1, 1) at shares (1/2, 1/4, 1/4): 12 orders, each (1/2)² × 1/4 × 1/4 = 1/64. + expect("the tuple (2, 1, 1) at (1/2, 1/4, 1/4)", near(falls([0.5, 0.25, 0.25], 4)[(2, 1, 1)], 12 / 64)) + + # Sh 1, e 2, rt 4. Both on node 0 (1/4): 1 + 2e = 5; one each (1/2): max(1 + e, rt + e) = 6; both remote (1/4): + # max(1, rt + 2e) = 8. Expected 1.25 + 3 + 2 = 6.25. Node 0's mean window: 1 + e at one slot on average, 3; the + # remote's: (1/2) × 6 + (1/4) × 8 = 5. + slowest, mean = layer([0.5, 0.5], 1.0, 2.0, 2, Link(4.0, 0.0, 0.0)) + expect("a pair's layer: the expected slowest window", near(slowest, 6.25)) + expect("a pair's layer: the mean windows", near(mean[0], 3.0) and near(mean[1], 5.0)) + # Three nodes at (0, 1/2, 1/2), two slots: one on each remote (1/2) posts once more and takes the other's rows + # ahead: rt + post + row + e = 4 + 1 + 0.5 + 2 = 7.5; both on one remote (1/2): rt + 2e = 8. Expected 7.75. + slowest, _ = layer([0.0, 0.5, 0.5], 0.0, 2.0, 2, Link(4.0, 1.0, 0.5)) + expect("a ring's layer: a post and the rows ahead of a second remote", near(slowest, 7.75)) + + mc = {"read": 1, "nic": 1, "pool": 100, "lambda_lo": 0, "lambda_hi": 0, "hop": 0, "post": 0, "row_value": 4, + "vocab": 0, "ruler": 10, "ruler_wide": 40, "s1_lo": 1, "s1_hi": 1} + sh = {"layers": 1, "experts": 4, "k": 1, "n_embd": 256, "serial": 0, "shared": 0, "expert": 10, + "fixed0": 70, "fixed1": 60} + c = Case(mc, sh, 2, 0) + # Rooms (100 - 70) / 10 = 3 and (100 - 60) / 10 = 4: one remote holds every id; with no link and no shared + # bytes the windows balance at half the ids each, inside both rooms. + expect("the rooms", c.rooms == (3, 4) and not c.pool_idle) + expect("no link, no shared bytes: half each", near(c.balance(1, c.bare), 0.5)) + expect("the bounds where the pools hold every id", c.limits(1) == [(0.0, 0.75), (0.0, 1.0)]) + # A shared read of 3 on node 0 against one slot of 10: s0 × 10 + 3 = (1 - s0) × 10 at s0 = 0.35. One slot: on + # node 0 (0.35) the window is 3 + 10, on the remote (0.65) max(3, 10): a step of 4.55 + 6.5 = 11.05. One id moved + # (a quarter): node 0 at 0.1 gives 1.3 + 9 = 10.3, at 0.6 gives 7.8 + 4 = 11.8, so the tolerance is 0.75. + c.Sh = 3.0 + p = c.plan(1, c.bare) + expect("plan(): node 0's shared bytes move ids to the remote, and its one-id tolerance", + near(p.s[0], 0.35) and near(p.step, 11.05) and near(p.tol, 0.75)) + c2 = Case(mc, dict(sh, fixed0=90, fixed1=90), 2, 0) + # Rooms 1 and 1 of 4 ids: the pools page 2, and each node keeps at least its room. + expect("paging: each node at least its room", c2.room(1) == 2 and c2.limits(1) == [(0.25, 1.0), (0.25, 1.0)]) + c3 = Case(mc, dict(sh, fixed1=95), 2, 0) + expect("a room-0 remote has paging bounds and fits its fixed-byte budget", + c3.rooms == (3, 0) and not c3.pool_idle and c3.limits(1) == [(0.75, 1.0), (0.0, 1.0)]) + _, m, idle, _, _ = cell(mc, dict(sh, fixed1=95), 2, 0, 10) + expect("a remote of room 0 must gain twice the ruler", m == 0 and idle == [(1, "gain")]) + _, m, idle, _, _ = cell(mc, dict(sh, fixed1=101), 3, 0, 10) + expect("a remote whose own bytes pass its pool is idle for its pool", m == 0 and idle == [(1, "pool"), (2, "pool")]) + try: + Case(mc, dict(sh, fixed0=101), 1, 0) + expect("node 0's bytes past its pool: not refused", False) + except Refused as e: + expect(f"node 0's bytes past its pool: {e}", "node 0" in str(e)) + # Four nodes on a switch: the hop joins λ, a ring of three has none. One slot, every id on node 1: a layer is + # λ + hop + e = 25 + 1.5 + 10 s at λ 25 s, so a step of A 2 + 36.5. + sw = dict(mc, hop=1_500_000_000, lambda_lo=25_000_000_000) + c4 = Case(sw, dict(sh, serial=2), 4, sw["lambda_lo"]) + expect("the switch's hop at N = 4", near(c4.step([0.0, 1.0, 0.0, 0.0], c4.link), 38.5) + and near(Case(sw, sh, 3, sw["lambda_lo"]).link.rt, 25.0)) + # NIC, node 0 at s0 = 1/2, k = 1, one layer, rows of 4 × 256 = 1,024 B: a remote slot half the time, so 512 B a + # token. Decode at a step of 1,024 s on a NIC of 1 B/s: half busy. The prompt at S₁ 1 with no A or Sh runs + # 10 / 5 = 2 tokens a second over the slowest node's FFN: 1,024 B a second, twice the NIC. + expect("the NIC: the prompt's share where it dominates", near(c.nic([0.5, 0.5], 1024.0, 1), 1024.0)) + expect("the NIC: the decode's share where it dominates", near(c.nic([0.5, 0.5], 1024.0, 1e-9), 0.5)) + # Three nodes at (1/2, 1/4, 1/4): node 0 takes every remote slot's rows, 512 B a token, each remote its own + # quarter, 256 B. Decode at a step of 1,024 s: node 0 half busy. + expect("the NIC: node 0 takes every remote's rows", near(Case(mc, sh, 3, 0).nic([0.5, 0.25, 0.25], 1024.0, 1e-9), + 0.5)) + # Sh of 10 on node 0 against R = 10: node 0's FFN is 10 + 5, the remote's 5, so the prompt runs 20 / 15 tokens a + # second at S₁ 1 and node 0 takes 512 B a token: 682.67 B a second. + cs = Case(mc, dict(sh, shared=10), 2, 0) + expect("the NIC: the prompt's slowest FFN is node 0's with Sh", near(cs.nic([0.5, 0.5], 1e9, 1), 512 * 20 / 15)) + # On a NIC of 10^6 B/s the same split's prompt at S₁ 488.28125 is busy one half (1,024 µB a token of S₁); node 0 + # at a quarter of the ids takes 768 µB at the same rate, so the band holds the line: an edge. + cn = Case(dict(mc, nic=10 ** 6), sh, 2, 0) + busy, tol, on = busiest(cn, cn.plan(1, cn.link), 1, 488.28125) + expect("the NIC on its half within one id: an edge", near(busy, 0.5) and tol > 0.3 and on) + # A gain on twice an integer ruler. A = 248 and two slots of 1 on one node: a step of 250. A second node takes one + # slot of two half the time, so the window falls to 1.5: a gain of 0.2 %, twice a ruler of 1 permille; one id + # moved (of two) moves the step by 0.5. The node is named an edge and used, the line not under it. + gm = dict(mc, nic=10 ** 12, pool=10 ** 6) + gs = dict(sh, experts=2, k=2, expert=1, serial=248, fixed0=0, fixed1=0) + _, m, idle, edges, _ = cell(gm, gs, 2, 0, 1) + expect("a gain on twice an integer ruler is an edge, on the used side", edges == ["gain:1:2000:2002"] and m == 1) + # A = 123, one id of a hundred: the gain is 0.5 / 125 = 0.4 %, between a ruler of 3 permille and twice it, and one + # id moved (0.01 of the share) moves the step by 0.0002: no edge, and the node is idle. + _, m, idle, edges, _ = cell(gm, dict(gs, experts=100, serial=123), 2, 0, 3) + expect("a gain under twice the ruler and over it leaves the node idle", (m, idle, edges) == (0, [(1, "gain")], [])) + # One slot, two ids, A 3.4, e 1 and a round trip of 2: the split balances at s0 = 3/4 (s0 = (1 - s0) × 3), so the + # step is 3.4 + 1.5 against 3.4 + 1 with no link: a link share of 0.5 / 4.9, past the tenth. One id moved puts + # s0 at 1/4 and the step at 3.4 + 2.5, so the band holds the tenth: an edge, and the node idle. + lm = dict(gm, read=10, lambda_lo=2 * 10 ** 9) + _, m, idle, edges, _ = cell(lm, dict(gs, k=1, expert=10, serial=34), 2, lm["lambda_lo"], 10) + expect("a link share on the tenth within one id: an edge, on its side", (m, idle) == (0, [(1, "link")]) + and edges == [f"link:1:{round(0.5 / 4.9 * 1e6)}:{int(1.0 / 4.9 * 1e6) + 2}"]) + cm = Case(mc, dict(sh, fixed0=0, fixed1=0), 3, 0) + moves = cm.moves([0.5, 0.25, 0.25]) + expect("one id moved: node 0 to and from each remote, and between the remotes", + len(moves) == 6 and [0.5, 0.0, 0.5] in moves and [0.5, 0.5, 0.0] in moves) + + machine, terms = load() + cells = derive(machine, terms) + by = {(x["model"], x["a"], x["n"], x["lambda_us"], x["ruler"], x["s1"]): x for x in cells} + glm2 = [by[("GLM", a, 2, lam, machine["ruler"], machine["s1_hi"])] for a in ("low", "high") for lam in (25, 70)] + expect("GLM held whole on two Sparks", all(x["paged"] == 0 and x["used"] == 2 for x in glm2)) + v41 = by[("V4.1", "low", 2, 25, machine["ruler"], machine["s1_hi"])] + expect("V4.1 pages on two Sparks, on both, and not on three", + v41["paged"] > 0 and all(int(x) > 0 for x in v41["paged_nodes"].split(",")) + and by[("V4.1", "low", 3, 25, machine["ruler"], machine["s1_hi"])]["paged"] == 0) + q4 = by[("Qwen3.8", "low", 4, 70, machine["ruler"], machine["s1_hi"])] + expect("Qwen3.8 on four switched Sparks at λ 70 leaves nodes idle for the link", "link" in q4["idle"]) + expect("V4.1 on four Sparks: a handoff and its rows for each of three remotes, none on one", + by[("V4.1", "low", 4, 25, machine["ruler"], machine["s1_hi"])]["msgs"] == 6 + and by[("V4.1", "low", 1, 25, machine["ruler"], machine["s1_hi"])]["msgs"] == 0) + expect("each cell's paged ids per node sum to its paged ids", + all(sum(map(int, x["paged_nodes"].split(","))) == x["paged"] for x in cells)) + + with open(INPUTS, encoding="utf-8") as f: + good = f.read() + lines = good.split("\n") + first = next(i for i, x in enumerate(lines) if x.startswith("read\t")) + + def row(name, value): + """The inputs with row `name`'s value replaced.""" + return "\n".join(f"{name}\t{value}\t" + x.split("\t", 2)[2] if x.startswith(name + "\t") else x + for x in lines) + + def refused(what, body, reason): + try: + inputs(body, "in.tsv") + except Refused as e: + expect(f"{what}: {e}", reason in str(e)) + return + expect(f"{what}: not refused", False) + + def accepted(what, body): + try: + inputs(body, "in.tsv") + expect(what, True) + except Refused as e: + expect(f"{what}: refused, {e}", False) + + accepted("the inputs as they are", good) + refused("a row of four cells", "\n".join(lines[:first] + [lines[first].rsplit("\t", 1)[0]] + lines[first + 1:]), + "has 4 cells") + refused("a name twice", "\n".join(lines[:first + 1] + [lines[first]] + lines[first + 1:]), "read twice") + refused("a kind of no kind", "\n".join(lines[:first] + [lines[first].replace("\tmeasured\t", "\tguessed\t")] + + lines[first + 1:]), "kind 'guessed'") + refused("a value not whole", row("read", "2.5"), "not a whole number") + refused("a value in other digits", row("read", "٢"), "not a whole number") + refused("a missing row", "\n".join(lines[:first] + lines[first + 1:]), "no read row") + refused("a row that is no input", good + "spare\t1\tx\tassumed\tx\n", "spare is no input") + refused("top-k 0", row("GLM/top_k", 0), "GLM/top_k 0") + refused("top-k past the experts", row("GLM/top_k", 289), "GLM/top_k 289") + accepted("top-k at the experts", row("GLM/top_k", 288)) + refused("a ruler of 0", row("ruler", 0), "ruler 0") + refused("a ruler past 1000", row("ruler_wide", 1001), "ruler_wide 1001") + accepted("rulers of 1 and 1000", row("ruler", 1).replace("ruler_wide\t40\t", "ruler_wide\t1000\t")) + refused("a read rate of 0", row("read", 0), "read is 0") + refused("a NIC of 0", row("nic", 0), "nic is 0") + refused("a prompt rate of 0", row("s1_lo", 0), "s1_lo is 0") + refused("no layers", row("V4.1/layers", 0), "V4.1/layers is 0") + refused("a row width under a block", row("Qwen3.8/n_embd", 255), "Qwen3.8/n_embd 255") + accepted("a row width of one block", row("Qwen3.8/n_embd", 256)) + refused("an expert under a row", row("Qwen3.8/expert", 1439), "Qwen3.8/expert 1439") + try: + derive(*inputs(row("GLM/fixed0_lo", machine["pool"] + 1), "in.tsv")) + expect("node 0's bytes past its pool in the inputs: not refused", False) + except Refused as e: + expect(f"node 0's bytes past its pool in the inputs: {e}", str(e).startswith("GLM A low: node 0's")) + bs = [["| a | b |", "|---|---|", "| 1 | 2 |"]] + expect("a doc that carries a block as printed", doc_differs(bs, "x\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\ny") == []) + expect("a doc whose block differs in a cell, has a row more, or lacks it", + all(doc_differs(bs, d) == ["| a | b |"] for d in ("| a | b |\n|---|---|\n| 1 | 3 |\n", + "| a | b |\n|---|---|\n| 1 | 2 |\n| 3 | 4 |\n", "x\n"))) + expect("the cells file and the doc's tables are the derivation's", check() == 0) + print(f"self-test: {'ok' if ok else 'FAIL'}") + return 0 if ok else 1 + + +def main(argv): + if len(argv) > 1 or (argv and argv[0] not in ("--write", "--check", "--table", "--self-test")): + print(f"nodes_scale: usage\n{__doc__}", file=sys.stderr) + return 64 + try: + if argv == ["--self-test"]: + return self_test() + if argv == ["--check"]: + return check() + machine, terms = load() + cells = derive(machine, terms) + except Refused as e: + print(f"nodes_scale: {e}", file=sys.stderr) + return 2 + if argv == ["--write"]: + with open(CELLS, "w", encoding="utf-8") as f: + f.write(text(cells)) + elif argv == ["--table"]: + print(table(machine, terms, cells)) + else: + sys.stdout.write(text(cells)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))