From 4e4807152e25fcfa74a6fbe4c4d33ca2890021b5 Mon Sep 17 00:00:00 2001 From: juliankang4 <200967947+juliankang4@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:04:55 +0900 Subject: [PATCH] placement, tools: nodes_scale holds the node plans to cells tools/flow/nodes_scale.py derives apart from the planner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The script reads crates/placement/tests/nodes_scale_inputs.tsv (each row with its kind and source) and writes the 144 cells of nodes_scale_cells.tsv: per model, A, N, λ, ruler and S₁ the nodes used, why each other node is idle, the ids paged in all and per node, the step, the busiest NIC and node 0's messages a layer, each value with the largest change one id moved between two used nodes makes. It splits by §9.2's rule in real shares and builds each window from the count tuples of a token's slots, one slot at a time, with no code or method shared with nodes.rs. A value whose band holds a line (the NIC's half, the link share's tenth, twice the ruler) is named an edge. nodes_scale first holds its machine, shapes and budgets to the inputs, then every plan to its cell (expected_sets): the nodes used and idle exactly, the paged ids per node within one, the step and the NIC within their bands, and each edge on the side SIDES pins by hand. The exact used and idle sets of clauses 4 and 7 move into expected_sets; clause 5 takes its firing cells and clause 6 node 0's messages from the cells. dgx-spark.md §9.7 gains the (a) rows at the gate's inputs and §9.8 the table of where each criterion fires, both the script's --table; §9.9 names the script and §10 what it recomputes and what still uses the first inputs. §9.7's capacity rows at N = 1 and 2 take GLM's cells from plan_nodes on the file's header read and V4.1's at N = 1 from node 0's budget: the node plans refuse the V4.1 file by name, since node 0's card plan reads its engram tables from the NVMe tier, and §10 lists the fill term they lack. --- crates/placement/src/placement/nodes_scale.rs | 561 +++++++++++-- crates/placement/tests/nodes_scale_cells.tsv | 146 ++++ crates/placement/tests/nodes_scale_inputs.tsv | 53 ++ docs/contrib/dgx-spark.md | 88 +- tools/check-recipes.sh | 1 + tools/flow/nodes_scale.py | 751 ++++++++++++++++++ 6 files changed, 1516 insertions(+), 84 deletions(-) create mode 100644 crates/placement/tests/nodes_scale_cells.tsv create mode 100644 crates/placement/tests/nodes_scale_inputs.tsv create mode 100755 tools/flow/nodes_scale.py diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index cd6ceb6f1..3731031c1 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -1,9 +1,10 @@ //! `nodes_scale` (`docs/contrib/dgx-spark.md` §9.9): the node plans on -//! synthetic DGX Sparks, N = 1 to 4, against the cells of §9.8's "where each -//! fires" table as the predictor derives them; a clause whose cells differ -//! from the table names the difference. Three synthetic shapes carry §9.7's -//! model terms for V4.1, GLM-5.3-Flash and Qwen3.8, A low and high; no file -//! is read but where a clause reads the Qwen3.8 file's tensor table. +//! synthetic DGX Sparks, N = 1 to 4, against the expected cells that +//! `tools/flow/nodes_scale.py` derives from §9.7's model and §9.8's +//! criteria apart from the planner (`tests/nodes_scale_cells.tsv`). Three +//! synthetic shapes carry §9.7's model terms for V4.1, GLM-5.3-Flash and +//! Qwen3.8, A low and high; no file is read but where a clause reads the +//! Qwen3.8 file's tensor table. use std::sync::OnceLock; @@ -412,6 +413,380 @@ fn used(plan: &NodePlan<'_>) -> u64 { .count() as u64 } +/// The inputs `tools/flow/nodes_scale.py` derives the expected cells from. +const INPUTS: &str = include_str!("../../tests/nodes_scale_inputs.tsv"); + +/// The expected cells (§9.9 "Expected sets"), as that script writes them. +const CELLS: &str = include_str!("../../tests/nodes_scale_cells.tsv"); + +/// The value of the inputs row `name`. +fn input(name: &str) -> u64 { + INPUTS + .lines() + .filter(|l| !l.starts_with('#')) + .skip(1) + .find_map(|l| { + let mut c = l.split('\t'); + (c.next() == Some(name)).then(|| c.next()) + }) + .flatten() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| panic!("nodes_scale_inputs.tsv: no whole number for {name}")) +} + +/// A derived value whose band holds its criterion's line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + /// The busiest NIC, on criterion 5's half. + Nic, + /// Node `node`'s link share in ppm, on criterion 2's tenth. + Link { node: usize, ppm: u64, tol: u64 }, + /// Node `node`'s gain in ppm, on criterion 7's twice the ruler. + Gain { node: usize, ppm: u64, tol: u64 }, +} + +impl Edge { + fn criterion(self) -> &'static str { + match self { + Edge::Nic => "nic", + Edge::Link { .. } => "link", + Edge::Gain { .. } => "gain", + } + } +} + +/// One derived cell: a cell at a ruler and S₁, what the script derives of +/// its plan, and each value's tolerance, the largest change one id moved +/// between two used nodes makes. +#[derive(Debug)] +struct Derived { + cell: Cell, + ruler: u64, + rate: u64, + used: u64, + /// The idle nodes and why, `node:pool`, `node:link` or `node:gain`, or + /// `-`. + idle: String, + paged: u64, + /// Each node's paged ids, to one id. + paged_nodes: Vec, + step_ns: u64, + step_tol: u64, + nic: u64, + nic_tol: u64, + fires: bool, + /// Node 0's messages in the layer that reaches the most remotes. + msgs: u64, + edges: Vec, +} + +/// Every derived cell. +fn derived() -> &'static [Derived] { + static DERIVED: OnceLock> = OnceLock::new(); + DERIVED.get_or_init(|| { + let mut rows = CELLS.lines().filter(|l| !l.starts_with('#')); + assert_eq!( + rows.next(), + Some( + "model\ta\tn\tlambda_us\truler\ts1\tused\tidle\tpaged\tpaged_nodes\tstep_ns\tstep_tol\tnic\tnic_tol\tfires\tmsgs\tedge" + ), + "nodes_scale_cells.tsv's header" + ); + rows.map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + let [model, a, n, lambda, ruler, rate, used, idle, paged, per, step, step_tol, nic, nic_tol, fires, msgs, edge] = + c[..] + else { + panic!("nodes_scale_cells.tsv: {l:?} is not 17 cells"); + }; + let num = |v: &str| -> u64 { + v.parse() + .unwrap_or_else(|_| panic!("nodes_scale_cells.tsv: {v:?} in {l:?}")) + }; + let name = SHAPES + .iter() + .map(|s| s.name) + .find(|&s| s == model) + .unwrap_or_else(|| panic!("nodes_scale_cells.tsv: no shape {model}")); + let edges = edge + .split(',') + .filter(|&e| e != "-") + .map(|e| { + let f: Vec<&str> = e.split(':').collect(); + match f[..] { + ["nic"] => Edge::Nic, + ["link", node, ppm, tol] => Edge::Link { + node: num(node) as usize, + ppm: num(ppm), + tol: num(tol), + }, + ["gain", node, ppm, tol] => Edge::Gain { + node: num(node) as usize, + ppm: num(ppm), + tol: num(tol), + }, + _ => panic!("nodes_scale_cells.tsv: edge {e:?} in {l:?}"), + } + }) + .collect(); + Derived { + cell: (name, num(n) as usize, num(lambda), a == "high"), + ruler: num(ruler), + rate: num(rate), + used: num(used), + idle: idle.to_string(), + paged: num(paged), + paged_nodes: per.split(',').map(num).collect(), + step_ns: num(step), + step_tol: num(step_tol), + nic: num(nic), + nic_tol: num(nic_tol), + fires: fires == "1", + msgs: num(msgs), + edges, + } + }) + .collect() + }) +} + +/// The derived cell of `cell` at `ruler` and S₁ `rate`. +fn derived_at(cell: Cell, ruler: u64, rate: u64) -> &'static Derived { + derived() + .iter() + .find(|d| d.cell == cell && d.ruler == ruler && d.rate == rate) + .unwrap_or_else(|| panic!("no derived cell {cell:?} at ruler {ruler}, S₁ {rate}")) +} + +/// The side of each derived edge, pinned here: a cell, its ruler and S₁, the +/// edge's criterion, and whether that criterion fires there (criterion 2 or +/// 7 leaves the node idle, criterion 5 names the NIC). +const SIDES: [(Cell, u64, u64, &str, bool); 4] = [ + (("V4.1", 2, 25, false), RULER, 3_000, "nic", true), + (("V4.1", 2, 25, false), 4 * RULER, 3_000, "nic", true), + (("Qwen3.8", 3, 70, false), RULER, 1_000, "link", false), + (("Qwen3.8", 3, 70, false), RULER, 3_000, "link", false), +]; + +/// Holds the derived side of `e` in `d` to its pin in [`SIDES`]; an edge +/// with no pin, or whose derived side is not its pin, panics by name. +fn side(d: &Derived, e: Edge) { + let fires = match e { + Edge::Nic => d.fires, + Edge::Link { node, .. } => d.idle.split(',').any(|i| i == format!("{node}:link")), + Edge::Gain { node, .. } => d.idle.split(',').any(|i| i == format!("{node}:gain")), + }; + let (_, _, _, _, pin) = SIDES + .iter() + .find(|s| s.0 == d.cell && s.1 == d.ruler && s.2 == d.rate && s.3 == e.criterion()) + .unwrap_or_else(|| { + panic!( + "{:?} at ruler {}, S₁ {}: a {} edge with no pinned side", + d.cell, + d.ruler, + d.rate, + e.criterion() + ) + }); + assert_eq!( + fires, + *pin, + "{:?} at ruler {}, S₁ {}: the derived {} side is not the pinned one", + d.cell, + d.ruler, + d.rate, + e.criterion() + ); +} + +/// Whether a link of `link_ns` can pass a tenth of a step of `step_ns`, +/// both rounded down to whole nanoseconds: the link is under `link_ns + 1` +/// and the step at least `step_ns`. +fn past_tenth(link_ns: u64, step_ns: u64) -> bool { + 10 * (link_ns + 1) > step_ns +} + +/// The idle nodes of `plan` as the derived cells name them. +fn idle_of(plan: &NodePlan<'_>) -> String { + let idle: Vec = plan + .idle() + .iter() + .map(|i| { + let why = match i.why { + IdleWhy::Pool { .. } => "pool", + IdleWhy::LinkShare { .. } => "link", + IdleWhy::Gain { .. } => "gain", + }; + format!("{}:{why}", i.node) + }) + .collect(); + if idle.is_empty() { + "-".to_string() + } else { + idle.join(",") + } +} + +/// §9.9 "Expected sets": the cells are `tools/flow/nodes_scale.py`'s, +/// derived from `nodes_scale_inputs.tsv`, whose rows are this gate's own +/// machine, shapes and budgets. In every cell the plan uses the derived +/// nodes and leaves the others idle for the derived criterion, pages the +/// derived ids in all and on each node to one id, and its step and busiest +/// NIC are the derived ones to one id's change. Where a derived value's band +/// holds its line, the side is pinned in [`SIDES`], and a link share is +/// held to the band, read from the node's `NodeIdle` or, where the node is +/// the last used, from the plan; so is a gain, from a node's `NodeIdle`. +#[test] +fn expected_sets() { + for (name, value) in [ + ("read", GB10_READ), + ("nic", GB10_NIC), + ("pool", case_of(("V4.1", 1, 25, false)).budgets[0].pool()), + ("lambda_lo", LAMBDAS[0]), + ("lambda_hi", LAMBDAS[1]), + ("hop", HOP_NS), + ("post", POST_NS), + ("row_value", 4), + ("vocab", VOCAB), + ("ruler", RULER), + ("ruler_wide", 4 * RULER), + ("s1_lo", PROMPT_RATES[0]), + ("s1_hi", PROMPT_RATES[1]), + ] { + assert_eq!(input(name), value, "{name}"); + } + for case in cases() { + let s = case.shape; + let end = if case.high { "hi" } else { "lo" }; + let at = usize::from(case.high); + for (term, value) in [ + ("layers".to_string(), s.layers as u64), + ("n_embd".to_string(), s.n_embd), + ("experts".to_string(), s.experts), + ("top_k".to_string(), s.used), + ("expert".to_string(), s.expert), + (format!("shared_{end}"), s.shared[at]), + (format!("serial_{end}"), s.serial[at]), + (format!("fixed0_{end}"), case.budgets[0].fixed()), + ("fixed1".to_string(), case.budgets[1].fixed()), + ] { + assert_eq!( + input(&format!("{}/{term}", s.name)), + value, + "{}/{term}", + s.name + ); + } + assert_eq!(case.budgets[1].pool(), case.budgets[0].pool()); + } + let mut seen = 0; + let mut pinned = Vec::new(); + for (ruler, rate) in [ + (RULER, PROMPT_RATES[0]), + (RULER, PROMPT_RATES[1]), + (4 * RULER, PROMPT_RATES[1]), + ] { + each_cell(ruler, rate, |cell, plan| { + seen += 1; + let d = derived_at(cell, ruler, rate); + let at = format!("{cell:?} at ruler {ruler}, S₁ {rate}"); + assert_eq!( + (used(plan), idle_of(plan)), + (d.used, d.idle.clone()), + "{at}: the nodes used and idle" + ); + let p = plan.predict().expect("a prediction"); + let id: u64 = plan.terms().layers.iter().map(|l| l.expert).sum(); + let paged: Vec = plan + .owned() + .iter() + .enumerate() + .map(|(j, &o)| { + let held = if o == 0 { + 0 + } else { + let fixed = case_of(cell).budgets[j.min(1)].fixed(); + p.per_node[j] + .pool + .checked_sub(fixed) + .expect("a used node's pool holds its budget") + / id + }; + o.checked_sub(held).expect("a node holds at most its ids") + }) + .collect(); + assert!( + paged.iter().sum::() == d.paged + && paged.len() == d.paged_nodes.len() + && paged + .iter() + .zip(&d.paged_nodes) + .all(|(a, b)| a.abs_diff(*b) <= 1), + "{at}: paged {paged:?}, derived {} as {:?} to one id each", + d.paged, + d.paged_nodes + ); + assert!( + p.step_ns.abs_diff(d.step_ns) <= d.step_tol, + "{at}: step {} ns, derived {} ± {} ns", + p.step_ns, + d.step_ns, + d.step_tol + ); + let busiest = p.per_node.iter().map(|l| l.nic_permille).max().unwrap_or(0); + assert!( + busiest.abs_diff(d.nic) <= d.nic_tol, + "{at}: NIC {busiest} ‰, derived {} ± {} ‰", + d.nic, + d.nic_tol + ); + for &e in &d.edges { + side(d, e); + pinned.push((cell, ruler, rate, e.criterion())); + let (node, want, tol) = match e { + Edge::Nic => continue, + Edge::Link { node, ppm, tol } | Edge::Gain { node, ppm, tol } => { + (node, ppm, tol) + } + }; + let why = plan.idle().iter().find(|i| i.node == node).map(|i| i.why); + let last = used(plan) == node as u64 + 1; + let ppm = match (e, why) { + (Edge::Link { .. }, Some(IdleWhy::LinkShare { link_ns, step_ns })) => { + assert!(past_tenth(link_ns, step_ns), "{at}: {why:?}"); + i128::from(link_ns) * 1_000_000 / i128::from(step_ns) + } + (Edge::Link { .. }, None) if last => { + i128::from(p.link_ns) * 1_000_000 / i128::from(p.step_ns) + } + ( + Edge::Gain { .. }, + Some(IdleWhy::Gain { + without_ns, + with_ns, + }), + ) => { + (i128::from(without_ns) - i128::from(with_ns)) * 1_000_000 + / i128::from(without_ns) + } + (Edge::Gain { .. }, None) => continue, + _ => panic!("{at}: no witness of {e:?} on node {node}, idle {why:?}"), + }; + assert!( + ppm.abs_diff(i128::from(want)) <= u128::from(tol), + "{at}: node {node}'s {} {ppm} ppm, derived {want} ± {tol}", + e.criterion() + ); + } + }); + } + assert_eq!(seen, derived().len()); + let mut sides: Vec<_> = SIDES.iter().map(|s| (s.0, s.1, s.2, s.3)).collect(); + pinned.sort(); + sides.sort(); + assert_eq!(pinned, sides, "a pinned side with no derived edge"); +} + /// Clause 1, `n1_is_today`: one Spark's machine is [`plan_on`] the GB10, /// and its plan is today's: held whole, node 0's bytes are today's pool sum /// ([`PoolNeed`]), or both refuse it by the same bytes; paged, node 0 holds @@ -818,15 +1193,23 @@ fn read_balance() { } /// Clause 4, `link_share`: the chosen plan's link term is at most a tenth of -/// its step, and a node is dropped for it exactly where criterion 2 fires, -/// as §9.8's table names: Qwen3.8 at N = 4, λ 70, A low, which then uses -/// two of the four nodes. The share is read on the exact steps: three ids -/// of 1,000 B at top-k 1 read at 10¹¹ B/s, λ 17,000 ps, split [2, 1], -/// expose 5,666⅔ of 55,666⅔ ps (10.2 %), and three of 19,199 B at -/// 10¹² B/s, λ 32,000 ps, expose 10,666⅔ of 106,661⅔ ps, past a tenth by -/// ½ ps. +/// its step, and a node dropped for it would pass the tenth, read on its +/// whole nanoseconds ([`past_tenth`]). [`expected_sets`] holds the nodes +/// dropped for it to the derived cells where criterion 2 fires; here, a plan +/// that drops one uses two nodes, and Qwen3.8 at N = 4, λ 70, A low is among +/// them. The share is read on the exact steps: three ids of 1,000 B at +/// top-k 1 read at 10¹¹ B/s, λ 17,000 ps, split [2, 1], expose 5,666⅔ of +/// 55,666⅔ ps (10.2 %), and three of 19,199 B at 10¹² B/s, λ 32,000 ps, +/// expose 10,666⅔ of 106,661⅔ ps, past a tenth by ½ ps. #[test] fn link_share() { + let dropped_past = |at: &dyn std::fmt::Debug, plan: &NodePlan<'_>| { + for i in plan.idle() { + if let IdleWhy::LinkShare { link_ns, step_ns } = i.why { + assert!(past_tenth(link_ns, step_ns), "{at:?}: {i:?}"); + } + } + }; let mut fired = Vec::new(); each_cell(RULER, 3_000, |cell, plan| { let step = plan.step_ps().expect("a step"); @@ -835,6 +1218,7 @@ fn link_share() { link.times(10).expect("ten links") <= step, "{cell:?}: link {link:?} of step {step:?}" ); + dropped_past(&cell, plan); if plan .idle() .iter() @@ -844,7 +1228,7 @@ fn link_share() { assert_eq!(used(plan), 2, "{cell:?}"); } }); - assert_eq!(fired, vec![("Qwen3.8", 4, 70, false)]); + assert!(fired.contains(&("Qwen3.8", 4, 70, false)), "{fired:?}"); let model = tiny(3, 1, 1_000); let machine = spark(GB10_POOL, 1); let link = LinkCost { round_trip_ns: 17 }; @@ -884,21 +1268,26 @@ fn link_share() { } }] ); + dropped_past(&"19,199 B ids", &plan); } -/// Node 0's NIC, in permille, on a plan of `owned` ids: a token's mean -/// rows, one a slot on a remote, kept a fraction, against the decode step -/// and a prompt batch at S₁ times the one-node token's bytes over A and the -/// slowest node's FFN, node 0's with Sh. -fn nic_of(shape: Shape, owned: &[u64], step_ns: u64, s1: u64) -> u128 { +/// Node 0's bytes a token on a plan of `owned` ids, times the experts: the +/// mean rows of its slots on the remotes, one a slot, kept a fraction. +fn nic_bytes(shape: Shape, owned: &[u64]) -> u128 { + let layers = shape.layers as u64; + let k = u128::from(shape.used); + let remote: u128 = owned[1..].iter().map(|&o| u128::from(o)).sum(); + u128::from(layers * 4 * shape.n_embd) * k * remote +} + +/// Node 0's NIC share of a prompt batch on such a plan at S₁ `s1`, a NIC of +/// one byte a second: S₁ times the one-node token's bytes over A and the +/// slowest node's FFN, node 0's with Sh, as a numerator and a denominator. +fn nic_prompt(shape: Shape, owned: &[u64], s1: u64) -> (u128, u128) { let (a, sh, e) = shape.bytes(false); let layers = shape.layers as u64; let (sh, id) = (u128::from(sh * layers), u128::from(e * layers)); let (k, ex) = (u128::from(shape.used), u128::from(shape.experts)); - let remote: u128 = owned[1..].iter().map(|&o| u128::from(o)).sum(); - let bytes = u128::from(layers * 4 * shape.n_embd) * k * remote; - let nic = u128::from(GB10_NIC); - let decode = bytes * PS * 1000 / (u128::from(step_ns) * 1000 * nic * ex); let ffn = owned .iter() .enumerate() @@ -906,33 +1295,43 @@ fn nic_of(shape: Shape, owned: &[u64], step_ns: u64, s1: u64) -> u128 { .max() .unwrap_or(0); let one = u128::from(a) + sh + k * id; - let prompt = u128::from(s1) * one * bytes * 1000 / ((u128::from(a) * ex + ffn) * nic); - decode.max(prompt) + ( + u128::from(s1) * one * nic_bytes(shape, owned), + u128::from(a) * ex + ffn, + ) +} + +/// Node 0's NIC, in permille, on such a plan: against the decode step and +/// the prompt batch, the larger. +fn nic_of(shape: Shape, owned: &[u64], step_ns: u64, s1: u64) -> u128 { + let nic = u128::from(GB10_NIC); + let ex = u128::from(shape.experts); + let decode = nic_bytes(shape, owned) * PS * 1000 / (u128::from(step_ns) * 1000 * nic * ex); + let (num, den) = nic_prompt(shape, owned, s1); + decode.max(num * 1000 / (den * nic)) } /// Clause 5, `nic_not_max`: a plan whose NIC passes half a decode step or a -/// prompt batch names the node and its busy share, which is the node's -/// reported one and the busiest, exactly where criterion 5 fires on the -/// chosen plan at its split: node 0 takes the rows of the remotes' share of -/// the slots. At S₁ 3,000 that is V4.1 and GLM at N ≥ 2 but three of -/// V4.1's four cells at N = 2 (all but λ 25, A low, on the line at 500 ‰), -/// and never Qwen3.8, whose node 0 keeps the most ids; none at 1,000. -/// §9.8's table, from an even split, names N ≥ 3 for all three. The prompt -/// counts the split's own shares: a remote that owns every id gains the -/// prompt only Sh. A token's mean bytes stay a fraction: three ids a layer -/// at top-k 1, owned [1, 2], take 8/3 B a token, two thirds of a 4 B row, -/// and node 0's prompt share is 4/7, past the line. The decode line reads -/// the exact step: the same split of ids of 666,666,664 B read at 10¹⁰ B/s, -/// a NIC of 16 B/s and λ 2,000 ps take a step of 333,333,333,333⅓ ps, twice -/// node 0's NIC time of 166,666,666,666⅔ ps, so on the line. Paged, each -/// window's read and fill round down once, together: forty ids of -/// 649,500,000 B at top-k 1 read at 10¹⁹ B/s on two nodes of one id's room -/// each, at a hit rate of 999 ‰ and a miss's fill of 1,000 ps, split -/// [20, 20], take a step of 325.7 ps that reads 324 ps (a read of 64.95 ps -/// and a fill of 19/20 ps a window). A row of 647 values at 8 × 10¹² B/s -/// gives node 0 a NIC time of 161.75 ps, half of 323.5 ps, 2.2 ps under the -/// step and so past the 2 ps a one-layer step can read low: no node is -/// named. +/// prompt batch names the node and its busy share, which is the node's reported +/// one and the busiest, exactly in the derived cells where criterion 5 fires on +/// the chosen plan at its split: node 0 takes the rows of the remotes' share of +/// the slots. On the line the side is the one [`expected_sets`] pins: V4.1 at +/// N = 2, λ 25, A low fires at S₁ 3,000. A share of exactly one half reads +/// 500 ‰ and does not fire, at ≤ and not <, and one just past it reads 500 ‰ +/// and fires. The prompt counts the split's own shares: a remote that owns +/// every id gains the prompt only Sh. A token's mean bytes stay a fraction: +/// three ids a layer at top-k 1, owned [1, 2], take 8/3 B a token, two thirds +/// of a 4 B row, and node 0's prompt share is 4/7, past the line. The decode +/// line reads the exact step: the same split of ids of 666,666,664 B read at +/// 10¹⁰ B/s, a NIC of 16 B/s and λ 2,000 ps take a step of +/// 333,333,333,333⅓ ps, twice node 0's NIC time of 166,666,666,666⅔ ps, so +/// on the line. Paged, each window's read and fill round down once, together: +/// forty ids of 649,500,000 B at top-k 1 read at 10¹⁹ B/s on two nodes of one +/// id's room each, at a hit rate of 999 ‰ and a miss's fill of 1,000 ps, split +/// [20, 20], take a step of 325.7 ps that reads 324 ps (a read of 64.95 ps and a +/// fill of 19/20 ps a window). A row of 647 values at 8 × 10¹² B/s gives node 0 +/// a NIC time of 161.75 ps, half of 323.5 ps, 2.2 ps under the step and so past +/// the 2 ps a one-layer step can read low: no node is named. #[test] fn nic_not_max() { for rate in PROMPT_RATES { @@ -955,9 +1354,7 @@ fn nic_not_max() { } None => assert!(busiest <= 500, "{cell:?}: {:?}", p.per_node), } - let below = - cell.0 == "V4.1" && cell.1 == 2 && cell.2 == 70 || cell == ("V4.1", 2, 25, true); - if rate == 3_000 && cell.1 >= 2 && cell.0 != "Qwen3.8" && !below { + if derived_at(cell, RULER, rate).fires { expected.push(cell); } }); @@ -995,6 +1392,27 @@ fn nic_not_max() { p.per_node[0].nic_permille ); assert_eq!(plan.nic_bound().expect("a prediction"), None); + let (_, den) = nic_prompt(shape, plan.owned(), 1); + let s1 = u64::try_from(den).expect("an S₁"); + let (num, _) = nic_prompt(shape, plan.owned(), s1); + let line = u64::try_from(2 * num / den).expect("a NIC rate"); + let at = |nic: u64| { + let mut nodes = spark_nodes(2, shape.layers, fabric(2, LAMBDAS[0])); + nodes.nic = nic; + let budgets = budgets_on(&model, &mut nodes, &machines); + let whole = Workload { + prompt_rate: s1, + ..whole + }; + let plan = plan_nodes(&nodes, &terms, &budgets, whole).expect("two Sparks"); + let bound = plan.nic_bound().expect("a prediction"); + ( + plan.predict().expect("a prediction").per_node[0].nic_permille, + bound.map(|b| b.node), + ) + }; + assert_eq!(at(line), (500, None), "a NIC at exactly half"); + assert_eq!(at(line - 1), (500, Some(0)), "a NIC just past half"); let id = 1_000_000_000; let model = tiny(3, 1, id); let link = LinkCost { round_trip_ns: 0 }; @@ -1071,7 +1489,8 @@ fn nic_not_max() { /// slot, and their posts take at most 2 % of the layer. Also where k is /// under N − 1: one expert a token of seven, two ids a pool, on four Sparks /// the pools all need, where a token's handoff goes only to the node its -/// slot lands on, and the step and node 0's NIC are that rule's. +/// slot lands on, and the step and node 0's NIC are that rule's. In every +/// cell node 0's count is the derived cell's. #[test] fn messages_o_n() { let check = |cell: Cell, plan: &NodePlan<'_>| { @@ -1090,7 +1509,15 @@ fn messages_o_n() { "{cell:?}: {busiest} messages a layer" ); }; - each_cell(RULER, 3_000, check); + each_cell(RULER, PROMPT_RATES[1], |cell, plan| { + check(cell, plan); + let p = plan.predict().expect("a prediction"); + assert_eq!( + p.per_node[0].msgs_per_layer, + derived_at(cell, RULER, PROMPT_RATES[1]).msgs, + "{cell:?}: node 0's messages" + ); + }); let shape = Shape { name: "k1", experts: 7, @@ -1130,16 +1557,16 @@ fn messages_o_n() { ); } -/// Clause 7, `never_slower`: a node is named idle for its gain exactly in -/// §9.8's criterion 7 cells but Qwen3.8 at N = 4, λ 70, A low, where -/// criterion 2 names it first, and no chosen plan is slower than the N − 1 -/// plan on the same link: a pair for a ring of three, three switched Sparks -/// for four. The ruler is read from the machine: on one whose ruler is 4 %, -/// Qwen3.8 names node 1 idle at N = 2 where it gains under 8 %, at λ 70. -/// The gain is read on the exact steps: three ids of 5,000 B at top-k 2 -/// read at 10¹² B/s take 30,000 ps on one node and 27,777⁷⁄₉ ps on two, a -/// gain of 7.4 %, under a ruler of 5 % twice, and read at 10¹⁴ B/s 300 and -/// 277⁷⁄₉ ps, a gain of 7.41 %, under a ruler of 3.8 % twice. +/// Clause 7, `never_slower`: no chosen plan is slower than the N − 1 plan on +/// the same link: a pair for a ring of three, three switched Sparks for four. +/// [`expected_sets`] holds the nodes named idle for their gain to the derived +/// cells where criterion 7 fires. The ruler is read from the machine: on one +/// whose ruler is 4 %, Qwen3.8 names node 1 idle at N = 2 where it gains +/// under 8 %, at λ 70, and at `RULER` it does not. The gain is read on the +/// exact steps: three ids of 5,000 B at top-k 2 read at 10¹² B/s take +/// 30,000 ps on one node and 27,777⁷⁄₉ ps on two, a gain of 7.4 %, under a +/// ruler of 5 % twice, and read at 10¹⁴ B/s 300 and 277⁷⁄₉ ps, a gain of +/// 7.41 %, under a ruler of 3.8 % twice. #[test] fn never_slower() { let gain = |plan: &NodePlan<'_>| { @@ -1150,7 +1577,7 @@ fn never_slower() { let mut fired = Vec::new(); let mut slower = Vec::new(); each_cell(RULER, 3_000, |cell, plan| { - if gain(plan) { + if cell.0 == "Qwen3.8" && cell.1 == 2 && gain(plan) { fired.push(cell); } let (_, n, lambda, _) = cell; @@ -1172,15 +1599,7 @@ fn never_slower() { } }); let qwen = |n, lambda, high| ("Qwen3.8", n, lambda, high); - assert_eq!( - fired, - vec![ - qwen(4, 25, false), - qwen(4, 25, true), - qwen(3, 70, true), - qwen(4, 70, true), - ] - ); + assert_eq!(fired, vec![]); assert_eq!(slower, vec![]); let mut wide = Vec::new(); each_cell(4 * RULER, 3_000, |cell, plan| { diff --git a/crates/placement/tests/nodes_scale_cells.tsv b/crates/placement/tests/nodes_scale_cells.tsv new file mode 100644 index 000000000..73352603a --- /dev/null +++ b/crates/placement/tests/nodes_scale_cells.tsv @@ -0,0 +1,146 @@ +# nodes_scale's expected cells (docs/contrib/dgx-spark.md §9.9), written by tools/flow/nodes_scale.py --write from nodes_scale_inputs.tsv; do not edit. +model a n lambda_us ruler s1 used idle paged paged_nodes step_ns step_tol nic nic_tol fires msgs edge +V4.1 low 1 25 10 1000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 25 10 1000 2 - 62 11,51 30612002 1750 167 3 0 2 - +V4.1 low 3 25 10 1000 3 - 0 0,0,0 27747601 585 256 3 0 4 - +V4.1 low 4 25 10 1000 4 - 0 0,0,0,0 26265584 1510 312 3 0 6 - +V4.1 low 1 70 10 1000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 70 10 1000 2 - 62 28,34 31422423 339 149 3 0 2 - +V4.1 low 3 70 10 1000 3 - 0 0,0,0 28986815 904 227 3 0 4 - +V4.1 low 4 70 10 1000 4 - 0 0,0,0,0 27660087 2483 277 3 0 6 - +V4.1 low 1 25 10 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 25 10 3000 2 - 62 11,51 30612002 1750 501 5 1 2 nic +V4.1 low 3 25 10 3000 3 - 0 0,0,0 27747601 585 769 6 1 4 - +V4.1 low 4 25 10 3000 4 - 0 0,0,0,0 26265584 1510 936 6 1 6 - +V4.1 low 1 70 10 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 70 10 3000 2 - 62 28,34 31422423 339 446 5 0 2 - +V4.1 low 3 70 10 3000 3 - 0 0,0,0 28986815 904 680 5 1 4 - +V4.1 low 4 70 10 3000 4 - 0 0,0,0,0 27660087 2483 830 6 1 6 - +V4.1 low 1 25 40 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 25 40 3000 2 - 62 11,51 30612002 1750 501 5 1 2 nic +V4.1 low 3 25 40 3000 3 - 0 0,0,0 27747601 585 769 6 1 4 - +V4.1 low 4 25 40 3000 3 3:gain 0 0,0,0,0 27792333 575 766 6 1 4 - +V4.1 low 1 70 40 3000 1 - 231 231 38429145 2 0 2 0 0 - +V4.1 low 2 70 40 3000 2 - 62 28,34 31422423 339 446 5 0 2 - +V4.1 low 3 70 40 3000 3 - 0 0,0,0 28986815 904 680 5 1 4 - +V4.1 low 4 70 40 3000 3 3:gain 0 0,0,0,0 29024289 956 677 5 1 4 - +V4.1 high 1 25 10 1000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 25 10 1000 2 - 67 0,67 44522408 12728 163 2 0 2 - +V4.1 high 3 25 10 1000 3 - 0 0,0,0 40991983 1460 264 3 0 4 - +V4.1 high 4 25 10 1000 4 - 0 0,0,0,0 39358988 4479 314 3 0 6 - +V4.1 high 1 70 10 1000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 70 10 1000 2 - 67 0,67 45557256 3135 163 2 0 2 - +V4.1 high 3 70 10 1000 3 - 0 0,0,0 42448939 1081 239 3 0 4 - +V4.1 high 4 70 10 1000 4 - 0 0,0,0,0 40883420 3798 285 3 0 6 - +V4.1 high 1 25 10 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 25 10 3000 2 - 67 0,67 44522408 12728 489 4 0 2 - +V4.1 high 3 25 10 3000 3 - 0 0,0,0 40991983 1460 793 5 1 4 - +V4.1 high 4 25 10 3000 4 - 0 0,0,0,0 39358988 4479 943 5 1 6 - +V4.1 high 1 70 10 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 70 10 3000 2 - 67 0,67 45557256 3135 489 4 0 2 - +V4.1 high 3 70 10 3000 3 - 0 0,0,0 42448939 1081 718 5 1 4 - +V4.1 high 4 70 10 3000 4 - 0 0,0,0,0 40883420 3798 856 5 1 6 - +V4.1 high 1 25 40 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 25 40 3000 2 - 67 0,67 44522408 12728 489 4 0 2 - +V4.1 high 3 25 40 3000 3 - 0 0,0,0 40991983 1460 793 5 1 4 - +V4.1 high 4 25 40 3000 3 3:gain 0 0,0,0,0 41044095 1519 791 5 1 4 - +V4.1 high 1 70 40 3000 1 - 236 236 54233834 2 0 2 0 0 - +V4.1 high 2 70 40 3000 2 - 67 0,67 45557256 3135 489 4 0 2 - +V4.1 high 3 70 40 3000 3 - 0 0,0,0 42448939 1081 718 5 1 4 - +V4.1 high 4 70 40 3000 3 3:gain 0 0,0,0,0 42493096 1017 716 5 1 4 - +GLM low 1 25 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 25 10 1000 2 - 0 0,0 55158319 12570 184 3 0 2 - +GLM low 3 25 10 1000 3 - 0 0,0,0 50939235 878 282 3 0 4 - +GLM low 4 25 10 1000 4 - 0 0,0,0,0 48718290 8205 335 3 0 6 - +GLM low 1 70 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 70 10 1000 2 - 0 0,0 56206586 4233 177 3 0 2 - +GLM low 3 70 10 1000 3 - 0 0,0,0 52318199 749 259 3 0 4 - +GLM low 4 70 10 1000 4 - 0 0,0,0,0 50299739 4173 307 3 0 6 - +GLM low 1 25 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 25 10 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM low 3 25 10 3000 3 - 0 0,0,0 50939235 878 847 7 1 4 - +GLM low 4 25 10 3000 4 - 0 0,0,0,0 48718290 8205 1004 7 1 6 - +GLM low 1 70 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 70 10 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM low 3 70 10 3000 3 - 0 0,0,0 52318199 749 776 6 1 4 - +GLM low 4 70 10 3000 4 - 0 0,0,0,0 50299739 4173 920 7 1 6 - +GLM low 1 25 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 25 40 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM low 3 25 40 3000 2 2:gain 0 0,0,0 55158319 12570 551 6 1 2 - +GLM low 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 55180155 12094 551 6 1 2 - +GLM low 1 70 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM low 2 70 40 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM low 3 70 40 3000 2 2:gain 0 0,0,0 56206586 4233 531 5 1 2 - +GLM low 4 70 40 3000 2 2:gain,3:gain 0 0,0,0,0 56244746 4184 529 5 1 2 - +GLM high 1 25 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 25 10 1000 2 - 0 0,0 55158319 12570 184 3 0 2 - +GLM high 3 25 10 1000 3 - 0 0,0,0 50939235 878 282 3 0 4 - +GLM high 4 25 10 1000 4 - 0 0,0,0,0 48718290 8205 335 3 0 6 - +GLM high 1 70 10 1000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 70 10 1000 2 - 0 0,0 56206586 4233 177 3 0 2 - +GLM high 3 70 10 1000 3 - 0 0,0,0 52318199 749 259 3 0 4 - +GLM high 4 70 10 1000 4 - 0 0,0,0,0 50299739 4173 307 3 0 6 - +GLM high 1 25 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 25 10 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM high 3 25 10 3000 3 - 0 0,0,0 50939235 878 847 7 1 4 - +GLM high 4 25 10 3000 4 - 0 0,0,0,0 48718290 8205 1004 7 1 6 - +GLM high 1 70 10 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 70 10 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM high 3 70 10 3000 3 - 0 0,0,0 52318199 749 776 6 1 4 - +GLM high 4 70 10 3000 4 - 0 0,0,0,0 50299739 4173 920 7 1 6 - +GLM high 1 25 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 25 40 3000 2 - 0 0,0 55158319 12570 551 6 1 2 - +GLM high 3 25 40 3000 2 2:gain 0 0,0,0 55158319 12570 551 6 1 2 - +GLM high 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 55180155 12094 551 6 1 2 - +GLM high 1 70 40 3000 1 - 145 145 67370273 2 0 2 0 0 - +GLM high 2 70 40 3000 2 - 0 0,0 56206586 4233 531 5 1 2 - +GLM high 3 70 40 3000 2 2:gain 0 0,0,0 56206586 4233 531 5 1 2 - +GLM high 4 70 40 3000 2 2:gain,3:gain 0 0,0,0,0 56244746 4184 529 5 1 2 - +Qwen3.8 low 1 25 10 1000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 25 10 1000 2 - 0 0,0 21798149 223 112 2 0 2 - +Qwen3.8 low 3 25 10 1000 3 - 0 0,0,0 20955294 226 155 2 0 4 - +Qwen3.8 low 4 25 10 1000 3 3:gain 0 0,0,0,0 20997033 258 153 2 0 4 - +Qwen3.8 low 1 70 10 1000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 70 10 1000 2 - 0 0,0 22798306 1442 73 2 0 2 - +Qwen3.8 low 3 70 10 1000 3 - 0 0,0,0 22311790 1629 105 2 0 4 link:2:99956:78 +Qwen3.8 low 4 70 10 1000 2 2:link,3:link 0 0,0,0,0 22823812 1499 72 2 0 2 - +Qwen3.8 low 1 25 10 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 25 10 3000 2 - 0 0,0 21798149 223 335 3 0 2 - +Qwen3.8 low 3 25 10 3000 3 - 0 0,0,0 20955294 226 466 3 0 4 - +Qwen3.8 low 4 25 10 3000 3 3:gain 0 0,0,0,0 20997033 258 460 3 0 4 - +Qwen3.8 low 1 70 10 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 70 10 3000 2 - 0 0,0 22798306 1442 220 3 0 2 - +Qwen3.8 low 3 70 10 3000 3 - 0 0,0,0 22311790 1629 315 3 0 4 link:2:99956:78 +Qwen3.8 low 4 70 10 3000 2 2:link,3:link 0 0,0,0,0 22823812 1499 217 3 0 2 - +Qwen3.8 low 1 25 40 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 25 40 3000 2 - 0 0,0 21798149 223 335 3 0 2 - +Qwen3.8 low 3 25 40 3000 2 2:gain 0 0,0,0 21798149 223 335 3 0 2 - +Qwen3.8 low 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 21835515 259 331 3 0 2 - +Qwen3.8 low 1 70 40 3000 1 - 0 0 24239786 2 0 2 0 0 - +Qwen3.8 low 2 70 40 3000 1 1:gain 0 0,0 24239786 2 0 2 0 0 - +Qwen3.8 low 3 70 40 3000 1 1:gain,2:gain 0 0,0,0 24239786 2 0 2 0 0 - +Qwen3.8 low 4 70 40 3000 1 1:gain,2:gain,3:gain 0 0,0,0,0 24239786 2 0 2 0 0 - +Qwen3.8 high 1 25 10 1000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 25 10 1000 2 - 0 0,0 26587556 223 109 2 0 2 - +Qwen3.8 high 3 25 10 1000 3 - 0 0,0,0 25744701 226 150 2 0 4 - +Qwen3.8 high 4 25 10 1000 3 3:gain 0 0,0,0,0 25786439 258 148 2 0 4 - +Qwen3.8 high 1 70 10 1000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 70 10 1000 2 - 0 0,0 27587713 1442 72 2 0 2 - +Qwen3.8 high 3 70 10 1000 2 2:gain 0 0,0,0 27587713 1442 72 2 0 2 - +Qwen3.8 high 4 70 10 1000 2 2:gain,3:gain 0 0,0,0,0 27613219 1499 71 2 0 2 - +Qwen3.8 high 1 25 10 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 25 10 3000 2 - 0 0,0 26587556 223 327 3 0 2 - +Qwen3.8 high 3 25 10 3000 3 - 0 0,0,0 25744701 226 449 3 0 4 - +Qwen3.8 high 4 25 10 3000 3 3:gain 0 0,0,0,0 25786439 258 444 3 0 4 - +Qwen3.8 high 1 70 10 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 70 10 3000 2 - 0 0,0 27587713 1442 217 3 0 2 - +Qwen3.8 high 3 70 10 3000 2 2:gain 0 0,0,0 27587713 1442 217 3 0 2 - +Qwen3.8 high 4 70 10 3000 2 2:gain,3:gain 0 0,0,0,0 27613219 1499 213 3 0 2 - +Qwen3.8 high 1 25 40 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 25 40 3000 2 - 0 0,0 26587556 223 327 3 0 2 - +Qwen3.8 high 3 25 40 3000 2 2:gain 0 0,0,0 26587556 223 327 3 0 2 - +Qwen3.8 high 4 25 40 3000 2 2:gain,3:gain 0 0,0,0,0 26624922 259 323 3 0 2 - +Qwen3.8 high 1 70 40 3000 1 - 0 0 29029193 2 0 2 0 0 - +Qwen3.8 high 2 70 40 3000 1 1:gain 0 0,0 29029193 2 0 2 0 0 - +Qwen3.8 high 3 70 40 3000 1 1:gain,2:gain 0 0,0,0 29029193 2 0 2 0 0 - +Qwen3.8 high 4 70 40 3000 1 1:gain,2:gain,3:gain 0 0,0,0,0 29029193 2 0 2 0 0 - diff --git a/crates/placement/tests/nodes_scale_inputs.tsv b/crates/placement/tests/nodes_scale_inputs.tsv new file mode 100644 index 000000000..b16b4cbd3 --- /dev/null +++ b/crates/placement/tests/nodes_scale_inputs.tsv @@ -0,0 +1,53 @@ +# nodes_scale's inputs (docs/contrib/dgx-spark.md §9.9): the rows tools/flow/nodes_scale.py derives the gate's cells from, and the +# rows the gate holds its own constants and budgets to. A model's rows are /. kind: measured (a recorded run), +# derived (arithmetic on recorded numbers, code or datasheets), assumed (no record). Every value is a whole number. +name value unit kind source +read 208800000000 B/s measured BW, our kernels' pool read on one Spark: §5 and §9.7 +nic 23700000000 B/s derived a Spark's NIC a direction, the SoC side: §9.1 +pool 115284164096 B derived a node's pool: a GB10 the census reads with 115 GB free (§9.7 U), as NodeBudget::pool reads its card plan +lambda_lo 25000 ns derived λ's band, the exposed round trip a layer: §9.7 +lambda_hi 70000 ns derived λ's band: §9.7 +hop 1500 ns assumed the switch's hop at N = 4: §9.7 +post 1000 ns assumed a further post of node 0's pump: §9.7, §9.8 criterion 6 +row_value 4 B derived a value of the handoff and the rows, one f32 +vocab 151552 rows assumed the synthetic shapes' head rows +ruler 10 permille derived the box's 95 % interval at four rounds: AGENTS.md, "Know the ruler" +ruler_wide 40 permille assumed a machine whose ruler is 4 %: §9.9 M7 +s1_lo 1000 tok/s assumed S₁ at the low end of §9.8's pins +s1_hi 3000 tok/s assumed S₁ at the high end of §9.8's pins +V4.1/layers 40 layers derived §9.7 model terms +V4.1/n_embd 5120 values derived §9.7 model terms +V4.1/experts 384 experts derived §9.7 model terms +V4.1/top_k 6 slots derived §9.7 model terms +V4.1/expert 16770000 B derived §9.7 model terms, an expert's bytes +V4.1/shared_lo 800000000 B derived §9.7 Sh, the band's low end +V4.1/shared_hi 1500000000 B derived §9.7 Sh, the band's high end +V4.1/serial_lo 3200000000 B derived §9.7 A, the band's low end +V4.1/serial_hi 5800000000 B derived §9.7 A, the band's high end +V4.1/fixed0_lo 12286314624 B derived node 0's bytes beside the routed ids at A low: NodeBudget::fixed of the shape's card plan, KV 2 GB at 4,096 positions +V4.1/fixed0_hi 15555949824 B derived the same at A high +V4.1/fixed1 1744456576 B derived a remote's bytes beside its routed ids: NodeBudget::fixed of the routed stacks' card plan, no KV +GLM/layers 42 layers derived §9.7 model terms +GLM/n_embd 4096 values derived §9.7 model terms +GLM/experts 288 experts derived §9.7 model terms +GLM/top_k 8 slots derived §9.7 model terms +GLM/expert 15200000 B derived §9.7 model terms, an expert's bytes +GLM/shared_lo 1600000000 B derived §9.7 Sh, the UD-Q4_K_XL header read +GLM/shared_hi 1600000000 B derived §9.7 Sh, the UD-Q4_K_XL header read +GLM/serial_lo 7360000000 B derived §9.7 A, the UD-Q4_K_XL header read +GLM/serial_hi 7360000000 B derived §9.7 A, the UD-Q4_K_XL header read +GLM/fixed0_lo 23768122598 B derived node 0's bytes beside the routed ids: NodeBudget::fixed of the shape's card plan, KV 2 GB at 4,096 positions +GLM/fixed0_hi 23768122598 B derived the same at A high +GLM/fixed1 1747080960 B derived a remote's bytes beside its routed ids: NodeBudget::fixed of the routed stacks' card plan, no KV +Qwen3.8/layers 48 layers derived §9.7 model terms +Qwen3.8/n_embd 2560 values derived §9.7 model terms +Qwen3.8/experts 512 experts derived §9.7 model terms +Qwen3.8/top_k 10 slots derived §9.7 model terms +Qwen3.8/expert 3170000 B derived §9.7 model terms, an expert's bytes +Qwen3.8/shared_lo 140000000 B derived §9.7 Sh +Qwen3.8/shared_hi 140000000 B derived §9.7 Sh +Qwen3.8/serial_lo 3400000000 B derived §9.7 A, the band's low end +Qwen3.8/serial_hi 4400000000 B derived §9.7 A, the band's high end +Qwen3.8/fixed0_lo 8341707616 B derived node 0's bytes beside the routed ids at A low: NodeBudget::fixed of the shape's card plan, KV 2 GB at 4,096 positions +Qwen3.8/fixed0_hi 9348340576 B derived the same at A high +Qwen3.8/fixed1 1698955264 B derived a remote's bytes beside its routed ids: NodeBudget::fixed of the routed stacks' card plan, no KV diff --git a/docs/contrib/dgx-spark.md b/docs/contrib/dgx-spark.md index 42bf11107..ac50e55b7 100644 --- a/docs/contrib/dgx-spark.md +++ b/docs/contrib/dgx-spark.md @@ -749,7 +749,9 @@ now written beside them are not fed back into the tables. - The (a) figures come from a Monte Carlo of the per-layer window: each routed slot lands on node j with the ownership probability the rule picks, and the window is the max over the nodes of `Sh + slots·e` (node 0) or `λ + hop + posts + rows ingress + slots·e` (a remote). Node 0's share is optimized: 0.55–0.72 at N = 2 for Qwen3.8 and 0.07–0.15 at N = 4 - for GLM and V4.1. The script moves into `tools/` beside the gate (9.9). + for GLM and V4.1. The script moves into `tools/` beside the gate (9.9). The gate's cells and the (a) rows at the + gate's inputs below come from `tools/flow/nodes_scale.py` instead, which sums over the count tuples of a token's slots + rather than sampling them (9.9). **Model terms** `[derived]` from each model's tensor table (file sizes: `docs/models.md`): @@ -784,10 +786,19 @@ routed share] → join`. The next layer's attention needs the join, so node 1 id | N | V4.1 (needs about 268 GB resident) | GLM (about 203 GB) | Qwen3.8 (about 114 GB) | |---|---|---|---| -| 1 | no: about 154 GB of routed on NVMe | no: about 88 GB on NVMe | yes, marginal | -| 2 | no: about 43 GB on NVMe ((b): about 19 GB a node) | yes, about 25 GB spare | yes | +| 1 | no: ~~about 154 GB of routed on NVMe~~ node 0's pool has room for 154 of the 384 routed ids at 4,096 positions and 153 at 32,768, so 230 (155.0 GB) and 231 (155.7 GB) do not fit | no: ~~about 88 GB on NVMe~~ 128 of 288 ids on NVMe, 82.4 GB | yes, marginal | +| 2 | no: about 43 GB on NVMe ((b): about 19 GB a node) | yes, ~~about 25 GB spare~~ every id held: the pools have room for 160 and 176 of the 288 ids | yes | | 4 | yes, about 190 GB spare ((b) about 67 GB a stage; (c) adds 6 GB of dense a node) | yes ((c) adds 11.4 GB of dense a node) | yes | +The cells that replace the struck figures are `[derived]` from the files' headers (V4.1 `Q3_K_M`, GLM `UD-Q4_K_XL`) +at `cb5688c9`, with node 0's card plan of the file and a pool a node of the 115 GB that 9.7 assumes, the same at 4,096 +and 32,768 positions where no position count is named. GLM's are the plans of `plan_nodes` +([#95](https://github.com/midagedev/bloomery/pull/95)). The node plans refuse the V4.1 file at every N: node 0's card +plan leaves its engram tables (84.5 GB) on the NVMe tier and reads 5,280 B of them a token, and `Terms::of` refuses +such rows by name (9.9, Terms). V4.1's cell at N = 1 comes from the room node 0's budget (`NodeBudget::of`) leaves for +routed ids: the pool the plan reads from those 115 GB, 115.28 GB, less its fixed bytes (10.86 GB at 4,096 positions, +12.14 GB at 32,768), over 673.9 MB an id. + For Qwen3.8 at N = 2 under (a), node 0 holds about 75 GB (4.5 non-routed + 28.8 PLE + about 39 routed + KV) against about 114 GB on one node, and node 1 about 40 GB. @@ -839,6 +850,26 @@ that binds:** | (b) pipeline | one boundary (16·n_embd B: 82, 66, 41 KB) and a token id | N − 1 boundaries and a token id | spread: each NIC one in, one out | | (c) EP | as (a) with uniform ownership | as (a) | spread: each node at most 4(N−1) a layer | +**(a) at the gate's inputs** `[derived]` by `tools/flow/nodes_scale.py --table` (9.9): BW 208.8 GB/s, the 23.7 GB/s +NIC, a 115 GB pool a node, the gate's synthetic shapes (GLM's A and Sh from the header read), λ from 25 to 70 µs and A +from low to high, one request. N is the Sparks offered; the plan may leave some of them idle (criteria 2 and 7 of 9.8), +and node 0's share is the split's, not an even one. Decode carries no NVMe term, so for a model that pages it is the +pool's time only. The prompt is the Amdahl term above with the slowest node's FFN at the split, and the NIC cap is node +0's rows ingress at the split's remote share. Messages count a handoff and its rows for each remote a token reaches. + +| (a) star at the gate's inputs | N | Nodes used | Decode, one request (tok/s) | Prompt (× S₁) | Node 0's NIC ingress cap (tok/s) | Link bytes a token (MB) | Messages a token | +|---|---|---|---|---|---|---|---| +| V4.1 | 2 | 2 | 22.0 to 32.7 (pool only: pages 62 to 67 ids) | 1.28 to 1.40 | 7,846 to 9,140 | 3.40 to 3.84 | 79 to 80 | +| V4.1 | 4 | 4 | 24.5 to 38.1 | 1.49 to 1.75 | 4,895 to 6,031 | 6.00 to 7.07 | 203 to 218 | +| GLM | 2 | 2 | 17.8 to 18.1 | 1.28 to 1.29 | 7,005 to 7,216 | 3.97 to 4.07 | 84 | +| GLM | 4 | 4 | 19.9 to 20.5 | 1.48 to 1.52 | 4,551 to 4,825 | 6.85 to 7.17 | 237 to 240 | +| Qwen3.8 | 2 | 2 | 36.2 to 45.9 | 1.09 to 1.16 | 10,396 to 15,055 | 2.06 to 2.77 | 94 to 96 | +| Qwen3.8 | 4 | 2 to 3 | 36.2 to 47.6 | 1.09 to 1.22 | 7,968 to 15,269 | 2.03 to 3.93 | 94 to 187 | + +The rest of the decode, prompt and link tables above is not recomputed. The one-node and NVMe rows need ρ and the +drive's rate, (b) a pipeline model, (c) the union R(2), and N requests in flight a batched step; none of them is an +input of the script (section 10). + **One GLM decode layer under (a), one request, N = 2 (midpoints), the resource timeline:** | Resource | Busy | Note | @@ -931,6 +962,22 @@ Nothing fires at S₁ = 1,000. At N = 2 nothing fires except GLM's prompt, which 8,610): the gate's boundary case, at ≤ and not <. The predictor in the tree reads the measured BW and node 0's real split, and differs from this table in criterion 5's column; 9.9 gives its cells. +**Where each fires at the gate's inputs** `[derived]` by `tools/flow/nodes_scale.py --table` (9.9): BW 208.8 GB/s, the +23.7 GB/s NIC, a 115 GB pool a node, GLM's header read, node 0's NIC at the split's own share, and the gate's `RULER`, +10 ‰ (question 1 of [#95](https://github.com/midagedev/bloomery/pull/95) asks whether that is the box's ruler). An N +alone names every λ and A at it. + +| Model | Decode step (ms) at the gate's inputs, N = 1 → 2 → 3 → 4, λ 25 / 70, A low | Criterion 2 fires | Criterion 7 fires | Criterion 5 fires, S₁ 1,000 | Criterion 5 fires, S₁ 3,000 | +|---|---|---|---|---|---| +| V4.1 | 38.4 → 30.6 / 31.4 → 27.7 / 29.0 → 26.3 / 27.7 | never | never | never | N = 2, λ 25, A low; N = 3; N = 4 | +| GLM | 67.4 → 55.2 / 56.2 → 50.9 / 52.3 → 48.7 / 50.3 | never | never | never | N = 2; N = 3; N = 4 | +| Qwen3.8 | 24.2 → 21.8 / 22.8 → 21.0 / 22.3 → 21.0 / 22.8 | N = 4, λ 70, A low | N = 3, λ 70, A high; N = 4, λ 25, A low and high; N = 4, λ 70, A high | never | never | + +Two cells sit on a line within the change one id moved makes. V4.1 at N = 2, λ 25, A low has its busiest NIC at +501 ± 5 ‰ at S₁ 3,000, on the 500 ‰ line, and criterion 5 fires there. Qwen3.8 at N = 3, λ 70, A low has its third +node's link share at 99,956 ± 78 ppm, on the tenth, and the node is used. The cells file names each as an edge in every +row of a ruler and S₁ it is derived at, four rows in all, and the gate pins its side (9.9). + ### 9.9 Reference: the gate, `nodes_scale` `nodes_scale` is the placement half's gate (answer 20), built with the shape of 9.2. It is a set of unit tests in @@ -951,8 +998,18 @@ Where the gate holds a clause differently from its words below, the clause says bytes the split holds them at. A row table left on the NVMe tier is refused by name, since the predictor has no fill term for it. The row width is an input, the model's embedding length. - **Expected sets.** The table of 9.8, pinned from a derivation script that moves into `tools/` beside the gate with its - own self-test, so the pins have one derivation. The script is not in the tree yet. Until it is, the gate pins the - predictor's own cells, which match 9.8's table in criteria 2 and 7 and differ in criterion 5 (clause 5). + own self-test, so the pins have one derivation. ~~The script is not in the tree yet. Until it is, the gate pins the + predictor's own cells, which match 9.8's table in criteria 2 and 7 and differ in criterion 5 (clause 5).~~ The script + is `tools/flow/nodes_scale.py`, with no code or method shared with the planner: node 0's share in real numbers by + 9.2's rule, found by bisection, and each window an expectation over the count tuples of a token's slots, built one + slot at a time. It reads `crates/placement/tests/nodes_scale_inputs.tsv`, one row per input with its kind (measured, + derived or assumed) and source, and writes the 144 cells (a model, N, λ and A, at the gate's `RULER` and S₁ 1,000 and + 3,000 and at a 4 % ruler and S₁ 3,000) to `nodes_scale_cells.tsv` beside it; `--table` prints 9.8's table at the + gate's inputs and 9.7's (a) rows, and `--check` holds the file and both doc tables to them. Each value carries the + largest change one id moved between two used nodes makes, because the planner splits whole ids. Where that band holds + a line (the NIC's half, the link share's tenth, twice the ruler), the cell names an edge, the gate pins its side by + hand, and the planner's value is held to the band besides. `--self-test` runs hand-worked cases, the refusals and + `--check`; `just check-recipes` runs it. - **Clauses:** 1. `n1_is_today`: every N = 1 plan equals today's plan byte for byte. The gate holds it where node 0 holds every id: node 0's bytes are today's pool need, or both plans refuse the model by the same bytes. Paged, it holds node 0 to @@ -965,11 +1022,12 @@ Where the gate holds a clause differently from its words below, the clause says 4. `link_share`: criterion 2 holds on the chosen plan. Qwen3.8 at N = 4, λ 70, A low uses at most 3 nodes. The predictor's plan of that cell uses two of the four: a third switched node exposes 2.270 of 22.351 ms, 10.2 %. 5. `nic_not_max`: criterion 5 either holds or carries the named prompt-mode note, exactly in the cells of the "fires" - column. The gate pins the predictor's cells: at S₁ 3,000, V4.1 and GLM at N ≥ 2, except three of V4.1's four cells - at N = 2, and never Qwen3.8; at 1,000, none. Node 0 takes the rows of the remotes' share as the split sets it, - where 9.8 assumed an even split: GLM at N = 2, λ 25, A low splits its ids [111, 177], and node 0's NIC is busy - 551 ‰. In the cells that fire, the busiest NIC runs from 500 to 941 ‰ for V4.1 and from 531 to 1,007 ‰ for GLM; - Qwen3.8's stays at or below 465 ‰. + column. The gate pins the ~~predictor's~~ derived cells: at S₁ 3,000, V4.1 and GLM at N ≥ 2, except three of V4.1's + four cells at N = 2, and never Qwen3.8; at 1,000, none. The figures that follow are the predictor's; the gate holds + each busiest NIC to the derived one within one id's change. Node 0 takes the rows of the remotes' share as the + split sets it, where 9.8 assumed an even split: GLM at N = 2, λ 25, A low splits its ids [111, 177], and node 0's + NIC is busy 551 ‰. In the cells that fire, the busiest NIC runs from 500 to 941 ‰ for V4.1 and from 531 to 1,007 ‰ + for GLM; Qwen3.8's stays at or below 465 ‰. 6. `messages_o_n`: criterion 6. The busiest node posts 2 × min(k, N − 1) messages a layer, because node 0 sends the handoff only to the remotes that hold one of the token's slots in that layer. 7. `never_slower`: `NodeIdle` names exactly the cells of criterion 7's column, and no chosen plan is slower than its @@ -1015,10 +1073,14 @@ Each open item; the owner is named where one is agreed. An item that a measureme **P5.** - The `[derived]` bands and criteria of 9.7 and 9.8 still use their first inputs. `nodes_scale`'s derivation script, in the placement half, recomputes them (the placement half as proposed in [#13](https://github.com/midagedev/bloomery/issues/13#issuecomment-6134316241), accepted in - answer 20). + answer 20). `tools/flow/nodes_scale.py` (9.9) recomputes 9.8's "where each fires" table and 9.7's (a) rows at the + gate's inputs. 9.7's one-node and NVMe rows, (b), (c) and N requests in flight still use their first inputs: they need + ρ, the NVMe rate, a pipeline model, the union R(2) and a batched step, none of which is an input of the script. +- A fill term in the node plans for rows read from the NVMe tier, so that they can plan the V4.1 file (9.7, 9.9 + Terms). - Where `nodes_scale` ([#95](https://github.com/midagedev/bloomery/pull/95)) holds a clause of 9.9 differently from its - words, for the maintainers to accept or change: until the derivation script lands, it pins the predictor's cells, - which differ from 9.8's table in criterion 5 (clause 5); clause 1 is byte for byte only where node 0 holds every id, + words, for the maintainers to accept or change: it pins the cells `tools/flow/nodes_scale.py` derives, which differ from + 9.8's table as first written in criterion 5 (clause 5); clause 1 is byte for byte only where node 0 holds every id, since today's plan of one Spark pages V4.1 and GLM under a card budget lowered to the NVMe tier's floor (`plan_in_pool`, section 8) and the node plans take node 0's terms before that split; clause 7 compares each plan with the N − 1 plan on the same link, which the planner can choose, not on the N − 1 plan's own fabric. diff --git a/tools/check-recipes.sh b/tools/check-recipes.sh index 0a0e38a51..d1cbb6168 100755 --- a/tools/check-recipes.sh +++ b/tools/check-recipes.sh @@ -473,6 +473,7 @@ blk_pytools() { "tools/check-ptx-names.py --self-test" "tools/check-records.py --self-test" "tools/flow/ds41_prefill.py --self-test" + "tools/flow/nodes_scale.py --self-test" "tools/flow/pplb.py --self-test" "tools/flow/q38_step.py --self-test" "tools/flow/q38width.py --self-test" diff --git a/tools/flow/nodes_scale.py b/tools/flow/nodes_scale.py new file mode 100755 index 000000000..603eea1b7 --- /dev/null +++ b/tools/flow/nodes_scale.py @@ -0,0 +1,751 @@ +#!/usr/bin/env python3 +"""The expected cells of `nodes_scale` (docs/contrib/dgx-spark.md §9.9), derived apart from the planner. + +What it derives. For each synthetic shape the gate plans (§9.7's terms for V4.1, GLM-5.3-Flash and Qwen3.8, A low and +high), on N = 1 to 4 Sparks (a pair, a ring of three, four on a switch), at λ 25 and 70 µs, under the gate's ruler and a +4 % one, and at S₁ 1,000 and 3,000 tok/s: the nodes the plan uses and why it leaves each other node idle (§9.8 criteria +1, 2 and 7), the routed ids the pools cannot hold, in all and per node (criterion 1), the decode step, node 0's messages +in the layer that reaches the most remotes (criterion 6) and the busiest NIC with whether criterion 5 fires. From the +cells it prints §9.8's "where each fires" table and §9.7's (a) rows at the gate's inputs, as the doc carries them. + +How, so that the cells do not repeat the planner's method (crates/placement/src/placement/nodes.rs): +- The split is §9.2's rule in real shares: node 0 takes the share at which its mean window equals a remote's, the + remotes share the rest equally, and each node is held to its pool's room where the pools hold every id, or keeps at + least its room where they do not. Bisection on node 0's share; the planner instead places whole ids one at a time. +- A layer's windows are expectations over the count tuples of a token's k slots, whose probabilities are built one slot + at a time: each slot moves a tuple's probability to the tuples one count higher, times the node's share (§9.7: "each + routed slot lands on node j with the ownership probability"). The planner sums closed-form multinomials instead. +- Floats with the four basic operations only, so every host writes the same file. + +The formulas and their sources: +- A layer (§9.7 (a), "the window is the max over the nodes of `Sh + slots·e` (node 0) or `λ + hop + posts + rows + ingress + slots·e` (a remote)"): node 0 reads the shared bytes and its slots' experts; a remote that a slot reaches + waits for the round trip (λ, plus the hop on a switch), a post for each other remote reached and the rows of the + other remotes' slots through node 0's NIC, then reads its slots' experts; the layer waits for the slowest. The step + is A plus every layer (§9.7 (a): "t = A/BW + max(...)/BW"). +- Criterion 2 (§9.8): a plan's link share is its step less the step of the same nodes split with no link term; a node + is used where the share stays within a tenth of the step. +- Criterion 7 (§9.8): a node past the pools that already hold every id is used where it gains at least twice the ruler. + Criterion 1: a node the pools need is used; a remote whose own bytes pass its pool holds nothing and is idle for its + pool; node 0's bytes past its pool refuse every plan. +- Criterion 5 (§9.8, "busy at most 50 % of the decode step and of the predicted prompt batch"): a node's NIC is busy, a + direction, by the rows of the slots on the remotes, one row of n_embd values a slot (§9.7's "rows ingress"): node 0 + takes them all in, a remote sends its own. The handoffs go the other way, one a remote a token reaches, and never + outnumber the rows, since a remote reached holds at least one slot. The prompt batch runs at + §9.7's Amdahl term at the split: S₁ times the one-node token's bytes, A + Sh + R, over A and the slowest node's FFN + bytes, node 0's with Sh. At an even split with no Sh that is 9.7's `1/(1−φ+φ/N)`. The planner states the same + definition; it is shared on purpose, as the criterion it pins. +- Criterion 6 (§9.8, "at most 2·min(k, N−1) for (a)"): node 0 posts a handoff and takes the rows of each remote, so + the layer that reaches the most remotes carries 2 × min(k, remotes used). Shared with the planner on purpose. + +What it takes as given: the rows of crates/placement/tests/nodes_scale_inputs.tsv, among them each node's bytes beside +the routed ids (fixed0, fixed1), which today's card plan sets and the gate reads back from it, and the shapes' rounding +to whole Q4_K rows of n_embd values (`shape`, the gate's `Shape::bytes`: the shapes are the gate's inputs). + +Tolerances and edges. The planner splits whole ids, so each value carries the largest change one id moved makes: +between node 0 and any single remote either way, and between two remotes, within each node's bounds (`*_tol`; a node's +paged ids carry one id). A cell whose NIC band holds the 50 % line, whose link share's band holds the tenth, or whose +gain's band holds twice the ruler is named in `edge` (`nic`, `link:::`, `gain:::`), and its side is the derivation's value; the gate pins each edge's side by name and holds the planner's value to +the band besides. + + nodes_scale.py print the cells + nodes_scale.py --write write crates/placement/tests/nodes_scale_cells.tsv + nodes_scale.py --check exit 1 when that file differs from the cells, or the doc's tables from --table + nodes_scale.py --table §9.8's "where each fires" table and §9.7's (a) rows at the gate's inputs + nodes_scale.py --self-test hand-worked cases, the refusals, and --check + +Refused by name (exit 2): an inputs row of another cell count, a name twice, a kind outside measured / derived / +assumed, a value that is not a whole number in ASCII digits, a missing or unknown row, an input outside its domain +(top-k outside 1 to the experts, a ruler outside 1 to 1000 permille, a rate, a pool, layers or a row size of 0, a term +under one row), and node 0's bytes past its pool. Exit 64: usage. + +Python 3 standard library only. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +TESTS = os.path.join(HERE, "..", "..", "crates", "placement", "tests") +INPUTS = os.path.join(TESTS, "nodes_scale_inputs.tsv") +CELLS = os.path.join(TESTS, "nodes_scale_cells.tsv") +DOC = os.path.join(HERE, "..", "..", "docs", "contrib", "dgx-spark.md") + +KINDS = ("measured", "derived", "assumed") +MACHINE = ("read", "nic", "pool", "lambda_lo", "lambda_hi", "hop", "post", "row_value", "vocab", "ruler", + "ruler_wide", "s1_lo", "s1_hi") +TERMS = ("layers", "n_embd", "experts", "top_k", "expert", "shared_lo", "shared_hi", "serial_lo", "serial_hi", + "fixed0_lo", "fixed0_hi", "fixed1") +COLUMNS = ("model", "a", "n", "lambda_us", "ruler", "s1", "used", "idle", "paged", "paged_nodes", "step_ns", "step_tol", + "nic", "nic_tol", "fires", "msgs", "edge") +NS = 1e9 +# Each band's slack in whole units (ns, ‰, ppm): the band is truncated (under 1), the value rounded to the nearest (½), +# and the planner's value rounded down from sums of whole picoseconds (under 1 and a fraction): under 3, so 2 whole. +SLACK = 2 + + +class Refused(Exception): + pass + + +def parse(text, path): + """The inputs file's rows as (name to int, models in file order).""" + rows, models, header = {}, [], None + for line in text.splitlines(): + if line.startswith("#") or not line.strip(): + continue + cells = line.split("\t") + if header is None: + header = cells + if header != ["name", "value", "unit", "kind", "source"]: + raise Refused(f"{path}: the header is {header}, not name, value, unit, kind, source") + continue + if len(cells) != 5: + raise Refused(f"{path}: {cells[0]!r} has {len(cells)} cells, the header 5") + name, value, _, kind, _ = cells + if name in rows: + raise Refused(f"{path}: {name} twice") + if kind not in KINDS: + raise Refused(f"{path}: {name}: kind {kind!r} is not one of {', '.join(KINDS)}") + if not (value.isascii() and value.isdigit()): + raise Refused(f"{path}: {name}: {value!r} is not a whole number") + rows[name] = int(value) + model = name.split("/")[0] if "/" in name else None + if model is not None and model not in models: + models.append(model) + if header is None: + raise Refused(f"{path}: no header row") + return rows, models + + +def inputs(text, path): + """The inputs as (machine, models): name to int, and model to term to int, models in file order.""" + rows, models = parse(text, path) + machine = {} + for k in MACHINE: + if k not in rows: + raise Refused(f"{path}: no {k} row") + machine[k] = rows[k] + terms = {} + for m in models: + terms[m] = {} + for t in TERMS: + if f"{m}/{t}" not in rows: + raise Refused(f"{path}: no {m}/{t} row") + terms[m][t] = rows[f"{m}/{t}"] + known = set(MACHINE) | {f"{m}/{t}" for m in models for t in TERMS} + for name in rows: + if name not in known: + raise Refused(f"{path}: {name} is no input") + domain(machine, terms, path) + return machine, terms + + +def domain(mc, terms, path): + """Refuses an input the model is undefined at, before anything is computed from it.""" + for k in ("read", "nic", "pool", "row_value", "s1_lo", "s1_hi"): + if mc[k] == 0: + raise Refused(f"{path}: {k} is 0") + for k in ("ruler", "ruler_wide"): + if not 1 <= mc[k] <= 1000: + raise Refused(f"{path}: {k} {mc[k]} is outside 1 to 1000 permille") + for m, t in terms.items(): + for k in ("layers", "experts"): + if t[k] == 0: + raise Refused(f"{path}: {m}/{k} is 0") + if not 1 <= t["top_k"] <= t["experts"]: + raise Refused(f"{path}: {m}/top_k {t['top_k']} is outside 1 to its {t['experts']} experts") + if t["n_embd"] < 256: + raise Refused(f"{path}: {m}/n_embd {t['n_embd']} is under one Q4_K block of 256 values") + row = t["n_embd"] // 256 * 144 + if t["expert"] < row: + raise Refused(f"{path}: {m}/expert {t['expert']} B is under one row of {row} B") + for end in ("lo", "hi"): + if t[f"serial_{end}"] // row < mc["vocab"]: + raise Refused(f"{path}: {m}/serial_{end} holds fewer rows than the head's {mc['vocab']}") + + +def load(): + try: + with open(INPUTS, encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise Refused(f"{INPUTS}: {e.strerror}") from None + return inputs(text, INPUTS) + + +def shape(mc, t, high): + """The bytes the gate's shape lays out, in whole Q4_K rows of n_embd values: A (its attention, the head and one + embedding row), a layer's Sh and one expert; and the rest of the case.""" + row = t["n_embd"] // 256 * 144 + layers = t["layers"] + end = "hi" if high else "lo" + attn = (t[f"serial_{end}"] // row - mc["vocab"]) // layers + return { + "layers": layers, "experts": t["experts"], "k": t["top_k"], "n_embd": t["n_embd"], + "serial": (attn * layers + mc["vocab"] + 1) * row, + "shared": t[f"shared_{end}"] // row // layers * row, + "expert": t["expert"] // row * row, + "fixed0": t[f"fixed0_{end}"], "fixed1": t["fixed1"], + } + + +def falls(s, k): + """The probability of each count tuple of a token's `k` slots on nodes of shares `s`, one slot at a time.""" + dist = {(0,) * len(s): 1.0} + for _ in range(k): + nxt = {} + for c, p in dist.items(): + for j, x in enumerate(s): + if x == 0.0: + continue + d = c[:j] + (c[j] + 1,) + c[j + 1:] + nxt[d] = nxt.get(d, 0.0) + p * x + dist = nxt + return dist + + +class Link: + """A remote's link terms a layer, in seconds: the round trip, a further post, one slot's rows through a NIC.""" + + def __init__(self, rt, post, row): + self.rt, self.post, self.row = rt, post, row + + +def layer(s, sh, e, k, link): + """A layer at shares `s` (node 0 first), in seconds: the expected slowest window, and each node's mean window.""" + n = len(s) + slowest, mean = 0.0, [0.0] * n + for c, p in falls(s, k).items(): + reached = sum(1 for cj in c[1:] if cj > 0) + remote = sum(c[1:]) + w = [sh + c[0] * e] + for cj in c[1:]: + w.append(link.rt + link.post * (reached - 1) + link.row * (remote - cj) + cj * e if cj > 0 else 0.0) + slowest += p * max(w) + for j in range(n): + mean[j] += p * w[j] + return slowest, mean + + +class Plan: + def __init__(self, s, step, tol): + self.s, self.step, self.tol = s, step, tol + + +class Case: + def __init__(self, mc, sh, n, lam): + self.mc, self.sh = mc, sh + read = mc["read"] + self.A = sh["serial"] / read + self.Sh = sh["shared"] / read + self.e = sh["expert"] / read + self.id = sh["layers"] * sh["expert"] + if sh["fixed0"] > mc["pool"]: + raise Refused(f"node 0's {sh['fixed0']} B beside the routed ids pass its pool of {mc['pool']} B") + self.rooms = ((mc["pool"] - sh["fixed0"]) // self.id, max(0, mc["pool"] - sh["fixed1"]) // self.id) + self.pool_idle = sh["fixed1"] > mc["pool"] + hop = mc["hop"] if n == 4 else 0 + self.link = Link((lam + hop) / NS, mc["post"] / NS, mc["row_value"] * sh["n_embd"] / mc["nic"]) + self.bare = Link(0.0, 0.0, 0.0) + + def room(self, m): + return self.rooms[0] + m * self.rooms[1] + + def limits(self, m): + """Each used node's share's bounds with `m` remotes: up to its room where the pools hold every id, from it + where they do not.""" + E = self.sh["experts"] + r = [min(1.0, self.rooms[0] / E)] + [min(1.0, self.rooms[1] / E)] * m + if self.room(m) >= E: + return [(0.0, x) for x in r] + return [(x, 1.0) for x in r] + + def shares(self, s0, m): + return [s0] + [(1.0 - s0) / m] * m if m else [1.0] + + def balance(self, m, link): + """§9.2's rule: node 0's share where its mean window equals a remote's, within the bounds.""" + if m == 0: + return 1.0 + b = self.limits(m) + lo, hi = max(b[0][0], 1.0 - m * b[1][1]), min(b[0][1], 1.0 - m * b[1][0]) + k = self.sh["k"] + + def diff(s0): + _, mean = layer(self.shares(s0, m), self.Sh, self.e, k, link) + return mean[0] - mean[1] + + if diff(lo) >= 0: + return lo + if diff(hi) <= 0: + return hi + for _ in range(64): + mid = (lo + hi) / 2 + if diff(mid) < 0: + lo = mid + else: + hi = mid + return (lo + hi) / 2 + + def step(self, s, link): + slowest, _ = layer(s, self.Sh, self.e, self.sh["k"], link) + return self.A + self.sh["layers"] * slowest + + def moves(self, s): + """The shares one id away from `s`: one id from any used node to any other, within each node's bounds.""" + d = 1.0 / self.sh["experts"] + b = self.limits(len(s) - 1) + out = [] + for i in range(len(s)): + for j in range(len(s)): + if i == j or s[i] - d < b[i][0] or s[j] + d > b[j][1]: + continue + t = list(s) + t[i] -= d + t[j] += d + out.append(t) + return out + + def plan(self, m, link): + """The split of `m` remotes: its shares, its step and the step's one-id tolerance.""" + s = self.shares(self.balance(m, link), m) + t = self.step(s, link) + return Plan(s, t, max([abs(self.step(x, link) - t) for x in self.moves(s)], default=0.0)) + + def nic(self, s, t, s1): + """The busiest node's NIC busy share, a fraction, of the plan at shares `s` and step `t`.""" + sh = self.sh + k, L, m = sh["k"], sh["layers"], len(s) - 1 + if m == 0: + return 0.0 + rows = [0.0] * (m + 1) + for c, p in falls(s, k).items(): + for j in range(1, m + 1): + rows[0] += p * c[j] + rows[j] += p * c[j] + row = self.mc["row_value"] * sh["n_embd"] + A, Sh, R = sh["serial"], sh["shared"] * L, k * self.id + ffn = max([Sh + s[0] * R] + [x * R for x in s[1:]]) + rate = s1 * (A + Sh + R) / (A + ffn) + busiest = 0.0 + for j in range(m + 1): + b = L * row * rows[j] / self.mc["nic"] + busiest = max(busiest, b / t, b * rate) + return busiest + + +def ppm(x): + return round(x * 1e6) + + +def busiest(c, p, m, s1): + """The plan's busiest NIC share, its one-id tolerance, and whether its band holds the 50 % line.""" + busy = c.nic(p.s, p.step, s1) + tol = max([abs(c.nic(x, c.step(x, c.link), s1) - busy) for x in c.moves(p.s)], default=0.0) + return busy, tol, m > 0 and abs(busy - 0.5) <= tol + + +def cell(mc, sh, n, lam, ruler): + """The plan of shape `sh` on `n` nodes at round trip `lam` ns: the case, m remotes used, idle [(node, why)], the + edges of the node choice, and the plan.""" + c = Case(mc, sh, n, lam) + E = sh["experts"] + m, idle, edges = 0, [], [] + now = c.plan(0, c.link) + for j in range(1, n): + if c.pool_idle: + idle.append((j, "pool")) + continue + nxt = c.plan(m + 1, c.link) + if c.room(m) < E and c.rooms[1] > 0: + m, now = m + 1, nxt + continue + bare = c.plan(m + 1, c.bare) + share = (nxt.step - bare.step) / nxt.step + share_tol = (nxt.tol + bare.tol) / nxt.step + if abs(share - 0.1) <= share_tol: + edges.append(f"link:{j}:{ppm(share)}:{int(share_tol * 1e6) + SLACK}") + if share > 0.1: + idle.append((j, "link")) + continue + gain = (now.step - nxt.step) / now.step + gain_tol = (now.tol + nxt.tol) / now.step + if abs(gain - 2 * ruler / 1000) <= gain_tol: + edges.append(f"gain:{j}:{ppm(gain)}:{int(gain_tol * 1e6) + SLACK}") + if gain < 2 * ruler / 1000: + idle.append((j, "gain")) + continue + m, now = m + 1, nxt + return c, m, idle, edges, now + + +def derive(mc, terms): + """Every cell, in the gate's order: model, A low then high, (ruler, S₁), λ, N.""" + out = [] + pairs = ((mc["ruler"], mc["s1_lo"]), (mc["ruler"], mc["s1_hi"]), (mc["ruler_wide"], mc["s1_hi"])) + for model, t in terms.items(): + for high in (False, True): + sh = shape(mc, t, high) + for ruler, s1 in pairs: + for lam in (mc["lambda_lo"], mc["lambda_hi"]): + for n in range(1, 5): + try: + c, m, idle, edges, p = cell(mc, sh, n, lam, ruler) + except Refused as e: + raise Refused(f"{model} A {'high' if high else 'low'}: {e}") from None + busy, busy_tol, on_line = busiest(c, p, m, s1) + if on_line: + edges.insert(0, "nic") + E = sh["experts"] + paging = c.room(m) < E + rooms = [c.rooms[0]] + [c.rooms[1]] * m + per = [round(x * E - r) if paging else 0 for x, r in zip(p.s, rooms)] + [0] * (n - 1 - m) + out.append({ + "model": model, "a": "high" if high else "low", "n": n, "lambda_us": lam // 1000, + "ruler": ruler, "s1": s1, "used": m + 1, + "idle": ",".join(f"{j}:{w}" for j, w in idle) or "-", + "paged": max(0, E - c.room(m)), "paged_nodes": ",".join(map(str, per)), + "step_ns": round(p.step * NS), "step_tol": int(p.tol * NS) + SLACK, + "nic": round(busy * 1000), "nic_tol": int(busy_tol * 1000) + SLACK, + "fires": int(m > 0 and busy > 0.5), "msgs": 2 * min(sh["k"], m), + "edge": ",".join(edges) or "-", + }) + return out + + +def text(cells): + lines = ["# nodes_scale's expected cells (docs/contrib/dgx-spark.md §9.9), written by tools/flow/nodes_scale.py " + "--write from nodes_scale_inputs.tsv; do not edit.", + "\t".join(COLUMNS)] + for c in cells: + lines.append("\t".join(str(c[k]) for k in COLUMNS)) + return "\n".join(lines) + "\n" + + +def span(values, fmt): + lo, hi = fmt(min(values)), fmt(max(values)) + return lo if lo == hi else f"{lo} to {hi}" + + +def blocks(mc, terms, cells): + """The doc's derived tables, each a list of lines: §9.8's "where each fires" and §9.7's (a) rows.""" + at = {(c["model"], c["a"], c["n"], c["lambda_us"], c["ruler"], c["s1"]): c for c in cells} + lo, hi = mc["lambda_lo"] // 1000, mc["lambda_hi"] // 1000 + r, s1_lo, s1_hi = mc["ruler"], mc["s1_lo"], mc["s1_hi"] + + def where(model, pick, rate=s1_hi): + """The cells `pick` names, by N; an N alone is every λ and A at it.""" + hits = [] + for n in range(1, 5): + each = [(lam, [a for a in ("low", "high") if pick(at[(model, a, n, lam, r, rate)])]) for lam in (lo, hi)] + if all(len(ends) == 2 for _, ends in each): + hits.append(f"N = {n}") + continue + hits += [f"N = {n}, λ {lam}, A {' and '.join(ends)}" for lam, ends in each if ends] + return "; ".join(hits) or "never" + + fires = [f"| Model | Decode step (ms) at the gate's inputs, N = 1 → 2 → 3 → 4, λ {lo} / {hi}, A low " + f"| Criterion 2 fires | Criterion 7 fires | Criterion 5 fires, S₁ {s1_lo:,} | Criterion 5 fires, " + f"S₁ {s1_hi:,} |", "|---|---|---|---|---|---|"] + for model in terms: + steps = [] + for n in range(1, 5): + a, b = (at[(model, "low", n, lam, r, s1_hi)]["step_ns"] / 1e6 for lam in (lo, hi)) + steps.append(f"{a:.1f}" if n == 1 else f"{a:.1f} / {b:.1f}") + fires.append(f"| {model} | {' → '.join(steps)} | {where(model, lambda c: 'link' in c['idle'])} " + f"| {where(model, lambda c: 'gain' in c['idle'])} " + f"| {where(model, lambda c: c['fires'] == 1, rate=s1_lo)} " + f"| {where(model, lambda c: c['fires'] == 1)} |") + + star = ["| (a) star at the gate's inputs | N | Nodes used | Decode, one request (tok/s) | Prompt (× S₁) " + "| Node 0's NIC ingress cap (tok/s) | Link bytes a token (MB) | Messages a token |", + "|---|---|---|---|---|---|---|---|"] + for model, t in terms.items(): + for n in (2, 4): + cs = [at[(model, a, n, lam, r, s1_hi)] for a in ("low", "high") for lam in (lo, hi)] + used, decode, prompt, cap, link, msgs = [], [], [], [], [], [] + for a in ("low", "high"): + sh = shape(mc, t, a == "high") + for lam in (lo, hi): + c, m, _, _, p = cell(mc, sh, n, lam * 1000, r) + k, L = sh["k"], sh["layers"] + A, Sh, R = sh["serial"], sh["shared"] * L, k * c.id + used.append(m + 1) + decode.append(NS / round(p.step * NS)) + prompt.append((A + Sh + R) / (A + max([Sh + p.s[0] * R] + [x * R for x in p.s[1:]]))) + row = mc["row_value"] * sh["n_embd"] + remote = 1.0 - p.s[0] + cap.append(mc["nic"] / (L * k * remote * row) if m else 0.0) + reached = 0.0 + for cc, q in falls(p.s, k).items(): + reached += q * sum(1 for x in cc[1:] if x > 0) + link.append(L * row * (reached + k * remote) / 1e6) + msgs.append(2 * L * reached) + paged = [x["paged"] for x in cs] + note = f" (pool only: pages {span(paged, str)} ids)" if max(paged) else "" + star.append(f"| {model} | {n} | {span(used, str)} | {span(decode, lambda v: f'{v:.1f}')}{note} " + f"| {span(prompt, lambda v: f'{v:.2f}')} | {span(cap, lambda v: f'{v:,.0f}')} " + f"| {span(link, lambda v: f'{v:.2f}')} | {span(msgs, lambda v: f'{v:.0f}')} |") + return [fires, star] + + +def table(mc, terms, cells): + return "\n\n".join("\n".join(b) for b in blocks(mc, terms, cells)) + + +def doc_differs(bs, text): + """The first line of each block of `bs` that the doc `text` does not carry as printed, alone under its header.""" + doc = text.split("\n") + off = [] + for b in bs: + heads = [i for i, line in enumerate(doc) if line == b[0]] + if len(heads) != 1 or doc[heads[0]:heads[0] + len(b)] != b or heads[0] + len(b) < len(doc) and \ + doc[heads[0] + len(b)].startswith("|"): + off.append(b[0]) + return off + + +def check(): + try: + with open(CELLS, encoding="utf-8") as f: + have = f.read() + except OSError as e: + print(f"nodes_scale: {CELLS}: {e.strerror}", file=sys.stderr) + return 1 + machine, terms = load() + cells = derive(machine, terms) + rc = 0 + if have != text(cells): + print(f"nodes_scale: {CELLS} is not the derivation of {INPUTS}: run tools/flow/nodes_scale.py --write", + file=sys.stderr) + rc = 1 + try: + with open(DOC, encoding="utf-8") as f: + doc = f.read() + except OSError as e: + print(f"nodes_scale: {DOC}: {e.strerror}", file=sys.stderr) + return 1 + for head in doc_differs(blocks(machine, terms, cells), doc): + print(f"nodes_scale: {DOC} does not carry the --table block {head!r} as printed", file=sys.stderr) + rc = 1 + return rc + + +def self_test(): + ok = True + + def expect(what, cond): + nonlocal ok + print(f"{'ok ' if cond else 'FAIL'} {what}") + ok = ok and cond + + def near(a, b): + return abs(a - b) <= 1e-12 * max(1.0, abs(b)) + + # Two slots at shares (1/2, 1/2): both on node 0 a quarter, one each a half, both remote a quarter. + d = falls([0.5, 0.5], 2) + expect("two slots on a pair: 1/4, 1/2, 1/4", d == {(2, 0): 0.25, (1, 1): 0.5, (0, 2): 0.25}) + d = falls([0.25, 0.25, 0.25, 0.25], 10) + expect("ten slots on four nodes: 286 tuples, one in all, (10, 0, 0, 0) at 4^-10", + len(d) == 286 and near(sum(d.values()), 1.0) and near(d[(10, 0, 0, 0)], 1 / 1048576)) + # (2, 1, 1) at shares (1/2, 1/4, 1/4): 12 orders, each (1/2)² × 1/4 × 1/4 = 1/64. + expect("the tuple (2, 1, 1) at (1/2, 1/4, 1/4)", near(falls([0.5, 0.25, 0.25], 4)[(2, 1, 1)], 12 / 64)) + + # Sh 1, e 2, rt 4. Both on node 0 (1/4): 1 + 2e = 5; one each (1/2): max(1 + e, rt + e) = 6; both remote (1/4): + # max(1, rt + 2e) = 8. Expected 1.25 + 3 + 2 = 6.25. Node 0's mean window: 1 + e at one slot on average, 3; the + # remote's: (1/2) × 6 + (1/4) × 8 = 5. + slowest, mean = layer([0.5, 0.5], 1.0, 2.0, 2, Link(4.0, 0.0, 0.0)) + expect("a pair's layer: the expected slowest window", near(slowest, 6.25)) + expect("a pair's layer: the mean windows", near(mean[0], 3.0) and near(mean[1], 5.0)) + # Three nodes at (0, 1/2, 1/2), two slots: one on each remote (1/2) posts once more and takes the other's rows + # ahead: rt + post + row + e = 4 + 1 + 0.5 + 2 = 7.5; both on one remote (1/2): rt + 2e = 8. Expected 7.75. + slowest, _ = layer([0.0, 0.5, 0.5], 0.0, 2.0, 2, Link(4.0, 1.0, 0.5)) + expect("a ring's layer: a post and the rows ahead of a second remote", near(slowest, 7.75)) + + mc = {"read": 1, "nic": 1, "pool": 100, "lambda_lo": 0, "lambda_hi": 0, "hop": 0, "post": 0, "row_value": 4, + "vocab": 0, "ruler": 10, "ruler_wide": 40, "s1_lo": 1, "s1_hi": 1} + sh = {"layers": 1, "experts": 4, "k": 1, "n_embd": 256, "serial": 0, "shared": 0, "expert": 10, + "fixed0": 70, "fixed1": 60} + c = Case(mc, sh, 2, 0) + # Rooms (100 - 70) / 10 = 3 and (100 - 60) / 10 = 4: one remote holds every id; with no link and no shared + # bytes the windows balance at half the ids each, inside both rooms. + expect("the rooms", c.rooms == (3, 4) and not c.pool_idle) + expect("no link, no shared bytes: half each", near(c.balance(1, c.bare), 0.5)) + expect("the bounds where the pools hold every id", c.limits(1) == [(0.0, 0.75), (0.0, 1.0)]) + # A shared read of 3 on node 0 against one slot of 10: s0 × 10 + 3 = (1 - s0) × 10 at s0 = 0.35. One slot: on + # node 0 (0.35) the window is 3 + 10, on the remote (0.65) max(3, 10): a step of 4.55 + 6.5 = 11.05. One id moved + # (a quarter): node 0 at 0.1 gives 1.3 + 9 = 10.3, at 0.6 gives 7.8 + 4 = 11.8, so the tolerance is 0.75. + c.Sh = 3.0 + p = c.plan(1, c.bare) + expect("plan(): node 0's shared bytes move ids to the remote, and its one-id tolerance", + near(p.s[0], 0.35) and near(p.step, 11.05) and near(p.tol, 0.75)) + c2 = Case(mc, dict(sh, fixed0=90, fixed1=90), 2, 0) + # Rooms 1 and 1 of 4 ids: the pools page 2, and each node keeps at least its room. + expect("paging: each node at least its room", c2.room(1) == 2 and c2.limits(1) == [(0.25, 1.0), (0.25, 1.0)]) + c3 = Case(mc, dict(sh, fixed1=95), 2, 0) + expect("a room-0 remote has paging bounds and fits its fixed-byte budget", + c3.rooms == (3, 0) and not c3.pool_idle and c3.limits(1) == [(0.75, 1.0), (0.0, 1.0)]) + _, m, idle, _, _ = cell(mc, dict(sh, fixed1=95), 2, 0, 10) + expect("a remote of room 0 must gain twice the ruler", m == 0 and idle == [(1, "gain")]) + _, m, idle, _, _ = cell(mc, dict(sh, fixed1=101), 3, 0, 10) + expect("a remote whose own bytes pass its pool is idle for its pool", m == 0 and idle == [(1, "pool"), (2, "pool")]) + try: + Case(mc, dict(sh, fixed0=101), 1, 0) + expect("node 0's bytes past its pool: not refused", False) + except Refused as e: + expect(f"node 0's bytes past its pool: {e}", "node 0" in str(e)) + # Four nodes on a switch: the hop joins λ, a ring of three has none. One slot, every id on node 1: a layer is + # λ + hop + e = 25 + 1.5 + 10 s at λ 25 s, so a step of A 2 + 36.5. + sw = dict(mc, hop=1_500_000_000, lambda_lo=25_000_000_000) + c4 = Case(sw, dict(sh, serial=2), 4, sw["lambda_lo"]) + expect("the switch's hop at N = 4", near(c4.step([0.0, 1.0, 0.0, 0.0], c4.link), 38.5) + and near(Case(sw, sh, 3, sw["lambda_lo"]).link.rt, 25.0)) + # NIC, node 0 at s0 = 1/2, k = 1, one layer, rows of 4 × 256 = 1,024 B: a remote slot half the time, so 512 B a + # token. Decode at a step of 1,024 s on a NIC of 1 B/s: half busy. The prompt at S₁ 1 with no A or Sh runs + # 10 / 5 = 2 tokens a second over the slowest node's FFN: 1,024 B a second, twice the NIC. + expect("the NIC: the prompt's share where it dominates", near(c.nic([0.5, 0.5], 1024.0, 1), 1024.0)) + expect("the NIC: the decode's share where it dominates", near(c.nic([0.5, 0.5], 1024.0, 1e-9), 0.5)) + # Three nodes at (1/2, 1/4, 1/4): node 0 takes every remote slot's rows, 512 B a token, each remote its own + # quarter, 256 B. Decode at a step of 1,024 s: node 0 half busy. + expect("the NIC: node 0 takes every remote's rows", near(Case(mc, sh, 3, 0).nic([0.5, 0.25, 0.25], 1024.0, 1e-9), + 0.5)) + # Sh of 10 on node 0 against R = 10: node 0's FFN is 10 + 5, the remote's 5, so the prompt runs 20 / 15 tokens a + # second at S₁ 1 and node 0 takes 512 B a token: 682.67 B a second. + cs = Case(mc, dict(sh, shared=10), 2, 0) + expect("the NIC: the prompt's slowest FFN is node 0's with Sh", near(cs.nic([0.5, 0.5], 1e9, 1), 512 * 20 / 15)) + # On a NIC of 10^6 B/s the same split's prompt at S₁ 488.28125 is busy one half (1,024 µB a token of S₁); node 0 + # at a quarter of the ids takes 768 µB at the same rate, so the band holds the line: an edge. + cn = Case(dict(mc, nic=10 ** 6), sh, 2, 0) + busy, tol, on = busiest(cn, cn.plan(1, cn.link), 1, 488.28125) + expect("the NIC on its half within one id: an edge", near(busy, 0.5) and tol > 0.3 and on) + # A gain on twice an integer ruler. A = 248 and two slots of 1 on one node: a step of 250. A second node takes one + # slot of two half the time, so the window falls to 1.5: a gain of 0.2 %, twice a ruler of 1 permille; one id + # moved (of two) moves the step by 0.5. The node is named an edge and used, the line not under it. + gm = dict(mc, nic=10 ** 12, pool=10 ** 6) + gs = dict(sh, experts=2, k=2, expert=1, serial=248, fixed0=0, fixed1=0) + _, m, idle, edges, _ = cell(gm, gs, 2, 0, 1) + expect("a gain on twice an integer ruler is an edge, on the used side", edges == ["gain:1:2000:2002"] and m == 1) + # A = 123, one id of a hundred: the gain is 0.5 / 125 = 0.4 %, between a ruler of 3 permille and twice it, and one + # id moved (0.01 of the share) moves the step by 0.0002: no edge, and the node is idle. + _, m, idle, edges, _ = cell(gm, dict(gs, experts=100, serial=123), 2, 0, 3) + expect("a gain under twice the ruler and over it leaves the node idle", (m, idle, edges) == (0, [(1, "gain")], [])) + # One slot, two ids, A 3.4, e 1 and a round trip of 2: the split balances at s0 = 3/4 (s0 = (1 - s0) × 3), so the + # step is 3.4 + 1.5 against 3.4 + 1 with no link: a link share of 0.5 / 4.9, past the tenth. One id moved puts + # s0 at 1/4 and the step at 3.4 + 2.5, so the band holds the tenth: an edge, and the node idle. + lm = dict(gm, read=10, lambda_lo=2 * 10 ** 9) + _, m, idle, edges, _ = cell(lm, dict(gs, k=1, expert=10, serial=34), 2, lm["lambda_lo"], 10) + expect("a link share on the tenth within one id: an edge, on its side", (m, idle) == (0, [(1, "link")]) + and edges == [f"link:1:{round(0.5 / 4.9 * 1e6)}:{int(1.0 / 4.9 * 1e6) + 2}"]) + cm = Case(mc, dict(sh, fixed0=0, fixed1=0), 3, 0) + moves = cm.moves([0.5, 0.25, 0.25]) + expect("one id moved: node 0 to and from each remote, and between the remotes", + len(moves) == 6 and [0.5, 0.0, 0.5] in moves and [0.5, 0.5, 0.0] in moves) + + machine, terms = load() + cells = derive(machine, terms) + by = {(x["model"], x["a"], x["n"], x["lambda_us"], x["ruler"], x["s1"]): x for x in cells} + glm2 = [by[("GLM", a, 2, lam, machine["ruler"], machine["s1_hi"])] for a in ("low", "high") for lam in (25, 70)] + expect("GLM held whole on two Sparks", all(x["paged"] == 0 and x["used"] == 2 for x in glm2)) + v41 = by[("V4.1", "low", 2, 25, machine["ruler"], machine["s1_hi"])] + expect("V4.1 pages on two Sparks, on both, and not on three", + v41["paged"] > 0 and all(int(x) > 0 for x in v41["paged_nodes"].split(",")) + and by[("V4.1", "low", 3, 25, machine["ruler"], machine["s1_hi"])]["paged"] == 0) + q4 = by[("Qwen3.8", "low", 4, 70, machine["ruler"], machine["s1_hi"])] + expect("Qwen3.8 on four switched Sparks at λ 70 leaves nodes idle for the link", "link" in q4["idle"]) + expect("V4.1 on four Sparks: a handoff and its rows for each of three remotes, none on one", + by[("V4.1", "low", 4, 25, machine["ruler"], machine["s1_hi"])]["msgs"] == 6 + and by[("V4.1", "low", 1, 25, machine["ruler"], machine["s1_hi"])]["msgs"] == 0) + expect("each cell's paged ids per node sum to its paged ids", + all(sum(map(int, x["paged_nodes"].split(","))) == x["paged"] for x in cells)) + + with open(INPUTS, encoding="utf-8") as f: + good = f.read() + lines = good.split("\n") + first = next(i for i, x in enumerate(lines) if x.startswith("read\t")) + + def row(name, value): + """The inputs with row `name`'s value replaced.""" + return "\n".join(f"{name}\t{value}\t" + x.split("\t", 2)[2] if x.startswith(name + "\t") else x + for x in lines) + + def refused(what, body, reason): + try: + inputs(body, "in.tsv") + except Refused as e: + expect(f"{what}: {e}", reason in str(e)) + return + expect(f"{what}: not refused", False) + + def accepted(what, body): + try: + inputs(body, "in.tsv") + expect(what, True) + except Refused as e: + expect(f"{what}: refused, {e}", False) + + accepted("the inputs as they are", good) + refused("a row of four cells", "\n".join(lines[:first] + [lines[first].rsplit("\t", 1)[0]] + lines[first + 1:]), + "has 4 cells") + refused("a name twice", "\n".join(lines[:first + 1] + [lines[first]] + lines[first + 1:]), "read twice") + refused("a kind of no kind", "\n".join(lines[:first] + [lines[first].replace("\tmeasured\t", "\tguessed\t")] + + lines[first + 1:]), "kind 'guessed'") + refused("a value not whole", row("read", "2.5"), "not a whole number") + refused("a value in other digits", row("read", "٢"), "not a whole number") + refused("a missing row", "\n".join(lines[:first] + lines[first + 1:]), "no read row") + refused("a row that is no input", good + "spare\t1\tx\tassumed\tx\n", "spare is no input") + refused("top-k 0", row("GLM/top_k", 0), "GLM/top_k 0") + refused("top-k past the experts", row("GLM/top_k", 289), "GLM/top_k 289") + accepted("top-k at the experts", row("GLM/top_k", 288)) + refused("a ruler of 0", row("ruler", 0), "ruler 0") + refused("a ruler past 1000", row("ruler_wide", 1001), "ruler_wide 1001") + accepted("rulers of 1 and 1000", row("ruler", 1).replace("ruler_wide\t40\t", "ruler_wide\t1000\t")) + refused("a read rate of 0", row("read", 0), "read is 0") + refused("a NIC of 0", row("nic", 0), "nic is 0") + refused("a prompt rate of 0", row("s1_lo", 0), "s1_lo is 0") + refused("no layers", row("V4.1/layers", 0), "V4.1/layers is 0") + refused("a row width under a block", row("Qwen3.8/n_embd", 255), "Qwen3.8/n_embd 255") + accepted("a row width of one block", row("Qwen3.8/n_embd", 256)) + refused("an expert under a row", row("Qwen3.8/expert", 1439), "Qwen3.8/expert 1439") + try: + derive(*inputs(row("GLM/fixed0_lo", machine["pool"] + 1), "in.tsv")) + expect("node 0's bytes past its pool in the inputs: not refused", False) + except Refused as e: + expect(f"node 0's bytes past its pool in the inputs: {e}", str(e).startswith("GLM A low: node 0's")) + bs = [["| a | b |", "|---|---|", "| 1 | 2 |"]] + expect("a doc that carries a block as printed", doc_differs(bs, "x\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\ny") == []) + expect("a doc whose block differs in a cell, has a row more, or lacks it", + all(doc_differs(bs, d) == ["| a | b |"] for d in ("| a | b |\n|---|---|\n| 1 | 3 |\n", + "| a | b |\n|---|---|\n| 1 | 2 |\n| 3 | 4 |\n", "x\n"))) + expect("the cells file and the doc's tables are the derivation's", check() == 0) + print(f"self-test: {'ok' if ok else 'FAIL'}") + return 0 if ok else 1 + + +def main(argv): + if len(argv) > 1 or (argv and argv[0] not in ("--write", "--check", "--table", "--self-test")): + print(f"nodes_scale: usage\n{__doc__}", file=sys.stderr) + return 64 + try: + if argv == ["--self-test"]: + return self_test() + if argv == ["--check"]: + return check() + machine, terms = load() + cells = derive(machine, terms) + except Refused as e: + print(f"nodes_scale: {e}", file=sys.stderr) + return 2 + if argv == ["--write"]: + with open(CELLS, "w", encoding="utf-8") as f: + f.write(text(cells)) + elif argv == ["--table"]: + print(table(machine, terms, cells)) + else: + sys.stdout.write(text(cells)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))