From d5f6440b01e9149ff1cc8af49bc32d79bc018fc4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:07:51 +0000 Subject: [PATCH 01/10] feat(devtools): stage_pipeline plans SQL queries over tables declared with --table Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/devtools/src/bin/stage_pipeline.rs | 117 ++++++++++++++++++---- crates/devtools/src/planning_stages.rs | 20 ++++ crates/devtools/tests/stage_pipeline.rs | 30 ++++++ 3 files changed, 148 insertions(+), 19 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 49fba9183..9dae50a21 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -4,6 +4,10 @@ // Patterns A and B; 3a has 243 candidates, all priced, 64 shown by default) // cargo run -p asap-devtools --bin stage_pipeline -- \ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json +// cargo run -p asap-devtools --bin stage_pipeline -- \ +// --table '{"name": "metrics", "columns": [{"name": "ts", "type": "timestamp"}, +// {"name": "service", "type": "utf8"}], "time_index": 0}' \ +// --sql "SELECT service, COUNT(*) FROM metrics GROUP BY service" --out run.json // // Writes an `asap-stage-pipeline/v1` document (tools/dag-viewer) with the // four planner stages (#509 MVP): @@ -28,9 +32,13 @@ // query. When selection fails, `selected` is null, with the rejections, // or with `error` for any other failure. // -// `--promql` may repeat. `--epsilon`/`--delta` apply to every `--promql` -// query; without them the queries are exact. `--interval-ms` is the source -// cadence PromQL needs (default 15000). +// `--promql` and `--sql` may repeat; a run is one language. `--epsilon`/ +// `--delta` apply to every query; without them the queries are exact. +// `--interval-ms` is the source cadence (default 15000). `--table ''` +// (repeatable) declares a table the SQL queries read: `{"name": ..., +// "columns": [{"name": ..., "type": "timestamp|utf8|string|float64|double| +// int64|bigint", "nullable": true}], "time_index": 0}` (`nullable` defaults +// to true; without `time_index` the table has no time column). use std::collections::HashSet; @@ -38,20 +46,23 @@ use asap_aware_mapping::stage3_plan_selection::{Rejection, Selection, SelectionE use asap_devtools::planner_layering_examples::{ declared, example1, pattern_a, pattern_b, PatternARecurrence, }; -use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages}; +use asap_devtools::planning_stages::{lower_promql_roots, lower_sql_roots, run_planning_stages}; +use asap_devtools::SqlCatalog; use asap_types::ir::flat::{flatten, FlatDag}; use asap_types::ir::QueryRoot; +use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataArrival, DataWorkload, DurationMs, LatencyRequirement, PlanningWorkload, Query, QueryLanguage, QueryRecurrence, QueryRequirements, QueryWorkload, - RepeatedDemand, + RepeatedDemand, SqlDialect, }; use serde_json::{json, Value}; const USAGE: &str = - "usage: stage_pipeline (--example planner-layering-{1,3a,3b} | --promql ... \ -[--epsilon --delta ] [--interval-ms ]) [--max-displayed ] --out "; + "usage: stage_pipeline (--example planner-layering-{1,3a,3b} | (--promql ... \ +| --table ... --sql ...) [--epsilon --delta ] [--interval-ms ]) \ +[--max-displayed ] --out "; fn main() { if let Err(message) = run(std::env::args().skip(1).collect()) { @@ -62,7 +73,7 @@ fn main() { fn run(args: Vec) -> Result<(), String> { let mut example = None; - let mut queries = Vec::new(); + let (mut promql, mut sql, mut tables) = (Vec::new(), Vec::new(), Vec::new()); let (mut epsilon, mut delta, mut interval_ms) = (None, None, 15_000u64); let mut max_displayed = 64usize; let mut out = None; @@ -72,7 +83,9 @@ fn run(args: Vec) -> Result<(), String> { let number = |v: String| v.parse::().map_err(|e| format!("{v}: {e}")); match flag.as_str() { "--example" => example = Some(value()?), - "--promql" => queries.push(value()?), + "--promql" => promql.push(value()?), + "--sql" => sql.push(value()?), + "--table" => tables.push(value()?), "--epsilon" => epsilon = Some(number(value()?)?), "--delta" => delta = Some(number(value()?)?), "--interval-ms" => interval_ms = value()?.parse().map_err(|e| format!("{e}"))?, @@ -81,6 +94,14 @@ fn run(args: Vec) -> Result<(), String> { other => return Err(format!("unknown argument {other}")), } } + let (language, queries) = match (promql.is_empty(), sql.is_empty()) { + (false, false) => return Err("a run is one language: give --promql or --sql".into()), + (true, false) => (QueryLanguage::SQL(SqlDialect::DataFusionSQL), sql), + _ => (QueryLanguage::PromQL, promql), + }; + if !tables.is_empty() && language == QueryLanguage::PromQL { + return Err("--table needs --sql".into()); + } let workload = match (example.as_deref(), queries.is_empty()) { (Some("planner-layering-1"), true) => example1(), (Some("planner-layering-3a"), true) => { @@ -95,18 +116,25 @@ fn run(args: Vec) -> Result<(), String> { (Some(epsilon), Some(delta)) => AccuracyTarget::EpsilonDelta { epsilon, delta }, (None, Some(_)) => return Err("--delta needs --epsilon".into()), }; - promql_batch(&queries, accuracy, interval_ms) + query_batch(language, &queries, accuracy, interval_ms) } - _ => return Err("give exactly one of --example or --promql".into()), + _ => return Err("give exactly one of --example, --promql or --sql".into()), }; let out = out.ok_or("--out is required")?; - let document = stage_pipeline(&workload, max_displayed)?; + let roots = match workload.query_workload.language { + QueryLanguage::PromQL => lower_promql_roots(&workload)?, + _ => lower_sql_roots(&workload, &sql_catalog(&tables)?)?, + }; + let document = stage_pipeline(&workload, roots, max_displayed)?; let text = serde_json::to_string_pretty(&document).map_err(|e| e.to_string())? + "\n"; std::fs::write(&out, text).map_err(|e| format!("{out}: {e}")) } -fn stage_pipeline(workload: &PlanningWorkload, max_displayed: usize) -> Result { - let roots = lower_promql_roots(workload)?; +fn stage_pipeline( + workload: &PlanningWorkload, + roots: Vec, + max_displayed: usize, +) -> Result { let run = run_planning_stages(workload, roots)?; // The display keeps the first `max_displayed` candidates and the // selected one; plan selection has already ranked all of them. @@ -233,6 +261,10 @@ fn export(roots: &[QueryRoot]) -> FlatDag { } fn workload_queries(workload: &PlanningWorkload) -> Vec { + let language = match workload.query_workload.language { + QueryLanguage::PromQL => "promql", + _ => "sql", + }; workload .query_workload .entries() @@ -256,7 +288,7 @@ fn workload_queries(workload: &PlanningWorkload) -> Vec { } json!({ "id": format!("q{}", index + 1), - "language": "promql", + "language": language, "text": entry.query.0, "requirements": requirements, }) @@ -264,14 +296,55 @@ fn workload_queries(workload: &PlanningWorkload) -> Vec { .collect() } -fn promql_batch( +/// The tables `--table` declares. +fn sql_catalog(tables: &[String]) -> Result { + let mut catalog = SqlCatalog::new(); + for raw in tables { + let table: Value = serde_json::from_str(raw).map_err(|e| format!("--table: {e}"))?; + let name = table["name"] + .as_str() + .ok_or("--table: name must be a string")?; + let columns = table["columns"] + .as_array() + .ok_or("--table: columns must be an array")? + .iter() + .map(|column| { + let column_name = column["name"] + .as_str() + .ok_or("--table: column name must be a string")?; + let dtype = match column["type"] + .as_str() + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("timestamp") => DataType::Timestamp, + Some("utf8" | "string") => DataType::Utf8, + Some("float64" | "double") => DataType::Float64, + Some("int64" | "bigint") => DataType::Int64, + other => return Err(format!("--table: unsupported column type {other:?}")), + }; + let nullable = column["nullable"].as_bool().unwrap_or(true); + Ok(Field::plain(column_name, dtype, nullable)) + }) + .collect::, String>>()?; + let schema = match table["time_index"].as_u64() { + Some(index) => Schema::with_time_index(columns, index as usize, vec![]), + None => Schema::new(columns), + }; + catalog = catalog.with_table(name, schema); + } + Ok(catalog) +} + +fn query_batch( + language: QueryLanguage, queries: &[String], accuracy: AccuracyTarget, interval_ms: u64, ) -> PlanningWorkload { PlanningWorkload { query_workload: QueryWorkload { - language: QueryLanguage::PromQL, + language, query_batch: Some( queries .iter() @@ -326,8 +399,14 @@ mod tests { /// candidate and its cost, chosen among all candidates. #[test] fn display_cap_keeps_the_selected_candidate() { - let workload = promql_batch(&["sum(m)".into()], AccuracyTarget::Exact, 15_000); - let document = stage_pipeline(&workload, 0).unwrap(); + let workload = query_batch( + QueryLanguage::PromQL, + &["sum(m)".into()], + AccuracyTarget::Exact, + 15_000, + ); + let roots = lower_promql_roots(&workload).unwrap(); + let document = stage_pipeline(&workload, roots, 0).unwrap(); let selected = document["stage3_plan_selection"]["selected"] .as_str() .unwrap(); diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 7d673eabd..74130e4c9 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -37,6 +37,26 @@ pub fn lower_promql_roots(workload: &PlanningWorkload) -> Result, .collect() } +/// Lower a SQL workload over the tables of `catalog`. +pub fn lower_sql_roots( + workload: &PlanningWorkload, + catalog: &asap_frontend_sql::SqlCatalog, +) -> Result, String> { + tokio::runtime::Builder::new_current_thread() + .build() + .map_err(|e| e.to_string())? + .block_on(asap_frontend_sql::lower_sql_batch( + &workload.query_workload, + catalog, + )) + .into_iter() + .map(|root| { + root.map(QueryRoot::Operator) + .map_err(|e| format!("lowering: {e}")) + }) + .collect() +} + /// Logical ASAP-aware optimization (Pass 1 and composition; no Pass 2), /// physical ASAP-aware optimization and plan selection over `roots`, the /// lowered entries of `workload`: the library's planner pipeline diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 60a559d8d..a358d1b60 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -205,3 +205,33 @@ fn example3a_selects_the_cheapest_of_all_candidates() { .unwrap(); assert!(physical.iter().any(|p| p["id"] == expected.as_str())); } + +/// SQL queries over tables declared with `--table` are planned through the +/// same stages, and the document records each query's language. +#[test] +fn sql_queries_over_declared_tables_are_planned() { + let out = std::env::temp_dir().join(format!("stage_pipeline_{}_sql.json", std::process::id())); + let status = Command::new(env!("CARGO_BIN_EXE_stage_pipeline")) + .args([ + "--table", + r#"{"name": "metrics", "columns": [{"name": "ts", "type": "timestamp", "nullable": false}, + {"name": "service", "type": "utf8", "nullable": false}, + {"name": "latency", "type": "float64", "nullable": false}], "time_index": 0}"#, + "--sql", + "SELECT service, COUNT(*) FROM metrics GROUP BY service", + "--sql", + "SELECT service, AVG(latency) FROM metrics GROUP BY service", + "--out", + ]) + .arg(&out) + .status() + .unwrap(); + assert!(status.success()); + let document: Value = serde_json::from_str(&std::fs::read_to_string(&out).unwrap()).unwrap(); + std::fs::remove_file(out).unwrap(); + let queries = document["workload"]["queries"].as_array().unwrap(); + assert_eq!(queries.len(), 2); + assert!(queries.iter().all(|q| q["language"] == "sql")); + validated_dag(&document["stage0_query_language_frontends"]["dag"], 2); + assert!(document["stage3_plan_selection"]["selected"].is_string()); +} From e8e4d49303eb6d15cce4eb4201def1639cfc16e7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:10:34 +0000 Subject: [PATCH 02/10] feat(dag-viewer): plan through stage_pipeline in the live viewer server.py's /api/plan runs stage_pipeline (one language per plan; SQL tables via --table) and returns its asap-stage-pipeline/v1 document; stage-workload.js shows it as Stage 0 before and the Stage 3 selected plan after, with per-node costs. generate-sample.sh writes dag.example.json with stage_pipeline; render.py embeds a stage document unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- tools/dag-viewer/dag.example.json | 11145 ++++++++++---------------- tools/dag-viewer/generate-sample.sh | 32 +- tools/dag-viewer/index.html | 5 +- tools/dag-viewer/node-style.js | 3 +- tools/dag-viewer/planner-ui.js | 12 +- tools/dag-viewer/render.py | 46 +- tools/dag-viewer/server.py | 114 +- tools/dag-viewer/stage-workload.js | 109 + tools/dag-viewer/test_render.py | 61 + tools/dag-viewer/test_server.py | 71 + tools/dag-viewer/viewer.js | 37 +- 11 files changed, 4532 insertions(+), 7103 deletions(-) create mode 100644 tools/dag-viewer/stage-workload.js create mode 100644 tools/dag-viewer/test_server.py diff --git a/tools/dag-viewer/dag.example.json b/tools/dag-viewer/dag.example.json index fdcedefed..c6782cd7b 100644 --- a/tools/dag-viewer/dag.example.json +++ b/tools/dag-viewer/dag.example.json @@ -1,308 +1,240 @@ { - "queries": [ - { - "name": "q1", - "source": "SELECT service, COUNT(*) FROM metrics GROUP BY service", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" + "format": "asap-stage-pipeline/v1", + "stage0_query_language_frontends": { + "dag": { + "nodes": [ + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" }, - "name": "bytes", - "nullable": false, - "table": "metrics" + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" } } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 0, - "hash": 6908994005896438407 + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] }, - { - "id": 1, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "kind": "count" + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Aggregate": { + "child": 0, + "filters": [], + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "kind": "count" + } + ], + "output_names": [ + "count(Int64(1))" + ], + "reduction": { + "Reduce": [ + 1 + ] } - ], - "output_names": [ - "count(Int64(1))" - ], - "reduction": { - "Reduce": [ - 1 - ] } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 0 - ], - "workload_node_id": 1, - "hash": 6455684992122519902, - "notes": [ + "name": "service", + "nullable": false, + "table": "metrics" + }, { - "kind": "SketchApproximation", - "reason": "count realizes as a Cms sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a CountSketch sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a shared HydraCms structure over Cms serving every subpopulation of this grouped aggregate, instead of one Cms instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and Cms has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's; count realizes as a shared HydraCountSketch structure over CountSketch serving every subpopulation of this grouped aggregate, instead of one CountSketch instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and CountSketch has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's" + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - { - "id": 2, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } - }, - { - "alias": "count(*)", - "expr": { - "Column": 1 - } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(*)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 1 - ], - "workload_node_id": 2, - "hash": 9357712178312893499 - } - ], - "root": 2 - }, - "replacements": [ + "timing": null + }, { - "decision_id": 0, - "target_pre_id": 1, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 1, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } }, - "source": { - "Table": { - "table_ref": "metrics" + { + "alias": "count(*)", + "expr": { + "Column": 1 } } + ], + "qualifier": null + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(*)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], "schema": { "closed": true, "fields": [ @@ -350,40 +282,214 @@ "time_index": 0, "unique_keys": [] }, - "children": [], - "hash": 6908994005896438407 + "source": { + "Table": { + "table_ref": "metrics" + } + } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, { - "id": 1, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "kind": "count" + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Aggregate": { + "child": 3, + "filters": [], + "having": null, + "measures": [ + { + "col": 3, + "kind": "avg" + } + ], + "output_names": [ + "avg(metrics.latency)" + ], + "reduction": { + "Reduce": [ + 1 + ] + } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 4, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": null, + "expr": { + "Column": 1 } - ], - "output_names": [ - "count(Int64(1))" - ], - "reduction": { - "Reduce": [ - 1 - ] } + ], + "qualifier": null + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], "schema": { "closed": true, "fields": [ { "dtype": { - "Plain": "utf8" + "Plain": "timestamp" }, - "name": "service", + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", "nullable": false, "table": "metrics" }, @@ -391,50 +497,378 @@ "dtype": { "Plain": "int64" }, - "name": "count(Int64(1))", + "name": "bytes", "nullable": false, - "table": null + "table": "metrics" } ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] + "time_index": 0, + "unique_keys": [] }, - "children": [ - 0 - ], - "hash": 6455684992122519902 + "source": { + "Table": { + "table_ref": "metrics" + } + } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" } ], - "root": 1 + "time_index": 0, + "unique_keys": [] }, - "after": { - "kind": "Summary", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "failure_probability": { - "op": "zero" + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "hosts" + } + } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Join": { + "kind": "Inner", + "left": 6, + "pred": { + "Compare": { + "left": { + "Column": 1 + }, + "op": "Eq", + "right": { + "Column": 5 + }, + "semantics": "sql" + } + }, + "right": 7 + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Aggregate": { + "child": 8, + "filters": [], + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 }, + "kind": "count" + } + ], + "output_names": [ + "count(Int64(1))" + ], + "reduction": { + "Reduce": [ + 1 + ] + } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 9, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": "count(*)", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(*)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 2 + }, + { + "Operator": 5 + }, + { + "Operator": 10 + } + ] + } + }, + "stage1_logical_asap_optimization": { + "candidates": [ + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { "predicates": [], "schema": { "closed": true, @@ -488,250 +922,130 @@ "table_ref": "metrics" } } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 384058483260984926 - }, - { - "id": 1, - "kind": "SummaryAgg", - "label": "SummaryAgg(Sketch(Cms))", - "detail": { - "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": "SampleValue" - }, - "weight": { - "Constant": 1.0 - }, - "weight_domain": { - "kind": "non_negative", - "proof": "unit_count" - } + { + "dtype": { + "Plain": "utf8" }, - "reduction": { - "Reduce": [ - 1 - ] - } + "name": "service", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "Cms", - "category": "Frequency", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, - "children": [ - 0 - ], - "hash": 15814889988858536838 - }, - { - "id": 2, - "kind": "SummaryEstimate", - "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", - "detail": { - "guarantee": { - "bound": { - "op": "constant", - "value": 0.009993683192864136 - }, - "failure_probability": { - "op": "constant", - "value": 0.006737946999085467 - }, - "metric": "frequency", - "provenance": [ - { - "algorithm": "Cms", - "contract": "count_min_l1_markov_v1", - "kind": "sketch_evaluation", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - }, - "query": "PointCount { key: SampleValue, value: None }" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "approximate_aggregate" - }, - "rule": "exact_input" - }, - { - "kind": "accuracy_target", - "target": { - "Epsilon": 0.01 - } - } - ] + { + "dtype": { + "Plain": "float64" }, - "query": "PointCount { key: SampleValue, value: None }" + "name": "latency", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 0, + "family": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null + "weight_domain": { + "kind": "unknown_or_signed" } - ], - "time_index": null, - "unique_keys": [ - [ - 0 + }, + "reduction": { + "Reduce": [ + 1 ] - ] - }, - "children": [ - 1 - ], - "hash": 15241615172005928158 + } + } } - ], - "root": 2 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - ], - "post_dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { "guarantee": { "bound": { "op": "zero" @@ -743,22 +1057,22 @@ "provenance": [ { "kind": "exact", - "reason": "RetainedExact" + "reason": "exact accumulator over exact input" } ] }, - "predicates": [], + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 1 + } + } + }, + "result_kind": "relation", "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, { "dtype": { "Plain": "utf8" @@ -767,812 +1081,182 @@ "nullable": false, "table": "metrics" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, { "dtype": { "Plain": "int64" }, - "name": "bytes", + "name": "count", "nullable": false, - "table": "metrics" + "table": null } ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" - } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 0, - "hash": 384058483260984926, - "decision": { - "id": 0, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 1, - "kind": "SummaryAgg", - "label": "SummaryAgg(Sketch(Cms))", - "detail": { - "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": "SampleValue" - }, - "weight": { - "Constant": 1.0 - }, - "weight_domain": { - "kind": "non_negative", - "proof": "unit_count" - } - }, - "reduction": { - "Reduce": [ - 1 - ] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "Cms", - "category": "Frequency", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] - ] + }, + "timing": null }, - "children": [ - 0 - ], - "workload_node_id": 1, - "hash": 15814889988858536838, - "decision": { - "id": 0, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 2, - "kind": "SummaryEstimate", - "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", - "detail": { + { "guarantee": { "bound": { - "op": "constant", - "value": 0.009993683192864136 + "op": "zero" }, "failure_probability": { - "op": "constant", - "value": 0.006737946999085467 + "op": "zero" }, - "metric": "frequency", + "metric": "absolute_value", "provenance": [ { - "algorithm": "Cms", - "contract": "count_min_l1_markov_v1", - "kind": "sketch_evaluation", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - }, - "query": "PointCount { key: SampleValue, value: None }" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "approximate_aggregate" - }, - "rule": "exact_input" - }, - { - "kind": "accuracy_target", - "target": { - "Epsilon": 0.01 - } + "kind": "exact", + "reason": "exact accumulator over exact input" } ] }, - "query": "PointCount { key: SampleValue, value: None }" - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 1 - ], - "workload_node_id": 2, - "hash": 15241615172005928158, - "decision": { - "id": 0, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 3, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } - }, - { - "alias": "count(*)", - "expr": { - "Column": 1 + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 2, + "cols": [ + { + "alias": "service", + "expr": { + "Column": 0 + } + }, + { + "alias": "count(Int64(1))", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null } } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(*)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2 - ], - "workload_node_id": 3, - "hash": 9357712178312893499 - } - ], - "root": 3 - }, - "workload_cost": { - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "name": "q2", - "source": "SELECT service, AVG(latency) FROM metrics GROUP BY service", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], + }, + "result_kind": "relation", "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, { "dtype": { "Plain": "utf8" }, "name": "service", "nullable": false, - "table": "metrics" + "table": null }, { "dtype": { - "Plain": "utf8" + "Plain": "int64" }, - "name": "region", + "name": "count(Int64(1))", "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" + "table": null } ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" - } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 0, - "hash": 6908994005896438407 - }, - { - "id": 1, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "col": 3, - "kind": "avg" - } - ], - "output_names": [ - "avg(metrics.latency)" - ], - "reduction": { - "Reduce": [ - 1 + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] - } + }, + "timing": null }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 0 - ], - "workload_node_id": 3, - "hash": 8656625452313770894 - }, - { - "id": 2, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } + "failure_probability": { + "op": "zero" }, - { - "alias": null, - "expr": { - "Column": 1 + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 ] - ] - }, - "children": [ - 1 - ], - "workload_node_id": 4, - "hash": 4102274562539239511 - } - ], - "root": 2 - }, - "replacements": [ - { - "decision_id": 1, - "target_pre_id": 1, - "strategy": "SemanticEquivalentRewriteStrategy", - "rationale": "Composes compatible nested aggregates using their declared algebraic intent while preserving the output schema.", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 3, + "cols": [ { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "alias": null, + "expr": { + "Column": 0 + } }, { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" + "alias": "count(*)", + "expr": { + "Column": 1 + } } ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" - } + "qualifier": null } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "int64" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 6908994005896438407 - }, - { - "id": 1, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "col": 3, - "kind": "avg" - } - ], - "output_names": [ - "avg(metrics.latency)" - ], - "reduction": { - "Reduce": [ - 1 - ] + "name": "count(*)", + "nullable": false, + "table": null } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] + ], + "time_index": null, + "unique_keys": [ + [ + 0 ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - "children": [ - 0 - ], - "hash": 8656625452313770894 - } - ], - "root": 1 - }, - "after": { - "kind": "Rewrite", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { "predicates": [], "schema": { "closed": true, @@ -1626,290 +1310,130 @@ "table_ref": "metrics" } } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "children": [], - "hash": 6908994005896438407 - }, - { - "id": 1, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "col": 3, - "kind": "sum" - } - ], - "output_names": [], - "reduction": { - "Reduce": [ - 1 - ] - } + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "sum", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" }, - "children": [ - 0 - ], - "hash": 10179062530039673039 - }, - { - "id": 2, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } - }, - { - "alias": "avg(metrics.latency)", - "expr": { - "Cast": { - "expr": { - "Column": 1 - }, - "to": "float64", - "try_cast": false + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 5, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" } } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 1 - ], - "hash": 5063523577177717082 - }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": "Exact", - "kind": "count" + "weight_domain": { + "kind": "unknown_or_signed" } - ], - "output_names": [], + }, "reduction": { "Reduce": [ 1 ] } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" ] - ] - }, - "children": [ + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ 0 - ], - "hash": 12295148967695211280 - }, - { - "id": 4, - "kind": "BinaryOp", - "label": "BinaryOp(/)", - "detail": { - "checked_finite_division": false, - "checked_relative_division": false, - "op": "/", - "return_bool": false, - "vector_match": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2, - 3 - ], - "hash": 5207996595762795408 - } - ], - "root": 4 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - ], - "post_dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { + ] + ] + }, + "timing": null + }, + { "guarantee": { "bound": { "op": "zero" @@ -1921,22 +1445,89 @@ "provenance": [ { "kind": "exact", - "reason": "RetainedExact" + "reason": "exact accumulator over exact input" } ] }, - "predicates": [], + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 + } + } + }, + "result_kind": "relation", "schema": { "closed": true, "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", + "name": "service", "nullable": false, "table": "metrics" }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 5, + "family": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ { "dtype": { "Plain": "utf8" @@ -1947,17 +1538,58 @@ }, { "dtype": { - "Plain": "utf8" + "ExactAggregate": [ + "Count", + "Count" + ] }, - "name": "region", + "name": "state", "nullable": false, - "table": "metrics" - }, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, - "name": "latency", + "name": "service", "nullable": false, "table": "metrics" }, @@ -1965,101 +1597,21 @@ "dtype": { "Plain": "int64" }, - "name": "bytes", + "name": "count", "nullable": false, - "table": "metrics" + "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, - "source": { - "Table": { - "table_ref": "metrics" - } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] + "timing": null }, - "children": [], - "workload_node_id": 0, - "hash": 384058483260984926, - "decision": { - "id": 7, - "strategy": "ASAPStrategies", - "rationale": "Sum { col: Some(3) } realizes as an exact Sum accumulator", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 1, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Sum))", - "detail": { - "family": "ExactAggregate(Sum, Sum)", - "grouping": "PerSubpopulationInstance", + { "guarantee": { "bound": { "op": "zero" @@ -2071,205 +1623,79 @@ "provenance": [ { "kind": "exact", - "reason": "ExactAggregate(Sum)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Join": { + "kind": "Inner", + "left": 7, + "pred": { + "BoolAnd": [ { - "kind": "exact", - "reason": "RetainedExact" + "Compare": { + "left": { + "Column": 0 + }, + "op": "Eq", + "right": { + "Column": 2 + }, + "semantics": "sql" + } } ] }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "exact_sum" - }, - "rule": "exact_input" - } - ] - }, - "input": { - "item": null, - "weight": { - "Column": { - "Qualified": { - "name": "latency", - "table": "metrics" - } + "right": 9 } - }, - "weight_domain": { - "kind": "unknown_or_signed" } }, - "reduction": { - "Reduce": [ - 1 - ] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "sum", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 0 - ], - "workload_node_id": 4, - "hash": 799178486167313977, - "decision": { - "id": 7, - "strategy": "ASAPStrategies", - "rationale": "Sum { col: Some(3) } realizes as an exact Sum accumulator", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 2, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } - }, - { - "alias": "avg(metrics.latency)", - "expr": { - "Cast": { - "expr": { - "Column": 1 - }, - "to": "float64", - "try_cast": false - } - } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "timing": null }, - "children": [ - 1 - ], - "workload_node_id": 5, - "hash": 5063523577177717082, - "decision": { - "id": 1, - "strategy": "SemanticEquivalentRewriteStrategy", - "rationale": "Composes compatible nested aggregates using their declared algebraic intent while preserving the output schema.", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 3, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Count))", - "detail": { - "family": "ExactAggregate(Count, Count)", - "grouping": "PerSubpopulationInstance", + { "guarantee": { "bound": { "op": "zero" @@ -2281,246 +1707,128 @@ "provenance": [ { "kind": "exact", - "reason": "ExactAggregate(Count): row count is independent of input values" + "reason": "exact over exact inputs" } ] }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 10, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": null, + "expr": { + "Arithmetic": { + "left": { + "Cast": { + "expr": { + "Column": 1 + }, + "to": "float64", + "try_cast": false + } + }, + "op": "Div", + "right": { + "Cast": { + "expr": { + "Column": 3 + }, + "to": "float64", + "try_cast": false + } + }, + "semantics": "sql" + } + } + } + ], + "qualifier": null + } } }, - "reduction": { - "Reduce": [ - 1 - ] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "ExactAggregate": [ - "Count", - "Count" - ] + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null }, - "name": "count", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 0 - ], - "workload_node_id": 6, - "hash": 7050639208991097747, - "decision": { - "id": 8, - "strategy": "ASAPStrategies", - "rationale": "count realizes as an exact Count accumulator", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 4, - "kind": "BinaryOp", - "label": "BinaryOp(/)", - "detail": { - "checked_finite_division": false, - "checked_relative_division": false, - "op": "/", - "return_bool": false, - "vector_match": null + { + "dtype": { + "Plain": "float64" + }, + "name": "col_1", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "timing": null }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2, - 3 - ], - "workload_node_id": 7, - "hash": 5207996595762795408, - "decision": { - "id": 1, - "strategy": "SemanticEquivalentRewriteStrategy", - "rationale": "Composes compatible nested aggregates using their declared algebraic intent while preserving the output schema.", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 5, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } + "failure_probability": { + "op": "zero" }, - { - "alias": null, - "expr": { - "Column": 1 + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "avg(metrics.latency)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 ] - ] - }, - "children": [ - 4 - ], - "workload_node_id": 8, - "hash": 4102274562539239511 - } - ], - "root": 5 - }, - "workload_cost": { - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "name": "q3", - "source": "topk(5, rate(http_requests_total[5m]))", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "service", + "expr": { + "Column": 0 + } + }, + { + "alias": "avg(metrics.latency)", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "relation", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", + "name": "service", "nullable": false, "table": null }, @@ -2528,345 +1836,104 @@ "dtype": { "Plain": "float64" }, - "name": "value", + "name": "avg(metrics.latency)", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [] }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "timing": null }, - "children": [], - "workload_node_id": 5, - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 - ], - "workload_node_id": 6, - "hash": 5944440839623837288 - }, - { - "id": 2, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "kind": "rate" - } - ], - "output_names": [ - "" - ], - "reduction": "PerEntity" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + "failure_probability": { + "op": "zero" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 1 - ], - "workload_node_id": 7, - "hash": 8785024119722619363 - }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "k": 5, - "kind": "top_k" - } - ], - "output_names": [], - "reduction": { - "Reduce": [] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk_5", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [] - }, - "children": [ - 2 - ], - "workload_node_id": 8, - "hash": 6186561604017396907 - } - ], - "root": 3 - }, - "replacements": [ - { - "decision_id": 3, - "target_pre_id": 2, - "strategy": "ASAPStrategies", - "rationale": "Rate realizes as an exact Rate accumulator", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, - "fields": [ + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 12, + "cols": [ { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + "alias": null, + "expr": { + "Column": 0 + } }, { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "alias": null, + "expr": { + "Column": 1 + } } ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } + "qualifier": null } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } ], - "hash": 5944440839623837288 + "time_index": null, + "unique_keys": [] }, - { - "id": 2, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "kind": "rate" - } - ], - "output_names": [ - "" - ], - "reduction": "PerEntity" + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "failure_probability": { + "op": "zero" }, - "children": [ - 1 - ], - "hash": 8785024119722619363 - } - ], - "root": 2 - }, - "after": { - "kind": "Summary", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -2874,973 +1941,355 @@ }, "name": "ts", "nullable": false, - "table": null + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, { "dtype": { "Plain": "float64" }, - "name": "value", + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", "nullable": false, - "table": null + "table": "metrics" } ], "time_index": 0, "unique_keys": [] }, "source": { - "TimeSeries": { - "metric": "http_requests_total" + "Table": { + "table_ref": "metrics" } } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "children": [], - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + { + "dtype": { + "Plain": "utf8" }, - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } + "name": "region", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" }, - "children": [ - 0 - ], - "hash": 2947655142598403385 + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "id": 2, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Rate))", - "detail": { - "family": "ExactAggregate(Rate, Rate)", - "grouping": "PerSubpopulationInstance", - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": "hosts" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "utf8" }, - "rule": "exact_input" + "name": "region", + "nullable": false, + "table": "hosts" } - ] + ], + "time_index": null, + "unique_keys": [] }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" + "source": { + "Table": { + "table_ref": "hosts" } + } + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "reduction": "PerEntity" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "name": "service", + "nullable": false, + "table": "hosts" }, - "children": [ - 1 - ], - "hash": 45212369069935119 - } - ], - "root": 2 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - }, - { - "decision_id": 2, - "target_pre_id": 3, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 + "name": "region", + "nullable": false, + "table": "hosts" } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 ], - "hash": 5944440839623837288 + "time_index": null, + "unique_keys": [] }, - { - "id": 2, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "kind": "rate" - } - ], - "output_names": [ - "" - ], - "reduction": "PerEntity" + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "failure_probability": { + "op": "zero" }, - "children": [ - 1 - ], - "hash": 8785024119722619363 - }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "k": 5, - "kind": "top_k" - } - ], - "output_names": [], - "reduction": { - "Reduce": [] + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk_5", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [] - }, - "children": [ - 2 - ], - "hash": 6186561604017396907 - } - ], - "root": 3 - }, - "after": { - "kind": "Summary", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Join": { + "kind": "Inner", + "left": 14, + "pred": { + "Compare": { + "left": { + "Column": 1 }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" + "op": "Eq", + "right": { + "Column": 5 }, - "name": "value", - "nullable": false, - "table": null + "semantics": "sql" } - ], - "time_index": 0, - "unique_keys": [] + }, + "right": 15 + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "children": [], - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + { + "dtype": { + "Plain": "utf8" }, - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } + "name": "service", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, - "children": [ - 0 - ], - "hash": 2947655142598403385 - }, - { - "id": 2, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Rate))", - "detail": { - "family": "ExactAggregate(Rate, Rate)", - "grouping": "PerSubpopulationInstance", - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" - } - ] + { + "dtype": { + "Plain": "float64" }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" }, - "reduction": "PerEntity" + "name": "bytes", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" }, - "children": [ - 1 - ], - "hash": 45212369069935119 - }, - { - "id": 3, - "kind": "FinalizeExactAccumulator", - "label": "FinalizeExactAccumulator", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" - } - ] + { + "dtype": { + "Plain": "utf8" }, - "timing": "query_time" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 2 - ], - "hash": 2647395323701112382 - }, - { - "id": 4, - "kind": "Sort", - "label": "Sort(1 keys)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" - } + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 16, + "family": { + "ExactAggregate": [ + "Count", + "Count" ] }, - "keys": [ - { - "ascending": false, - "expr": { - "Column": 1 - }, - "nulls_first": false - } - ], - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 3 - ], - "hash": 6807625048810781615 - }, - { - "id": 5, - "kind": "Limit", - "label": "Limit(5)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } } - ] - }, - "n": 5, - "offset": 0, - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "weight_domain": { + "kind": "unknown_or_signed" } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 4 - ], - "hash": 4927452090234187032 + }, + "reduction": { + "Reduce": [ + 1 + ] + } + } } - ], - "root": 5 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - ], - "post_dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], + }, + "result_kind": "state", "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", + "name": "service", "nullable": false, - "table": null + "table": "metrics" }, { "dtype": { - "Plain": "float64" + "ExactAggregate": [ + "Count", + "Count" + ] }, - "name": "value", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "timing": null }, - "children": [], - "workload_node_id": 9, - "hash": 16928591705169832851, - "decision": { - "id": 2, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { + { "guarantee": { "bound": { "op": "zero" @@ -3852,187 +2301,49 @@ "provenance": [ { "kind": "exact", - "reason": "RetainedExact" + "reason": "exact accumulator over exact input" } ] }, - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 17 + } } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 - ], - "workload_node_id": 10, - "hash": 2947655142598403385, - "decision": { - "id": 2, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 2, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Rate))", - "detail": { - "family": "ExactAggregate(Rate, Rate)", - "grouping": "PerSubpopulationInstance", - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": "metrics" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "int64" }, - "rule": "exact_input" + "name": "count", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": "PerEntity" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "timing": null }, - "children": [ - 1 - ], - "workload_node_id": 11, - "hash": 45212369069935119, - "decision": { - "id": 2, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 3, - "kind": "FinalizeExactAccumulator", - "label": "FinalizeExactAccumulator", - "detail": { + { "guarantee": { "bound": { "op": "zero" @@ -4044,95 +2355,64 @@ "provenance": [ { "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 18, + "cols": [ + { + "alias": "service", + "expr": { + "Column": 0 + } }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" + { + "alias": "count(Int64(1))", + "expr": { + "Column": 1 } - ] + } + ], + "qualifier": null + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": null }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "int64" }, - "rule": "exact_input" + "name": "count(Int64(1))", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "timing": "query_time" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "timing": null }, - "children": [ - 2 - ], - "workload_node_id": 12, - "hash": 2647395323701112382, - "decision": { - "id": 2, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 4, - "kind": "Sort", - "label": "Sort(1 keys)", - "detail": { + { "guarantee": { "bound": { "op": "zero" @@ -4144,235 +2424,97 @@ "provenance": [ { "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" + "reason": "exact accumulator over exact input" } ] }, - "keys": [ - { - "ascending": false, - "expr": { - "Column": 1 - }, - "nulls_first": false - } - ], - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 3 - ], - "workload_node_id": 13, - "hash": 6807625048810781615, - "decision": { - "id": 2, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 5, - "kind": "Limit", - "label": "Limit(5)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 19, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" + { + "alias": "count(*)", + "expr": { + "Column": 1 } - ] + } + ], + "qualifier": null + } + } + }, + "result_kind": "relation", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": null }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "int64" }, - "rule": "exact_input" + "name": "count(*)", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "n": 5, - "offset": 0, - "partition_by": [] + "timing": null + } + ], + "roots": [ + { + "Operator": 4 }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "Operator": 13 }, - "children": [ - 4 - ], - "workload_node_id": 14, - "hash": 4927452090234187032, - "decision": { - "id": 2, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } + { + "Operator": 20 } - } - ], - "root": 5 - }, - "workload_cost": { - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" + ] }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } + "display_name": "Q1 exact (Count acc) · Q2 exact (Sum/Count acc) · Q3 exact (Count acc)", + "id": "L1" } - }, - { - "name": "q4", - "source": "topk(10, rate(http_requests_total[5m]))", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, + ], + "capped": true, + "combinations": 16 + }, + "stage2_physical_asap_optimization": { + "candidates": [ + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, "fields": [ { "dtype": { @@ -4380,776 +2522,917 @@ }, "name": "ts", "nullable": false, - "table": null + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, { "dtype": { "Plain": "float64" }, - "name": "value", + "name": "latency", "nullable": false, - "table": null + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" } ], "time_index": 0, "unique_keys": [] }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } + "producer": 0, + "role": "Input", + "window": "NotApplicable" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" + { + "consumer": 2, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 5, - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" + { + "consumer": 3, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 - ], - "workload_node_id": 6, - "hash": 5944440839623837288 - }, - { - "id": 2, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "kind": "rate" - } - ], - "output_names": [ - "" - ], - "reduction": "PerEntity" + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" }, - "children": [ - 1 - ], - "workload_node_id": 7, - "hash": 8785024119722619363 - }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 + { + "consumer": 6, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "k": 10, - "kind": "top_k" - } - ], - "output_names": [], - "reduction": { - "Reduce": [] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "topk_10", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [] - }, - "children": [ - 2 - ], - "workload_node_id": 9, - "hash": 17276237346142150766 - } - ], - "root": 3 - }, - "replacements": [ - { - "decision_id": 3, - "target_pre_id": 2, - "strategy": "ASAPStrategies", - "rationale": "Rate realizes as an exact Rate accumulator", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + { + "dtype": { + "Plain": "float64" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 16928591705169832851 + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 + "producer": 5, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } ], - "hash": 5944440839623837288 + "time_index": 0, + "unique_keys": [] }, - { - "id": 2, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "kind": "rate" - } - ], - "output_names": [ - "" - ], - "reduction": "PerEntity" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + "producer": 5, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 1 + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "state", + "nullable": false, + "table": null + } ], - "hash": 8785024119722619363 - } - ], - "root": 2 - }, - "after": { - "kind": "Summary", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 7, + "role": "Left", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 9, + "role": "Right", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "col_1", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 13, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 12, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 16, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 14, + "role": "Left", + "window": "NotApplicable" + }, + { + "consumer": 16, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "name": "service", + "nullable": false, + "table": "hosts" }, - "children": [], - "hash": 16928591705169832851 - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + { + "dtype": { + "Plain": "utf8" }, - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "producer": 15, + "role": "Right", + "window": "NotApplicable" + }, + { + "consumer": 17, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "children": [ - 0 - ], - "hash": 2947655142598403385 - }, - { - "id": 2, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Rate))", - "detail": { - "family": "ExactAggregate(Rate, Rate)", - "grouping": "PerSubpopulationInstance", - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" - } - ] + { + "dtype": { + "Plain": "utf8" }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" }, - "reduction": "PerEntity" + "name": "latency", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" }, - "children": [ - 1 - ], - "hash": 45212369069935119 - } - ], - "root": 2 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - }, - { - "decision_id": 4, - "target_pre_id": 3, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + ], + "time_index": 0, + "unique_keys": [] + }, + "producer": 16, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 18, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 16928591705169832851 + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 + "producer": 17, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 19, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 18, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 20, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } ], - "hash": 5944440839623837288 + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, - { - "id": 2, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "kind": "rate" - } - ], - "output_names": [ - "" - ], - "reduction": "PerEntity" + "producer": 19, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "failure_probability": { + "op": "zero" }, - "children": [ - 1 - ], - "hash": 8785024119722619363 + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "k": 10, - "kind": "top_k" - } - ], - "output_names": [], - "reduction": { - "Reduce": [] + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk_10", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [] - }, - "children": [ - 2 ], - "hash": 17276237346142150766 - } - ], - "root": 3 - }, - "after": { - "kind": "Summary", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { "predicates": [], "schema": { - "closed": false, + "closed": true, "fields": [ { "dtype": { @@ -5157,548 +3440,530 @@ }, "name": "ts", "nullable": false, - "table": null + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, { "dtype": { "Plain": "float64" }, - "name": "value", + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", "nullable": false, - "table": null + "table": "metrics" } ], "time_index": 0, "unique_keys": [] }, "source": { - "TimeSeries": { - "metric": "http_requests_total" + "Table": { + "table_ref": "metrics" + } + } + } + } + } + }, + { + "guarantee": null, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 0, + "family": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" } + }, + "reduction": { + "Reduce": [ + 1 + ] } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 1 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null }, - "schema": { - "closed": false, - "fields": [ + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 2, + "cols": [ { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + "alias": "service", + "expr": { + "Column": 0 + } }, { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "alias": "count(Int64(1))", + "expr": { + "Column": 1 + } } ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 16928591705169832851 + "qualifier": null + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } + "name": "service", + "nullable": false, + "table": null }, - "schema": { - "closed": false, - "fields": [ + { + "dtype": { + "Plain": "int64" + }, + "name": "count(*)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 3, + "cols": [ { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null + "alias": null, + "expr": { + "Column": 0 + } }, { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "alias": "count(*)", + "expr": { + "Column": 1 + } } ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 - ], - "hash": 2947655142598403385 + "qualifier": null + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "id": 2, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Rate))", - "detail": { - "family": "ExactAggregate(Rate, Rate)", - "grouping": "PerSubpopulationInstance", - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" - } - ] - }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" }, - "reduction": "PerEntity" + "name": "ts", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "children": [ - 1 - ], - "hash": 45212369069935119 - }, - { - "id": 3, - "kind": "FinalizeExactAccumulator", - "label": "FinalizeExactAccumulator", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" - } - ] + { + "dtype": { + "Plain": "utf8" }, - "timing": "query_time" + "name": "region", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" }, - "children": [ - 2 - ], - "hash": 2647395323701112382 - }, - { - "id": 4, - "kind": "Sort", - "label": "Sort(1 keys)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "timestamp" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "ts", + "nullable": false, + "table": "metrics" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "utf8" }, - "rule": "exact_input" - } - ] - }, - "keys": [ - { - "ascending": false, - "expr": { - "Column": 1 - }, - "nulls_first": false - } - ], - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" + "name": "service", + "nullable": false, + "table": "metrics" }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 3 - ], - "hash": 6807625048810781615 - }, - { - "id": 5, - "kind": "Limit", - "label": "Limit(10)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ { - "kind": "exact", - "reason": "ExactAggregate(Rate)" + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + { + "dtype": { + "Plain": "float64" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "latency", + "nullable": false, + "table": "metrics" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "int64" }, - "rule": "exact_input" + "name": "bytes", + "nullable": false, + "table": "metrics" } - ] + ], + "time_index": 0, + "unique_keys": [] }, - "n": 10, - "offset": 0, - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "source": { + "Table": { + "table_ref": "metrics" } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 4 - ], - "hash": 3322862873989621581 + } + } } - ], - "root": 5 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - ], - "post_dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(http_requests_total)", - "detail": { - "predicates": [], - "schema": { - "closed": false, + } + }, + { + "guarantee": null, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", + "name": "service", "nullable": false, - "table": null + "table": "metrics" }, { "dtype": { - "Plain": "float64" + "ExactAggregate": [ + "Sum", + "Sum" + ] }, - "name": "value", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 5, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + } } } }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 9, - "hash": 16928591705169832851, - "decision": { - "id": 4, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 1, - "kind": "TimeRange", - "label": "TimeRange(Range, 300s)", - "detail": { + { "guarantee": { "bound": { "op": "zero" @@ -5710,287 +3975,125 @@ "provenance": [ { "kind": "exact", - "reason": "RetainedExact" + "reason": "exact accumulator over exact input" } ] }, - "kind": "range", - "range": { - "nanos": 0, - "secs": 300 - } - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0 - ], - "workload_node_id": 10, - "hash": 2947655142598403385, - "decision": { - "id": 4, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 2, - "kind": "SummaryAgg", - "label": "SummaryAgg(ExactAggregate(Rate))", - "detail": { - "family": "ExactAggregate(Rate, Rate)", - "grouping": "PerSubpopulationInstance", - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": "metrics" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "float64" }, - "rule": "exact_input" + "name": "sum", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } + "output_state": { + "primitive": "Raw", + "timing": "query_time" }, - "reduction": "PerEntity" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "value", - "nullable": false, - "table": null + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 + } } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 1 - ], - "workload_node_id": 11, - "hash": 45212369069935119, - "decision": { - "id": 4, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" } - } - }, - { - "id": 3, - "kind": "FinalizeExactAccumulator", - "label": "FinalizeExactAccumulator", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": "metrics" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] }, - "rule": "exact_input" + "name": "state", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "timing": "query_time" - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 5, + "family": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + } } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 2 - ], - "workload_node_id": 12, - "hash": 2647395323701112382, - "decision": { - "id": 4, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" } - } - }, - { - "id": 4, - "kind": "Sort", - "label": "Sort(1 keys)", - "detail": { + }, + { "guarantee": { "bound": { "op": "zero" @@ -6002,104 +4105,52 @@ "provenance": [ { "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": "metrics" }, { - "kind": "composition_step", - "operator": { - "op": "counter_rate" + "dtype": { + "Plain": "int64" }, - "rule": "exact_input" + "name": "count", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "keys": [ - { - "ascending": false, - "expr": { - "Column": 1 - }, - "nulls_first": false - } - ], - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 3 - ], - "workload_node_id": 13, - "hash": 6807625048810781615, - "decision": { - "id": 4, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" } - } - }, - { - "id": 5, - "kind": "Limit", - "label": "Limit(10)", - "detail": { + }, + { "guarantee": { "bound": { "op": "zero" @@ -6107,139 +4158,19 @@ "failure_probability": { "op": "zero" }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "ExactAggregate(Rate)" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, + "metric": "absolute_value", + "provenance": [ { - "kind": "composition_step", - "operator": { - "op": "counter_rate" - }, - "rule": "exact_input" + "kind": "exact", + "reason": "exact over exact inputs" } ] }, - "n": 10, - "offset": 0, - "partition_by": [] - }, - "schema": { - "closed": false, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 4 - ], - "workload_node_id": 15, - "hash": 3322862873989621581, - "decision": { - "id": 4, - "strategy": "ASAPStrategies", - "rationale": "select exact Top-K from independently maintained temporal values", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - } - ], - "root": 5 - }, - "workload_cost": { - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "name": "q6", - "source": "SELECT metrics.service, COUNT(*) FROM metrics JOIN hosts ON metrics.service = hosts.service GROUP BY metrics.service", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], - "schema": { + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, { "dtype": { "Plain": "utf8" @@ -6250,17 +4181,17 @@ }, { "dtype": { - "Plain": "utf8" + "Plain": "float64" }, - "name": "region", + "name": "sum", "nullable": false, - "table": "metrics" + "table": null }, { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, - "name": "latency", + "name": "service", "nullable": false, "table": "metrics" }, @@ -6268,78 +4199,63 @@ "dtype": { "Plain": "int64" }, - "name": "bytes", + "name": "count", "nullable": false, - "table": "metrics" + "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [] }, - "source": { - "Table": { - "table_ref": "metrics" + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Join": { + "kind": "Inner", + "left": 7, + "pred": { + "BoolAnd": [ + { + "Compare": { + "left": { + "Column": 0 + }, + "op": "Eq", + "right": { + "Column": 2 + }, + "semantics": "sql" + } + } + ] + }, + "right": 9 + } } } }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "failure_probability": { + "op": "zero" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 0, - "hash": 6908994005896438407 - }, - { - "id": 1, - "kind": "Scan", - "label": "Scan(hosts)", - "detail": { - "predicates": [], - "schema": { + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ { @@ -6348,596 +4264,276 @@ }, "name": "service", "nullable": false, - "table": "hosts" + "table": null }, { "dtype": { - "Plain": "utf8" + "Plain": "float64" }, - "name": "region", + "name": "col_1", "nullable": false, - "table": "hosts" + "table": null } ], "time_index": null, "unique_keys": [] }, - "source": { - "Table": { - "table_ref": "hosts" - } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": null, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 10, - "hash": 749177952914250128 - }, - { - "id": 2, - "kind": "Join", - "label": "Join(Inner)", - "detail": { - "kind": "Inner", - "pred": { - "Compare": { - "left": { - "Column": 1 - }, - "op": "Eq", - "right": { - "Column": 5 - }, - "semantics": "sql" + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 10, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": null, + "expr": { + "Arithmetic": { + "left": { + "Cast": { + "expr": { + "Column": 1 + }, + "to": "float64", + "try_cast": false + } + }, + "op": "Div", + "right": { + "Cast": { + "expr": { + "Column": 3 + }, + "to": "float64", + "try_cast": false + } + }, + "semantics": "sql" + } + } + } + ], + "qualifier": null + } } } }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" + "failure_probability": { + "op": "zero" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0, - 1 - ], - "workload_node_id": 11, - "hash": 3592026688202666165 - }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "kind": "count" - } - ], - "output_names": [ - "count(Int64(1))" - ], - "reduction": { - "Reduce": [ - 1 + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } ] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" + }, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2 - ], - "workload_node_id": 12, - "hash": 16087426938996590503, - "notes": [ - { - "kind": "SketchApproximation", - "reason": "count realizes as a Cms sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a CountSketch sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a shared HydraCms structure over Cms serving every subpopulation of this grouped aggregate, instead of one Cms instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and Cms has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's; count realizes as a shared HydraCountSketch structure over CountSketch serving every subpopulation of this grouped aggregate, instead of one CountSketch instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and CountSketch has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's" - } - ] - }, - { - "id": 4, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } - }, - { - "alias": "count(*)", - "expr": { - "Column": 1 + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(*)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 3 - ], - "workload_node_id": 13, - "hash": 7086496332485428501 - } - ], - "root": 4 - }, - "replacements": [ - { - "decision_id": 6, - "target_pre_id": 3, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "before": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "alias": "service", + "expr": { + "Column": 0 + } }, { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" + "alias": "avg(metrics.latency)", + "expr": { + "Column": 1 + } } ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" - } + "qualifier": null } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 13, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "hash": 6908994005896438407 + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] }, - { - "id": 1, - "kind": "Scan", - "label": "Scan(hosts)", - "detail": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 12, + "cols": [ { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" + "alias": null, + "expr": { + "Column": 0 + } }, { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" + "alias": null, + "expr": { + "Column": 1 + } } ], - "time_index": null, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "hosts" - } + "qualifier": null } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": null, - "unique_keys": [] + "failure_probability": { + "op": "zero" }, - "children": [], - "hash": 749177952914250128 - }, - { - "id": 2, - "kind": "Join", - "label": "Join(Inner)", - "detail": { - "kind": "Inner", - "pred": { - "Compare": { - "left": { - "Column": 1 - }, - "op": "Eq", - "right": { - "Column": 5 - }, - "semantics": "sql" - } + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" + ] + }, + "id": 14, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0, - 1 - ], - "hash": 3592026688202666165 - }, - { - "id": 3, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": { - "having": null, - "measures": [ - { - "accuracy": { - "Epsilon": 0.01 - }, - "kind": "count" - } - ], - "output_names": [ - "count(Int64(1))" - ], - "reduction": { - "Reduce": [ - 1 - ] + "name": "bytes", + "nullable": false, + "table": "metrics" } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2 ], - "hash": 16087426938996590503 - } - ], - "root": 3 - }, - "after": { - "kind": "Summary", - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { "predicates": [], "schema": { "closed": true, @@ -6986,67 +4582,63 @@ "time_index": 0, "unique_keys": [] }, - "source": { - "Table": { - "table_ref": "metrics" - } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] + "source": { + "Table": { + "table_ref": "metrics" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 15, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" }, - "children": [], - "hash": 6908994005896438407 - }, - { - "id": 1, - "kind": "Scan", - "label": "Scan(hosts)", - "detail": { + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { "predicates": [], "schema": { "closed": true, @@ -7076,54 +4668,100 @@ "table_ref": "hosts" } } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 16, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": null, - "unique_keys": [] + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "children": [], - "hash": 749177952914250128 - }, - { - "id": 2, - "kind": "Join", - "label": "Join(Inner)", - "detail": { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Join": { "kind": "Inner", + "left": 14, "pred": { "Compare": { "left": { @@ -7135,94 +4773,74 @@ }, "semantics": "sql" } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": 0, - "unique_keys": [] + }, + "right": 15 + } + } + } + }, + { + "guarantee": null, + "id": 17, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" }, - "children": [ - 0, - 1 - ], - "hash": 3667654325373069547 - }, - { - "id": 3, - "kind": "SummaryAgg", - "label": "SummaryAgg(Sketch(Cms))", - "detail": { - "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 16, + "family": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": { - "Column": "SampleValue" - }, + "item": null, "weight": { - "Constant": 1.0 + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } }, "weight_domain": { - "kind": "non_negative", - "proof": "unit_count" + "kind": "unknown_or_signed" } }, "reduction": { @@ -7230,187 +4848,31 @@ 1 ] } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "Cms", - "category": "Frequency", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2 - ], - "hash": 1063284308144682453 - }, - { - "id": 4, - "kind": "SummaryEstimate", - "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", - "detail": { - "guarantee": { - "bound": { - "op": "constant", - "value": 0.009993683192864136 - }, - "failure_probability": { - "op": "constant", - "value": 0.006737946999085467 - }, - "metric": "frequency", - "provenance": [ - { - "algorithm": "Cms", - "contract": "count_min_l1_markov_v1", - "kind": "sketch_evaluation", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - }, - "query": "PointCount { key: SampleValue, value: None }" - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] - }, - "input_index": 0, - "kind": "child_guarantee" - }, - { - "kind": "composition_step", - "operator": { - "op": "approximate_aggregate" - }, - "rule": "exact_input" - }, - { - "kind": "accuracy_target", - "target": { - "Epsilon": 0.01 - } - } - ] - }, - "query": "PointCount { key: SampleValue, value: None }" - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 3 - ], - "hash": 18298277158829120767 + } } - ], - "root": 4 - } - }, - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - ], - "post_dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "detail": { - "predicates": [], - "schema": { + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 18, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, { "dtype": { "Plain": "utf8" @@ -7419,121 +4881,53 @@ "nullable": false, "table": "metrics" }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, { "dtype": { "Plain": "int64" }, - "name": "bytes", + "name": "count", "nullable": false, - "table": "metrics" + "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" }, - "source": { - "Table": { - "table_ref": "metrics" + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 17 + } } } }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" + { + "guarantee": { + "bound": { + "op": "zero" }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" + "failure_probability": { + "op": "zero" }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 16, - "hash": 6908994005896438407, - "decision": { - "id": 6, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 1, - "kind": "Scan", - "label": "Scan(hosts)", - "detail": { - "predicates": [], - "schema": { + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 19, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ { @@ -7542,81 +4936,52 @@ }, "name": "service", "nullable": false, - "table": "hosts" + "table": null }, { "dtype": { - "Plain": "utf8" + "Plain": "int64" }, - "name": "region", + "name": "count(Int64(1))", "nullable": false, - "table": "hosts" + "table": null } ], "time_index": null, - "unique_keys": [] + "unique_keys": [ + [ + 0 + ] + ] }, - "source": { - "Table": { - "table_ref": "hosts" + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 18, + "cols": [ + { + "alias": "service", + "expr": { + "Column": 0 + } + }, + { + "alias": "count(Int64(1))", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } } } }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": null, - "unique_keys": [] - }, - "children": [], - "workload_node_id": 17, - "hash": 749177952914250128, - "decision": { - "id": 6, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 2, - "kind": "Join", - "label": "Join(Inner)", - "detail": { + { "guarantee": { "bound": { "op": "zero" @@ -7628,421 +4993,231 @@ "provenance": [ { "kind": "exact", - "reason": "RetainedExact" + "reason": "exact accumulator over exact input" } ] }, - "kind": "Inner", - "pred": { - "Compare": { - "left": { - "Column": 1 - }, - "op": "Eq", - "right": { - "Column": 5 - }, - "semantics": "sql" - } - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "bytes", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "hosts" - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "region", - "nullable": false, - "table": "hosts" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "children": [ - 0, - 1 - ], - "workload_node_id": 18, - "hash": 3667654325373069547, - "decision": { - "id": 6, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 3, - "kind": "SummaryAgg", - "label": "SummaryAgg(Sketch(Cms))", - "detail": { - "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": "SampleValue" - }, - "weight": { - "Constant": 1.0 - }, - "weight_domain": { - "kind": "non_negative", - "proof": "unit_count" - } - }, - "reduction": { - "Reduce": [ - 1 - ] - } - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "Cms", - "category": "Frequency", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 2 - ], - "workload_node_id": 19, - "hash": 1063284308144682453, - "decision": { - "id": 6, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_region", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } - } - }, - { - "id": 4, - "kind": "SummaryEstimate", - "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", - "detail": { - "guarantee": { - "bound": { - "op": "constant", - "value": 0.009993683192864136 - }, - "failure_probability": { - "op": "constant", - "value": 0.006737946999085467 - }, - "metric": "frequency", - "provenance": [ - { - "algorithm": "Cms", - "contract": "count_min_l1_markov_v1", - "kind": "sketch_evaluation", - "params": { - "Cms": { - "depth": 5, - "width": 272 - } - }, - "query": "PointCount { key: SampleValue, value: None }" - }, + "id": 20, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "RetainedExact" - } - ] + "dtype": { + "Plain": "utf8" }, - "input_index": 0, - "kind": "child_guarantee" + "name": "service", + "nullable": false, + "table": null }, { - "kind": "composition_step", - "operator": { - "op": "approximate_aggregate" + "dtype": { + "Plain": "int64" }, - "rule": "exact_input" - }, - { - "kind": "accuracy_target", - "target": { - "Epsilon": 0.01 - } + "name": "count(*)", + "nullable": false, + "table": null } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] ] }, - "query": "PointCount { key: SampleValue, value: None }" - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(Int64(1))", - "nullable": false, - "table": null + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 19, + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": "count(*)", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 3 - ], - "workload_node_id": 20, - "hash": 18298277158829120767, - "decision": { - "id": 6, - "strategy": "ASAPStrategies", - "rationale": "count realizes as a Cms sketch", - "rank": 0, - "cost": null, - "role": "replacement_root", - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" } } + ], + "roots": [ + 4, + 13, + 20 + ], + "scalar_roots": [] + }, + "display_name": "Q1 exact (Count acc) · Q2 exact (Sum/Count acc) · Q3 exact (Count acc)", + "from_logical": "L1", + "id": "P1" + } + ], + "pruned": [] + }, + "stage3_plan_selection": { + "costs": { + "P1": { + "per_node": { + "0": { + "cost": 0.39599999999999996, + "detail": "scan 60000 samples", + "multiplier": 1 }, - { - "id": 5, - "kind": "Project", - "label": "Project(2 cols)", - "detail": { - "cols": [ - { - "alias": null, - "expr": { - "Column": 0 - } - }, - { - "alias": "count(*)", - "expr": { - "Column": 1 - } - } - ], - "qualifier": null - }, - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "service", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "int64" - }, - "name": "count(*)", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "children": [ - 4 - ], - "workload_node_id": 21, - "hash": 7086496332485428501 + "1": { + "cost": 0.06, + "detail": "build exact Count accumulator into 100 states: 60000 rows x depth 1", + "multiplier": 1 + }, + "10": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "11": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "12": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "13": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "14": { + "cost": 0.39599999999999996, + "detail": "scan 60000 samples", + "multiplier": 1 + }, + "15": { + "cost": 0.252, + "detail": "scan 60000 samples", + "multiplier": 1 + }, + "16": { + "cost": 0.06, + "detail": "operator: pass 60000 rows", + "multiplier": 1 + }, + "17": { + "cost": 0.06, + "detail": "build exact Count accumulator into 100 states: 60000 rows x depth 1", + "multiplier": 1 + }, + "18": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + }, + "19": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "2": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + }, + "20": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "3": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "4": { + "cost": 0.00009999999999999999, + "detail": "operator: pass 100 rows", + "multiplier": 1 + }, + "5": { + "cost": 0.39599999999999996, + "detail": "scan 60000 samples", + "multiplier": 1 + }, + "6": { + "cost": 0.06, + "detail": "build exact Sum accumulator into 100 states: 60000 rows x depth 1", + "multiplier": 1 + }, + "7": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + }, + "8": { + "cost": 0.06, + "detail": "build exact Count accumulator into 100 states: 60000 rows x depth 1", + "multiplier": 1 + }, + "9": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + } + }, + "source": "analytical-cost-v1 (illustrative statistics)", + "total": 1.7411999999999999, + "unit": "cpu_ms_per_workload_evaluation" + } + }, + "rejected": [], + "selected": "P1" + }, + "workload": { + "queries": [ + { + "id": "q1", + "language": "sql", + "requirements": { + "accuracy": { + "epsilon": 0.01 } - ], - "root": 5 + }, + "text": "SELECT service, COUNT(*) FROM metrics GROUP BY service" }, - "workload_cost": { - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" + { + "id": "q2", + "language": "sql", + "requirements": { + "accuracy": { + "epsilon": 0.01 + } }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" + "text": "SELECT service, AVG(latency) FROM metrics GROUP BY service" + }, + { + "id": "q3", + "language": "sql", + "requirements": { + "accuracy": { + "epsilon": 0.01 + } }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } + "text": "SELECT metrics.service, COUNT(*) FROM metrics JOIN hosts ON metrics.service = hosts.service GROUP BY metrics.service" } - } - ], - "workload_cost": { - "baseline_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "selected_cost": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - }, - "benefit": { - "value": null, - "unit": "CostUnits", - "source": "Unavailable" - } + ] } } diff --git a/tools/dag-viewer/generate-sample.sh b/tools/dag-viewer/generate-sample.sh index 15662d30b..def15c68a 100755 --- a/tools/dag-viewer/generate-sample.sh +++ b/tools/dag-viewer/generate-sample.sh @@ -1,24 +1,22 @@ #!/usr/bin/env bash -# Regenerates tools/dag-viewer/dag.example.json from real example queries. -# Run from anywhere in the repo; edit the --sql/--promql lines below to try -# your own queries instead. +# Regenerates tools/dag-viewer/dag.example.json, the stage_pipeline document +# the viewer opens with: the Query planner's default SQL workload over its +# default tables. Run from anywhere in the repo; edit the --table/--sql lines +# below to try your own queries (or use --promql; a run is one language). set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." # --epsilon asks for an approximate accuracy target instead of the default -# Exact, so ASAPStrategies actually has a sketch alternative to -# report — without it, no query below would ever pick up a `notes` badge -# (see crates/devtools/src/bin/dag_export.rs's own `--epsilon` doc comment). -cargo run -p asap-devtools --bin dag_export -- \ - --post-asap --progress \ - --epsilon 0.01 \ - --data-ingestion-interval-ms 15000 \ - --default-cost \ - --sql "SELECT service, COUNT(*) FROM metrics GROUP BY service" --name q1 \ - --sql "SELECT service, AVG(latency) FROM metrics GROUP BY service" --name q2 \ - --promql "topk(5, rate(http_requests_total[5m]))" --name q3 \ - --promql "topk(10, rate(http_requests_total[5m]))" --name q4 \ - --sql "SELECT metrics.service, COUNT(*) FROM metrics JOIN hosts ON metrics.service = hosts.service GROUP BY metrics.service" --name q6 \ - > tools/dag-viewer/dag.example.json +# Exact, so Stage 1 has sketch candidates to offer. The viewer shows the +# selected plan, so the document lists only that candidate (display only: +# Stage 3 still selects among all of them). +cargo run -q -p asap-devtools --bin stage_pipeline -- \ + --epsilon 0.01 --max-displayed 0 \ + --table '{"name":"metrics","columns":[{"name":"ts","type":"timestamp","nullable":false},{"name":"service","type":"utf8","nullable":false},{"name":"region","type":"utf8","nullable":false},{"name":"latency","type":"float64","nullable":false},{"name":"bytes","type":"int64","nullable":false}],"time_index":0}' \ + --table '{"name":"hosts","columns":[{"name":"service","type":"utf8","nullable":false},{"name":"region","type":"utf8","nullable":false}]}' \ + --sql "SELECT service, COUNT(*) FROM metrics GROUP BY service" \ + --sql "SELECT service, AVG(latency) FROM metrics GROUP BY service" \ + --sql "SELECT metrics.service, COUNT(*) FROM metrics JOIN hosts ON metrics.service = hosts.service GROUP BY metrics.service" \ + --out tools/dag-viewer/dag.example.json echo "wrote tools/dag-viewer/dag.example.json" diff --git a/tools/dag-viewer/index.html b/tools/dag-viewer/index.html index 17754d280..c657df82a 100644 --- a/tools/dag-viewer/index.html +++ b/tools/dag-viewer/index.html @@ -8,6 +8,7 @@ +

ASAP query DAG viewer

-
Drop WorkloadDAG JSON here, or click to choose file(s)
- +
Drop a stage_pipeline document here, or click to choose one
+
diff --git a/tools/dag-viewer/render.py b/tools/dag-viewer/render.py index 9e478fc08..07020462d 100755 --- a/tools/dag-viewer/render.py +++ b/tools/dag-viewer/render.py @@ -1,62 +1,38 @@ #!/usr/bin/env python3 -"""Bake a stage_pipeline document (or WorkloadDAG JSON) into a single, portable Pre/Post-ASAP HTML page, -with the same query selection, workload-union, and node details as index.html, -but with -the vendored JS libraries and the query data all inlined into one file. +"""Bake a stage_pipeline document into a single, portable Pre/Post-ASAP HTML +page, with the same query selection, workload union, Stage 3 candidate table +and node details as index.html, but with the vendored JS libraries and the +document all inlined into one file. Why this exists alongside index.html: - No server or drag-and-drop needed: open the output file and the data is - already loaded, so it works headlessly — piped over SSH, opened in an - environment with no attached browser to drive interactively, attached to - a bug report, etc. - - index.html's own "auto-load a sibling dag.json" convenience is a - fetch(), which most browsers block as cross-origin when the page is - opened via file:// — embedding the JSON directly sidesteps that too, not - just the no-server case. + already loaded, so it works headlessly — piped over SSH, attached to a + bug report, etc. + - index.html's own example load is a fetch(), which most browsers block + as cross-origin when the page is opened via file://. -This does not add anything index.html doesn't already do — it shares -viewer.js and node-style.js with it verbatim (see viewer.js's header -comment) and only differs in packaging: one query's worth of exported -`OperatorNode` detail *is* its plan (see the side panel on node click), and -shared-hash highlighting *is* what this repo has for CSE today — both a -hash-based proxy, not real CSE output; see README.md's "Shared-sub-DAG -highlighting is a proxy" section. Structured cost/benefit annotations -(issue #286, a WorkloadDAG `CostAnnotation`) pass through -this deep-copy untouched, same as everything else `prepare_workload` below -doesn't explicitly rewrite — viewer.js reads `decision.baseline_cost` / -`.selected_cost` / `.benefit`, `NamedDAG.workload_cost`, and -`ExportDAG.edge_annotations` directly, with no help needed from this file. - -A stage_pipeline document is embedded unchanged; stage-workload.js reads -it in the page. +The document is embedded unchanged; stage-workload.js reads it in the page, +exactly as index.html does. Usage: cargo run -p asap-devtools --bin stage_pipeline -- --promql "..." --out run.json python3 tools/dag-viewer/render.py run.json -o rendered.html - - python3 tools/dag-viewer/render.py dag1.json dag2.json -o rendered.html - -Multiple input files are merged exactly like index.html's multi-file drop: a -query name colliding with an earlier one is disambiguated by suffixing the -source filename. """ from __future__ import annotations import argparse -import copy import json -import re import sys from pathlib import Path HERE = Path(__file__).resolve().parent +STAGE_PIPELINE_FORMAT = "asap-stage-pipeline/v1" -# The vendored libraries + node-style.js load as plain ", 1) - embedded = f'' + embedded = f'' viewer_tag = '' if viewer_tag not in html: raise RuntimeError(f"render.py: expected to find {viewer_tag!r} in index.html — did it move or get renamed?") - html = html.replace( + return html.replace( viewer_tag, f"{embedded}\n", 1, ) - return html def main() -> None: @@ -323,36 +86,23 @@ def main() -> None: description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument( - "files", - nargs="*", - type=Path, - help="a stage_pipeline document or WorkloadDAG JSON file(s); reads stdin if none are given", - ) + parser.add_argument("file", nargs="?", type=Path, help="a stage_pipeline document; reads stdin if omitted") parser.add_argument("-o", "--output", type=Path, required=True, help="output HTML file path") args = parser.parse_args() try: - workload = load_workload(args.files) if args.files else json.loads(sys.stdin.read()) + document = load_document(args.file) if args.file else check_document(json.loads(sys.stdin.read())) except FileNotFoundError as err: print(f"render.py: {err.filename} — no such file (run stage_pipeline first?)", file=sys.stderr) sys.exit(1) except json.JSONDecodeError as err: - source = args.files[0] if len(args.files) == 1 else "input" - print(f"render.py: {source} isn't valid JSON — {err}", file=sys.stderr) + print(f"render.py: {args.file or 'input'} isn't valid JSON — {err}", file=sys.stderr) sys.exit(1) except ValueError as err: print(f"render.py: {err}", file=sys.stderr) sys.exit(1) - if workload.get("format") == STAGE_PIPELINE_FORMAT: - workload_queries = workload.get("workload", {}).get("queries", []) - else: - workload_queries = workload.get("queries") - if not workload_queries: - print("render.py: no queries in input — nothing to render", file=sys.stderr) - sys.exit(1) - args.output.write_text(render(workload)) - n = len(workload_queries) + args.output.write_text(render(document)) + n = len(document.get("workload", {}).get("queries", [])) print(f"wrote {args.output} ({n} quer{'y' if n == 1 else 'ies'})") diff --git a/tools/dag-viewer/stage-workload.js b/tools/dag-viewer/stage-workload.js index 5665caebd..5bb86aaab 100644 --- a/tools/dag-viewer/stage-workload.js +++ b/tools/dag-viewer/stage-workload.js @@ -107,3 +107,31 @@ function stageWorkload(doc) { }), }; } + +// Stage 3's view of the candidates the document shows: the selected one, +// the other valid ones cheapest first ("costlier"), then the rejected and +// the Stage 2 pruned ones with their reasons. +function stageCandidateRows(doc) { + const selection = doc.stage3_plan_selection || {}; + const costs = selection.costs || {}; + const rejected = new Map((selection.rejected || []).map((r) => [r.id, r])); + const physical = doc.stage2_physical_asap_optimization || {}; + const rows = (physical.candidates || []).map((candidate) => { + const cost = costs[candidate.id]; + const rejection = rejected.get(candidate.id); + const status = candidate.id === selection.selected ? 'selected' : cost ? 'costlier' : 'rejected'; + return { + id: candidate.id, + display_name: candidate.display_name, + total: cost ? cost.total : null, + unit: cost ? cost.unit : null, + status, + reason: status === 'rejected' && rejection ? rejection.reason : null, + }; + }); + (physical.pruned || []).forEach((p) => rows.push({ + id: p.from_logical, display_name: p.display_name, total: null, unit: null, status: 'pruned', reason: p.reason, + })); + const order = { selected: 0, costlier: 1, rejected: 2, pruned: 3 }; + return rows.sort((a, b) => order[a.status] - order[b.status] || (a.total ?? 0) - (b.total ?? 0)); +} diff --git a/tools/dag-viewer/test_render.py b/tools/dag-viewer/test_render.py index 4e2a478ee..8216f8c8a 100644 --- a/tools/dag-viewer/test_render.py +++ b/tools/dag-viewer/test_render.py @@ -1,17 +1,9 @@ -"""Tests for render.py's data merging and HTML assembly, plus viewer cache -provenance checks executed with optional py_mini_racer (no browser required). - -Run with (from the repo root): python3 -m unittest discover -s tools/dag-viewer -p 'test_render.py' -(or `cd tools/dag-viewer && python3 -m unittest test_render`, or -`python3 -m pytest tools/dag-viewer/` if pytest is available -- these are -plain `unittest.TestCase`s, any of those runners work). Plain -`python3 -m unittest tools/dag-viewer/test_render.py` does NOT work run -from the repo root: unittest resolves that path to a bare `test_render` -module without adding tools/dag-viewer/ to sys.path, so this file's own -`from render import ...` below fails with `ModuleNotFoundError: No module -named 'render'` -- discover's `-s` (or a plain module name run from inside -the directory) puts the right directory on sys.path instead. Not wired -into CI -- this repo doesn't run Python in CI at all today. +"""Tests for the DAG viewer: render.py's standalone page, stage-workload.js +(which reads the asap-stage-pipeline/v1 document the viewer shows) and the +viewer's selection logic, the JS run headless with optional py_mini_racer. + +Run from tools/dag-viewer: `python3 test_render.py` (or +`python3 -m unittest test_render`). Not wired into CI. """ from __future__ import annotations @@ -24,9 +16,10 @@ import unittest from pathlib import Path -from render import _json_script, _semantic_label, load_workload, prepare_workload, render +from render import _json_script, load_document, render HERE = Path(__file__).resolve().parent +STAGE_DOCUMENT = HERE / "examples" / "planner-layering-example1.json" try: from py_mini_racer import py_mini_racer @@ -34,177 +27,61 @@ py_mini_racer = None -def named_dag(name: str, source: str = "SELECT 1") -> dict: - """A minimal well-formed NamedDAG: one leaf Scan node, its own root.""" - return { - "name": name, - "source": source, - "dag": { - "nodes": [ - { - "id": 0, - "kind": "Scan", - "label": "Scan(t)", - "detail": {}, - "children": [], - "hash": 1, - } - ], - "root": 0, - }, - } - - -class LoadWorkloadTests(unittest.TestCase): - def test_boundary_terms_and_provenance_survive_standalone_export(self): - """The standalone viewer retains byte totals, physical terms, and provenance.""" - dag = named_dag("boundary-example") - annotation = { - "value": 1080.0, - "unit": "CostUnits", - "source": "Modeled", - "model_version": "physical-boundary-bytes-v1+bytes-v1", - "evidence_version": "evidence-v1", - "inputs": [ - {"name": "network_bytes", "value": 480, "unit": "bytes"}, - {"name": "materialization_bytes", "value": 40, "unit": "bytes"}, - {"name": "physical_node:scan:network_bytes", "value": 480, "unit": "bytes"}, - {"name": "boundary:persist:materialization_bytes", "value": 40, "unit": "bytes"}, - ], - } - dag["dag"]["nodes"][0]["selected_cost"] = annotation - html = render({"queries": [dag]}) - for term in annotation["inputs"]: - self.assertIn(term["name"], html) - self.assertIn(annotation["model_version"], html) - self.assertIn(annotation["evidence_version"], html) - - def test_merges_queries_across_files_in_order(self): - with tempfile.TemporaryDirectory() as d: - f1 = Path(d) / "a.json" - f2 = Path(d) / "b.json" - f1.write_text(json.dumps({"queries": [named_dag("q1")]})) - f2.write_text(json.dumps({"queries": [named_dag("q2")]})) +def document() -> dict: + return json.loads(STAGE_DOCUMENT.read_text()) - workload = load_workload([f1, f2]) - self.assertEqual([q["name"] for q in workload["queries"]], ["q1", "q2"]) +def embedded(html: str) -> dict: + match = re.search(r'', html, re.S) + return json.loads(match.group(1)) - def test_colliding_name_is_disambiguated_by_source_filename(self): - # Matches viewer.js's loadFiles(): a later file's query named the - # same as an earlier one gets " (filename)" appended instead of - # silently overwriting or erroring. - with tempfile.TemporaryDirectory() as d: - f1 = Path(d) / "a.json" - f2 = Path(d) / "b.json" - f1.write_text(json.dumps({"queries": [named_dag("q1")]})) - f2.write_text(json.dumps({"queries": [named_dag("q1")]})) - workload = load_workload([f1, f2]) +class LoadDocumentTests(unittest.TestCase): + def test_reads_a_stage_document(self): + self.assertEqual(load_document(STAGE_DOCUMENT), document()) - self.assertEqual(workload["queries"][0]["name"], "q1") - self.assertEqual(workload["queries"][1]["name"], "q1 (b.json)") - - def test_non_colliding_names_pass_through_unchanged(self): + def test_rejects_any_other_json(self): with tempfile.TemporaryDirectory() as d: - f1 = Path(d) / "a.json" - f1.write_text(json.dumps({"queries": [named_dag("q1"), named_dag("q2")]})) - - workload = load_workload([f1]) - - self.assertEqual([q["name"] for q in workload["queries"]], ["q1", "q2"]) + old = Path(d) / "old.json" + old.write_text(json.dumps({"queries": []})) + with self.assertRaisesRegex(ValueError, "asap-stage-pipeline/v1"): + load_document(old) class RenderTests(unittest.TestCase): def test_no_script_src_tags_remain(self): - # Substring-only "') - - def test_embedded_workload_round_trips(self): - workload = {"queries": [named_dag("q1"), named_dag("q2")]} - html = render(workload) - - m = re.search( - r'', - html, - re.S, - ) - self.assertIsNotNone(m, "embedded-workload ') + + def test_embedded_document_round_trips_with_its_reader(self): + html = render(document()) + self.assertEqual(embedded(html), document()) + self.assertIn("function stageWorkload", html) def test_standalone_export_carries_the_bulk_selection_control(self): - """A generated page gets Select-all for free: render.py inlines the - markup and viewer.js verbatim, so neither fix needs its own step.""" - html = render({"queries": [named_dag("q1"), named_dag("q2")]}) + html = render(document()) self.assertIn('id="selectAllToggle"', html) self.assertIn("function bulkSelectionState(", html) - # And it must stay a selection control, not a second Clear all: the - # embedded workload is parsed once at load, so dropping `queries` - # from a generated page is unrecoverable without a reload. - self.assertNotRegex( - html, r"selectAllToggle\.addEventListener[^}]*queries = \[\]" - ) + self.assertNotRegex(html, r"selectAllToggle\.addEventListener[^}]*queries = \[\]") def test_standalone_export_lanes_are_pannable(self): - """Dragging the lane background pans the viewport in the generated - page too -- `grabbable: false` alone made it a dead zone.""" - html = render({"queries": [named_dag("q1")]}) - lanes = re.findall(r"classes: 'laneParent'[^}]*}", html) + lanes = re.findall(r"classes: 'laneParent'[^}]*}", render(document())) self.assertEqual(len(lanes), 2, "expected the union and single-query lanes") for lane in lanes: self.assertIn("pannable: true", lane) - def test_render_does_not_mutate_callers_workload(self): - workload = {"queries": [named_dag("q1")]} - original = json.loads(json.dumps(workload)) - render(workload) - self.assertEqual(workload, original) - - def test_render_adds_no_legacy_mode_config(self): - # Bare "__DAG_RENDER__" also matches viewer.js's own header comment - # (which documents the window.__DAG_RENDER__ config object in prose) - # once that file is inlined verbatim -- match the actual assignment - # statement instead. - workload = {"queries": [named_dag("q1"), named_dag("q2")]} - html = render(workload) - self.assertNotRegex(html, r"window\.__DAG_RENDER__ =") + def test_render_does_not_mutate_callers_document(self): + doc = document() + render(doc) + self.assertEqual(doc, document()) def test_embedded_data_placed_before_viewer_js_body(self): - # viewer.js's top-level startup code reads #embedded-workload and - # synchronously as soon as it runs, so it must appear earlier in the - # document than viewer.js's own inlined - # " substring must not prematurely close the embedded - # '")]} - html = render(workload) - - m = re.search( - r'', - html, - re.S, - ) - self.assertIsNotNone(m) - parsed = json.loads(m.group(1)) - self.assertEqual(parsed["queries"][0]["source"], "SELECT ''") + html = render(document()) + self.assertLess(html.index('id="embedded-workload"'), html.index("cytoscape.use(window.cytoscapeDagre)")) + + def test_angle_bracket_in_query_text_does_not_break_out_of_script_tag(self): + doc = document() + doc["workload"]["queries"][0]["text"] = "m{a=''}" + self.assertEqual(embedded(render(doc)), doc) class JsonScriptTests(unittest.TestCase): @@ -214,109 +91,9 @@ def test_escapes_angle_bracket_but_stays_valid_json(self): self.assertEqual(json.loads(encoded), {"x": ""}) -class SemanticLabelTests(unittest.TestCase): - def test_scan_recovers_source_from_legacy_label(self): - node = {"kind": "Scan", "label": "Scan(metrics)", "detail": {}} - self.assertEqual(_semantic_label(node), "Scan\nsource: metrics") - - def test_aggregate_names_measure_input_and_grouping(self): - node = { - "kind": "Aggregate", - "detail": { - "measures": [{"kind": "quantile", "col": 6, "q": 0.95}], - "reduction": {"Reduce": [1]}, - }, - } - self.assertEqual( - _semantic_label(node), - "Aggregate\nmeasure: quantile(col[6], q=0.95)\ngroup by col[1]", - ) - - def test_summary_aggregate_names_reduction_as_group_by(self): - node = { - "kind": "SummaryAgg", - "detail": { - "family": "Sketch(Cms)", - "col": "SampleValue", - "reduction": {"Reduce": [1]}, - "grouping": "PerSubpopulationInstance", - }, - } - self.assertEqual( - _semantic_label(node), - "SummaryAgg\nfamily: Sketch(Cms)\ninput: SampleValue\n… +2 more", - ) - - def test_sort_names_expression_direction_and_null_order(self): - node = { - "kind": "Sort", - "detail": { - "keys": [{"expr": {"Column": 2}, "ascending": False, "nulls_first": True}] - }, - } - self.assertEqual(_semantic_label(node), "Sort\nsort: col[2] descending, nulls first") - - def test_project_names_columns_from_fields_and_legacy_columns(self): - """Exports write `fields`; older ones wrote `columns`. Both name columns.""" - for key, dtype in (("fields", {"Plain": "utf8"}), ("columns", "utf8")): - node = { - "kind": "Project", - "detail": {"cols": [0]}, - "schema": {key: [{"name": "service", "dtype": dtype, "nullable": False}]}, - } - self.assertEqual(_semantic_label(node), "Project\ncolumns: service", key) - - def test_prepares_before_after_and_whole_post_asap_dags(self): - node = { - "id": 0, - "kind": "Aggregate", - "label": "Aggregate(1 measures)", - "detail": {"measures": [{"kind": "avg", "col": 3}]}, - "children": [], - } - def dag(): - return {"nodes": [dict(node)], "root": 0} - workload = { - "queries": [ - { - "dag": dag(), - "post_dag": dag(), - "replacements": [{"before": dag(), "after": {"dag": dag()}}], - } - ] - } - prepared = prepare_workload(workload) - query = prepared["queries"][0] - labels = [ - query["dag"]["nodes"][0]["label"], - query["post_dag"]["nodes"][0]["label"], - ] - self.assertEqual(labels, ["Aggregate\nmeasure: avg(col[3])"] * 2) - self.assertEqual( - query["replacements"][0]["before"]["nodes"][0]["label"], - "Aggregate(1 measures)", - ) - - def test_replacement_sub_dags_are_not_prepared_for_the_current_viewer(self): - def dag(): - return {"nodes": [{"id": 0, "kind": "Scan", "label": "legacy", "detail": {}, "children": []}], "root": 0} - workload = {"queries": [{"dag": dag(), "replacements": [{"before": dag(), "after": {"dag": dag()}}]}]} - replacement = prepare_workload(workload)["queries"][0]["replacements"][0] - self.assertEqual(replacement["before"]["nodes"][0]["label"], "legacy") - self.assertEqual(replacement["after"]["dag"]["nodes"][0]["label"], "legacy") - - class MainCliTests(unittest.TestCase): - """Exercises main()'s own error handling via subprocess, rather than - calling main() in-process, since what's under test here is exactly the - CLI's user-facing stderr message + exit code, not internal state.""" - def run_render_py(self, *args: str) -> subprocess.CompletedProcess: - return subprocess.run( - [sys.executable, str(HERE / "render.py"), *args], - capture_output=True, - text=True, - ) + return subprocess.run([sys.executable, str(HERE / "render.py"), *args], capture_output=True, text=True) def test_missing_input_file_gives_a_clean_error_not_a_traceback(self): result = self.run_render_py("/no/such/path/dag.json", "-o", "/dev/null") @@ -333,25 +110,14 @@ def test_malformed_json_gives_a_clean_error_not_a_traceback(self): self.assertIn("isn't valid JSON", result.stderr) self.assertNotIn("Traceback", result.stderr) - -STAGE_DOCUMENT = HERE / "examples" / "planner-layering-example1.json" - - -class StageDocumentRenderTests(unittest.TestCase): - """render.py passes a stage_pipeline document through for the viewer's - stage-workload.js to read.""" - - def test_stage_document_is_embedded_unchanged_with_its_reader(self): - document = json.loads(STAGE_DOCUMENT.read_text()) - self.assertEqual(load_workload([STAGE_DOCUMENT]), document) - html = render(document) - embedded = re.search(r'', html, re.S) - self.assertEqual(json.loads(embedded.group(1)), document) - self.assertIn("function stageWorkload", html) - - def test_two_stage_documents_are_rejected(self): - with self.assertRaises(ValueError): - load_workload([STAGE_DOCUMENT, STAGE_DOCUMENT]) + def test_other_json_gives_a_clean_error(self): + with tempfile.TemporaryDirectory() as d: + old = Path(d) / "old.json" + old.write_text(json.dumps({"queries": []})) + result = self.run_render_py(str(old), "-o", "/dev/null") + self.assertEqual(result.returncode, 1) + self.assertIn("asap-stage-pipeline/v1", result.stderr) + self.assertNotIn("Traceback", result.stderr) @unittest.skipIf(py_mini_racer is None, "viewer tests require py_mini_racer") @@ -363,7 +129,7 @@ class StageWorkloadTests(unittest.TestCase): def setUp(self): self.js = py_mini_racer.MiniRacer() self.js.eval((HERE / "stage-workload.js").read_text()) - self.document = json.loads(STAGE_DOCUMENT.read_text()) + self.document = document() def test_one_query_per_workload_query_with_reachable_dags(self): workload = self.js.call("stageWorkload", self.document) @@ -389,6 +155,25 @@ def test_post_nodes_carry_the_selected_plans_costs(self): self.assertIn("\ncost ", node["label"]) self.assertEqual(node["detail"]["cost"], cost) + def test_candidates_rank_the_selected_plan_first_then_by_cost(self): + """The Stage 3 table: the selected candidate, the other costed ones + cheapest first, then the rejected ones with their reasons.""" + selection = self.document["stage3_plan_selection"] + rows = self.js.call("stageCandidateRows", self.document) + self.assertEqual(rows[0]["id"], selection["selected"]) + self.assertEqual(rows[0]["status"], "selected") + self.assertEqual(rows[0]["total"], selection["costs"][selection["selected"]]["total"]) + totals = [row["total"] for row in rows[1:] if row["status"] == "costlier"] + self.assertEqual(totals, sorted(totals)) + self.assertTrue(all(row["total"] >= rows[0]["total"] for row in rows[1:] if row["status"] == "costlier")) + rejected = {r["id"]: r["reason"] for r in selection["rejected"] if not r["valid"]} + for row in rows: + if row["status"] == "rejected": + self.assertEqual(row["reason"], rejected[row["id"]]) + physical = self.document["stage2_physical_asap_optimization"] + self.assertEqual(len(rows), len(physical["candidates"]) + len(physical["pruned"])) + self.assertTrue(all(row["display_name"] for row in rows)) + def test_failed_selection_has_no_post_dag(self): self.document["stage3_plan_selection"] = {"costs": {}, "selected": None, "rejected": []} queries = self.js.call("stageWorkload", self.document)["queries"] @@ -396,31 +181,15 @@ def test_failed_selection_has_no_post_dag(self): @unittest.skipIf(py_mini_racer is None, "viewer tests require py_mini_racer") -class ViewerCacheTests(unittest.TestCase): +class ViewerSelectionTests(unittest.TestCase): def setUp(self): self.js = py_mini_racer.MiniRacer() source = (HERE / "viewer.js").read_text() - for name in ["escapeHtml", "formatCostUnit", "formatBaselineRef", - "formatCostNumber", "renderCostAnnotation", - "computeSelectionWorkloadCost", "bulkSelectionState"]: + for name in ["escapeHtml", "bulkSelectionState"]: function = re.search(r"^function " + name + r"\(.*?^}", source, re.M | re.S) self.assertIsNotNone(function, name) self.js.eval(function.group(0)) - @staticmethod - def query(profile, selected_profile=None, batch=0): - def annotation(value, cache): - result = {"value": value, "unit": "CostUnits", "source": "Modeled"} - if cache is not None: - result["cache_profile"] = cache - return result - - return {"sourceBatch": batch, "post_dag": {"nodes": [{"decision": { - "id": 1, - "baseline_cost": annotation(10, profile), - "selected_cost": annotation(4, selected_profile or profile), - }}]}} - def test_bulk_selection_distinguishes_none_some_and_all(self): """The checkbox has three states to show, and `indeterminate` is the one a plain checkbox cannot: with fifty queries, "some" is normal.""" @@ -470,43 +239,6 @@ def test_empty_render_resets_bulk_selection(self): self.assertFalse(self.js.eval("selectAllToggle.checked")) self.assertFalse(self.js.eval("selectAllToggle.indeterminate")) - def test_cache_profile_and_inputs_are_rendered(self): - """The sidebar exposes the assumptions behind a cache-adjusted cost.""" - annotation = self.query("warm-cache-v1")["post_dag"]["nodes"][0]["decision"]["baseline_cost"] - annotation["inputs"] = [{"name": "result_cache_hit_ratio", "value": 0.5, "unit": "ratio"}] - html = self.js.call("renderCostAnnotation", "Baseline", annotation) - self.assertIn("cache warm-cache-v1", html) - self.assertIn("result_cache_hit_ratio", html) - self.assertIn("0.5 ratio", html) - - def test_same_cache_profile_aggregates_and_preserves_provenance(self): - """Comparable decisions total normally and retain their common profile.""" - result = self.js.call("computeSelectionWorkloadCost", [self.query("warm-v1"), self.query("warm-v1", batch=1)]) - self.assertEqual(result["baseline_cost"]["value"], 20) - self.assertEqual(result["benefit"]["value"], 12) - self.assertEqual(result["benefit"]["cache_profile"], "warm-v1") - - def test_legacy_profiles_preserve_existing_totals(self): - """Models without cache provenance retain their existing aggregation.""" - result = self.js.call("computeSelectionWorkloadCost", [self.query(None), self.query(None, batch=1)]) - self.assertEqual(result["benefit"]["value"], 12) - self.assertIsNone(result["benefit"]["cache_profile"]) - - def test_different_or_mixed_cache_profiles_make_totals_unavailable(self): - """Selection totals cannot claim benefits across incompatible assumptions.""" - cases = [ - [self.query("cold-v1"), self.query("warm-v1", batch=1)], - [self.query("cold-v1", selected_profile="warm-v1")], - [self.query("warm-v1"), self.query(None, batch=1)], - [self.query(None), self.query("warm-v1", batch=1)], - ] - for queries in cases: - with self.subTest(queries=queries): - result = self.js.call("computeSelectionWorkloadCost", queries) - for annotation in result.values(): - self.assertIsNone(annotation["value"]) - self.assertEqual(annotation["source"], "Unavailable") - if __name__ == "__main__": unittest.main() diff --git a/tools/dag-viewer/viewer.js b/tools/dag-viewer/viewer.js index 8a5e71999..2aec17c6e 100644 --- a/tools/dag-viewer/viewer.js +++ b/tools/dag-viewer/viewer.js @@ -1,31 +1,16 @@ -// Shared viewer logic for tools/dag-viewer, used two ways: -// 1. index.html loads this via