Skip to content

Commit ecfa89c

Browse files
milindsrivastava1997claude
milindsrivastava1997
and
claude
authored
test(query-engine): add SQL query support runner (#823)
* test(query-engine): add SQL compliance runner Plans each SQL suite query in isolation and builds the engine's execution context against that plan, without ingesting data. Each case declares whether it is expected to be supported, so a change in SQL coverage fails the run until the suite is updated. Suites mirror the promql-compliance suites case-for-case, plus a basic suite covering single-layer shapes, time-range forms, and top-k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(query-engine): rename SQL compliance runner to sql_query_support The runner checks plan-path acceptance, not result compliance. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0fd93d4 commit ecfa89c

9 files changed

Lines changed: 1002 additions & 0 deletions

File tree

‎asap-query-engine/tests/sql_query_support.rs‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
//! SQL query support runner: plans each suite query in isolation and builds the
2+
//! engine's execution context for it, without ingesting data or executing.
3+
//!
4+
//! A query is `supported` when both the planner accepts it and the engine
5+
//! resolves an execution context against that plan. Each suite case declares
6+
//! the expected outcome, so a query that starts or stops being supported
7+
//! fails the run until its suite entry is updated.
8+
9+
use asap_planner::{SQLController, SQLRuntimeOptions, StreamingEngine};
10+
use asap_types::enums::{CleanupPolicy, QueryLanguage};
11+
use query_engine_rust::{SimpleEngine, SimpleMapStore};
12+
use serde::Deserialize;
13+
use serde_json::json;
14+
use std::path::{Path, PathBuf};
15+
use std::sync::Arc;
16+
17+
const SUITES_DIR: &str = "tests/sql_query_support";
18+
const SUITE_EXTENSION: &str = "yaml";
19+
/// Fixed evaluation time (seconds) so NOW()-relative queries plan and match
20+
/// deterministically.
21+
const QUERY_EVALUATION_TIME_SECS: f64 = 1_000_000.0;
22+
const ACCURACY_SLA: f64 = 0.95;
23+
24+
#[derive(Debug, Deserialize)]
25+
#[serde(deny_unknown_fields)]
26+
struct Suite {
27+
/// Planner table definitions, passed through verbatim.
28+
tables: serde_json::Value,
29+
data_ingestion_interval_ms: u64,
30+
repetition_delay_ms: u64,
31+
cases: Vec<Case>,
32+
}
33+
34+
#[derive(Debug, Deserialize)]
35+
#[serde(deny_unknown_fields)]
36+
struct Case {
37+
name: String,
38+
query: String,
39+
expected: Outcome,
40+
}
41+
42+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
43+
#[serde(rename_all = "snake_case")]
44+
enum Outcome {
45+
Supported,
46+
Unsupported,
47+
}
48+
49+
/// Where a query stopped, with the reason. `None` means it is supported.
50+
type Failure = Option<String>;
51+
52+
fn run_case(suite: &Suite, query: &str) -> Failure {
53+
let planner_config = json!({
54+
"tables": suite.tables,
55+
"query_groups": [{
56+
"queries": [query],
57+
"repetition_delay_ms": suite.repetition_delay_ms,
58+
"controller_options": { "accuracy_sla": ACCURACY_SLA },
59+
}],
60+
});
61+
let planner_yaml = serde_yaml::to_string(&planner_config).expect("planner config serializes");
62+
let opts = SQLRuntimeOptions {
63+
streaming_engine: StreamingEngine::Precompute,
64+
query_evaluation_time: Some(QUERY_EVALUATION_TIME_SECS),
65+
data_ingestion_interval_ms: suite.data_ingestion_interval_ms,
66+
};
67+
let output = match SQLController::from_yaml(&planner_yaml, opts).and_then(|c| c.generate()) {
68+
Ok(output) => output,
69+
Err(error) => return Some(format!("plan: {error}")),
70+
};
71+
72+
let inference_config = output
73+
.to_inference_config(QueryLanguage::sql)
74+
.expect("planner output converts to inference config");
75+
let streaming_config = Arc::new(
76+
output
77+
.to_streaming_config(QueryLanguage::sql)
78+
.expect("planner output converts to streaming config"),
79+
);
80+
let store = Arc::new(SimpleMapStore::new(
81+
streaming_config.clone(),
82+
CleanupPolicy::NoCleanup,
83+
));
84+
let engine = SimpleEngine::new(
85+
store,
86+
inference_config,
87+
streaming_config,
88+
suite.data_ingestion_interval_ms,
89+
QueryLanguage::sql,
90+
);
91+
match engine.build_query_execution_context_sql(query.to_string(), QUERY_EVALUATION_TIME_SECS) {
92+
Some(_) => None,
93+
None => Some("engine: no execution context for planned query".to_string()),
94+
}
95+
}
96+
97+
fn suite_paths() -> Vec<PathBuf> {
98+
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(SUITES_DIR);
99+
let mut paths: Vec<PathBuf> = std::fs::read_dir(&dir)
100+
.unwrap_or_else(|e| panic!("read {}: {e}", dir.display()))
101+
.map(|entry| entry.expect("suite dir entry").path())
102+
.filter(|path| path.extension().is_some_and(|ext| ext == SUITE_EXTENSION))
103+
.collect();
104+
paths.sort();
105+
assert!(!paths.is_empty(), "no suites in {}", dir.display());
106+
paths
107+
}
108+
109+
#[test]
110+
fn sql_queries_match_expected_support() {
111+
let mut mismatches = Vec::new();
112+
for path in suite_paths() {
113+
let suite_name = path.file_stem().unwrap().to_string_lossy().into_owned();
114+
let contents = std::fs::read_to_string(&path).expect("read suite");
115+
let suite: Suite = serde_yaml::from_str(&contents)
116+
.unwrap_or_else(|e| panic!("parse {}: {e}", path.display()));
117+
118+
for case in &suite.cases {
119+
let failure = run_case(&suite, &case.query);
120+
let actual = match failure {
121+
None => Outcome::Supported,
122+
Some(_) => Outcome::Unsupported,
123+
};
124+
let status = if actual == case.expected {
125+
"ok"
126+
} else {
127+
"MISMATCH"
128+
};
129+
let reason = failure.as_deref().unwrap_or("");
130+
println!("[{status}] {suite_name}/{}: {actual:?} {reason}", case.name);
131+
if actual != case.expected {
132+
mismatches.push(format!(
133+
"{suite_name}/{}: expected {:?}, got {actual:?} {reason}",
134+
case.name, case.expected
135+
));
136+
}
137+
}
138+
}
139+
assert!(
140+
mismatches.is_empty(),
141+
"SQL query support mismatches:\n{}",
142+
mismatches.join("\n")
143+
);
144+
}

‎asap-query-engine/tests/sql_query_support/aggregations.yaml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
# SQL equivalents of promql-compliance/suites/aggregations.yaml; case names match.
2+
#
3+
# Translation: `[5m]` range functions aggregate over a 300s window; instant
4+
# vectors aggregate over one ingestion interval (1s). A per-series result
5+
# groups by every label (job, instance). Per-group top-k uses `LIMIT k BY`.
6+
#
7+
# Skipped, no SQL equivalent for counter rate/increase: increase, rate,
8+
# topk-3-rate, topk-3-rate-by-job, topk-3-rate-by-job-instance,
9+
# sum-by-job-rate, sum-by-job-instance-rate.
10+
tables:
11+
- name: data
12+
time_column: time
13+
value_columns: [value]
14+
metadata_columns: [job, instance]
15+
data_ingestion_interval_ms: 1000
16+
repetition_delay_ms: 60000
17+
18+
cases:
19+
- name: sum-over-time
20+
query: SELECT job, instance, SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
21+
expected: supported
22+
- name: count-over-time
23+
query: SELECT job, instance, COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
24+
expected: supported
25+
- name: avg-over-time
26+
query: SELECT job, instance, AVG(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
27+
expected: unsupported
28+
- name: min-over-time
29+
query: SELECT job, instance, MIN(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
30+
expected: supported
31+
- name: max-over-time
32+
query: SELECT job, instance, MAX(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
33+
expected: supported
34+
35+
- name: topk-1-bare
36+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 1
37+
expected: supported
38+
- name: topk-3-bare
39+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 3
40+
expected: supported
41+
- name: topk-5-bare
42+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 5
43+
expected: supported
44+
- name: topk-10-bare
45+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 10
46+
expected: supported
47+
- name: topk-3-bare-by-job
48+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 3 BY job
49+
expected: unsupported
50+
- name: topk-3-bare-by-job-instance
51+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 3 BY job, instance
52+
expected: unsupported
53+
54+
- name: topk-1-sum-over-time
55+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 1
56+
expected: supported
57+
- name: topk-3-sum-over-time
58+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 3
59+
expected: supported
60+
- name: topk-5-sum-over-time
61+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 5
62+
expected: supported
63+
- name: topk-10-sum-over-time
64+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 10
65+
expected: supported
66+
- name: topk-1-sum-over-time-by-job
67+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 1 BY job
68+
expected: unsupported
69+
- name: topk-3-sum-over-time-by-job
70+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 3 BY job
71+
expected: unsupported
72+
- name: topk-5-sum-over-time-by-job
73+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 5 BY job
74+
expected: unsupported
75+
- name: topk-10-sum-over-time-by-job
76+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 10 BY job
77+
expected: unsupported
78+
- name: topk-1-sum-over-time-by-job-instance
79+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 1 BY job, instance
80+
expected: unsupported
81+
- name: topk-3-sum-over-time-by-job-instance
82+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 3 BY job, instance
83+
expected: unsupported
84+
- name: topk-5-sum-over-time-by-job-instance
85+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 5 BY job, instance
86+
expected: unsupported
87+
- name: topk-10-sum-over-time-by-job-instance
88+
query: SELECT job, instance, SUM(value) AS v FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY v DESC LIMIT 10 BY job, instance
89+
expected: unsupported
90+
91+
- name: topk-1-count-over-time
92+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 1
93+
expected: supported
94+
- name: topk-3-count-over-time
95+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 3
96+
expected: supported
97+
- name: topk-5-count-over-time
98+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 5
99+
expected: supported
100+
- name: topk-10-count-over-time
101+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 10
102+
expected: supported
103+
- name: topk-1-count-over-time-by-job
104+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 1 BY job
105+
expected: unsupported
106+
- name: topk-3-count-over-time-by-job
107+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 3 BY job
108+
expected: unsupported
109+
- name: topk-5-count-over-time-by-job
110+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 5 BY job
111+
expected: unsupported
112+
- name: topk-10-count-over-time-by-job
113+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 10 BY job
114+
expected: unsupported
115+
- name: topk-1-count-over-time-by-job-instance
116+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 1 BY job, instance
117+
expected: unsupported
118+
- name: topk-3-count-over-time-by-job-instance
119+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 3 BY job, instance
120+
expected: unsupported
121+
- name: topk-5-count-over-time-by-job-instance
122+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 5 BY job, instance
123+
expected: unsupported
124+
- name: topk-10-count-over-time-by-job-instance
125+
query: SELECT job, instance, COUNT(value) AS c FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance ORDER BY c DESC LIMIT 10 BY job, instance
126+
expected: unsupported
127+
128+
- name: sum
129+
query: SELECT SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW()
130+
expected: unsupported
131+
- name: sum-by-job
132+
query: SELECT job, SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job
133+
expected: supported
134+
- name: sum-by-job-instance
135+
query: SELECT job, instance, SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance
136+
expected: supported
137+
- name: count
138+
query: SELECT COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW()
139+
expected: unsupported
140+
- name: count-by-job
141+
query: SELECT job, COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job
142+
expected: supported
143+
- name: count-by-job-instance
144+
query: SELECT job, instance, COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -1, NOW()) AND NOW() GROUP BY job, instance
145+
expected: supported
146+
147+
- name: sum-by-job-sum-over-time
148+
query: SELECT job, SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job
149+
expected: supported
150+
- name: sum-by-job-instance-sum-over-time
151+
query: SELECT job, instance, SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
152+
expected: supported
153+
- name: sum-sum-over-time
154+
query: SELECT SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW()
155+
expected: unsupported
156+
- name: sum-without-job-sum-over-time
157+
query: SELECT instance, SUM(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY instance
158+
expected: supported
159+
- name: sum-count-over-time
160+
query: SELECT COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW()
161+
expected: unsupported
162+
- name: sum-by-job-count-over-time
163+
query: SELECT job, COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job
164+
expected: supported
165+
- name: sum-by-job-instance-count-over-time
166+
query: SELECT job, instance, COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
167+
expected: supported
168+
- name: sum-without-job-count-over-time
169+
query: SELECT instance, COUNT(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY instance
170+
expected: supported
171+
- name: min-min-over-time
172+
query: SELECT MIN(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW()
173+
expected: unsupported
174+
- name: min-by-job-min-over-time
175+
query: SELECT job, MIN(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job
176+
expected: supported
177+
- name: min-by-job-instance-min-over-time
178+
query: SELECT job, instance, MIN(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
179+
expected: supported
180+
- name: min-without-job-min-over-time
181+
query: SELECT instance, MIN(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY instance
182+
expected: supported
183+
- name: max-max-over-time
184+
query: SELECT MAX(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW()
185+
expected: unsupported
186+
- name: max-by-job-max-over-time
187+
query: SELECT job, MAX(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job
188+
expected: supported
189+
- name: max-by-job-instance-max-over-time
190+
query: SELECT job, instance, MAX(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY job, instance
191+
expected: supported
192+
- name: max-without-job-max-over-time
193+
query: SELECT instance, MAX(value) FROM data WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY instance
194+
expected: supported

0 commit comments

Comments
 (0)