fix(asap-tools): refuse misaligned or plain-string sql_file

A mode whose SQL file holds a different number of statements than
baseline's for the same group would pair later queries with the wrong
baseline queries; a plain-string sql_file failed with an AttributeError.
Both now fail with a clear error, and the docs' template uses the dict
form.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: milindsrivastava1997

asap-tools/experiments/HYDRA_CONFIG_USAGE.md

@@ -174,7 +174,8 @@ experiment_params:
     max_rows: 0            # 0 = load all rows
   query_groups:
     - name: default_queries
-      sql_file: ???        # path to SQL file generated by benchmark/generate_queries.py
+      sql_file:            # mode -> SQL file generated by benchmark/generate_queries.py
+        baseline: ???      # a sketchdb mode needs its own entry (may be the same file)
       repetition_delay: 0
       client_options:
         starting_delay: 0
@@ -202,6 +203,9 @@ python experiment_run_clickhouse.py \
   'experiment_params.query_groups[0].sql_file={baseline:/path/to/queries.sql}'
 ```
 
+With a `sketchdb` mode in `experiment_params.experiment`, give it its own
+entry, e.g. `sql_file={baseline:/path/to/queries.sql,sketchdb:/path/to/queries.sql}`.
+
 Optional overrides:
 ```bash
 # Skip data loading step, keep ClickHouse running after experiment

asap-tools/experiments/experiment_utils/config.py

@@ -95,14 +95,20 @@ def _mode_sql_files(query_groups: Any, mode: str) -> List[Tuple[int, Any, str]]:
     sketchdb). Every mode but sketchdb needs an entry in every group.
     sketchdb skips the groups without one (e.g. queries ASAPQuery has no SQL
     for), but only trailing groups: results are matched across modes by
-    global query index, so sketchdb's queries must be a prefix of baseline's.
+    global query index, so sketchdb's queries must be a prefix of baseline's,
+    and each file it runs must hold as many statements as baseline's.
     """
     runs = []
     for idx, group in enumerate(query_groups):
         name = group.get("name", str(idx))
         sql_file = group.get("sql_file")
         if not sql_file:
             raise ValueError(f"Query group {idx} ({name!r}) missing 'sql_file'")
+        if not hasattr(sql_file, "get"):
+            raise ValueError(
+                f"Query group {idx} ({name!r}) sql_file must map mode name to "
+                f"SQL file, e.g. {{baseline: {sql_file}}}; got {sql_file!r}"
+            )
         path = sql_file.get(mode)
         if path:
             if len(runs) != idx:
@@ -111,6 +117,20 @@ def _mode_sql_files(query_groups: Any, mode: str) -> List[Tuple[int, Any, str]]:
                     f"{mode!r} after a group without one; a mode may skip "
                     "only trailing groups, so its query indices match baseline's"
                 )
+            baseline_path = sql_file.get(constants.BASELINE_EXPERIMENT_NAME)
+            if (
+                mode != constants.BASELINE_EXPERIMENT_NAME
+                and baseline_path
+                and os.path.exists(path)
+                and os.path.exists(baseline_path)
+                and len(_load_sql_queries(path))
+                != len(_load_sql_queries(baseline_path))
+            ):
+                raise ValueError(
+                    f"Query group {idx} ({name!r}): mode {mode!r}'s {path!r} and "
+                    f"baseline's {baseline_path!r} hold different numbers of "
+                    "statements, so their query indices would not match"
+                )
             runs.append((idx, group, path))
         elif mode != constants.SKETCHDB_EXPERIMENT_NAME:
             raise ValueError(

asap-tools/experiments/tests/test_clickhouse_client_configs.py

@@ -68,6 +68,21 @@ def test_sketchdb_without_any_sql_is_refused(self):
         with self.assertRaisesRegex(ValueError, "No query group .* 'sketchdb'"):
             self.client_queries(groups)
 
+    def test_statement_count_mismatch_is_refused(self):
+        # sketchdb's later queries would be scored against the wrong
+        # baseline queries.
+        two = os.path.join(self.tmp.name, "two.sql")
+        with open(two, "w") as f:
+            f.write("SELECT 1;\nSELECT 2;\n")
+        groups = [{"sql_file": {"baseline": two, "sketchdb": self.sql("s")}}]
+        with self.assertRaisesRegex(ValueError, "different numbers of statements"):
+            self.client_queries(groups)
+
+    def test_plain_string_sql_file_is_refused_clearly(self):
+        groups = [{"sql_file": self.sql("b")}]
+        with self.assertRaisesRegex(ValueError, "must map mode name"):
+            self.client_queries(groups, {"baseline": URLS["baseline"]})
+
     def test_other_modes_need_every_group(self):
         groups = [
             {"sql_file": {"baseline": self.sql("b0")}},