fix(asap-tools): give sketchdb its own SQL in ClickHouse runs

asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md

@@ -527,6 +527,10 @@ table with `sketch_bench_e2e/sketch_bench_e2e_configs.py`.
 - **Validation**: Must be valid PromQL syntax
 - **SCHEMA ISSUE**: Inconsistent time ranges ([1m] vs [10m]) across configs
 
+#### `experiment_params.query_groups[].sql_file` (dict, required; `experiment_run_clickhouse.py`)
+- **Description**: Mode name -> SQL file (`;`-separated statements) the mode runs for this group, in place of `queries`. Every mode but `sketchdb` needs an entry in every group. `sketchdb` runs its own entry (the SQL ASAPQuery receives, and the planner plans) and skips groups without one, but only trailing groups: results pair across modes by global query index, so `sketchdb`'s queries must be the first of baseline's
+- **Example**: `{baseline: /path/to/q_baseline.sql, sketchdb: /path/to/q_asap.sql}`
+
 #### `experiment_params.query_groups[].repetition_delay` (int, required)
 - **Description**: Delay in seconds between query repetitions
 - **Range**: 1-3600 seconds

asap-tools/experiments/HYDRA_CONFIG_USAGE.md

@@ -160,7 +160,7 @@ cat > config/experiment_type/clickhouse.yaml << 'EOF'
 #     providers.cloudlab.hostname_suffix=myexp.cloudlab.us \
 #     experiment_params.dataset.name=clickbench \
 #     experiment_params.dataset.local_data_file=/path/to/hits.json \
-#     'experiment_params.query_groups[0].sql_file=/path/to/queries.sql'
+#     'experiment_params.query_groups[0].sql_file={baseline:/path/to/queries.sql}'
 
 experiment_params:
   monitoring:
@@ -174,7 +174,8 @@ experiment_params:
     max_rows: 0            # 0 = load all rows
   query_groups:
     - name: default_queries
-      sql_file: ???        # path to SQL file generated by benchmark/generate_queries.py
+      sql_file:            # mode -> SQL file generated by benchmark/generate_queries.py
+        baseline: ???      # a sketchdb mode needs its own entry (may be the same file)
       repetition_delay: 0
       client_options:
         starting_delay: 0
@@ -199,9 +200,12 @@ python experiment_run_clickhouse.py \
   controller.punting=false \
   experiment_params.dataset.name=clickbench \
   experiment_params.dataset.local_data_file=/path/to/hits.json \
-  'experiment_params.query_groups[0].sql_file=/path/to/queries.sql'
+  'experiment_params.query_groups[0].sql_file={baseline:/path/to/queries.sql}'
 ```
 
+With a `sketchdb` mode in `experiment_params.experiment`, give it its own
+entry, e.g. `sql_file={baseline:/path/to/queries.sql,sketchdb:/path/to/queries.sql}`.
+
 Optional overrides:
 ```bash
 # Skip data loading step, keep ClickHouse running after experiment
@@ -239,7 +243,7 @@ python experiment_run_clickhouse.py \
   experiment_type=clickhouse \
   experiment.name=frozen_plan_eval \
   controller.prebuilt_output_dir=/path/to/frozen-sql-plan \
-  'experiment_params.query_groups[0].sql_file=/path/to/queries.sql'
+  'experiment_params.query_groups[0].sql_file={baseline:/path/to/queries.sql,sketchdb:/path/to/queries.sql}'
 ```
 
 ### experiment_run_e2e.py (Full E2E Pipeline)

asap-tools/experiments/experiment_run_clickhouse.py

@@ -73,7 +73,7 @@
     providers.cloudlab.username=myuser \\
     providers.cloudlab.hostname_suffix=myexp.cloudlab.us \\
     experiment_params.dataset.local_data_file=/path/to/hits.json \\
-    'experiment_params.query_groups[0].sql_file=/path/to/clickbench.sql'
+    'experiment_params.query_groups[0].sql_file={baseline:/path/to/clickbench.sql}'
 
 Baseline + sketchdb:
   python experiment_run_clickhouse.py \\
@@ -83,7 +83,7 @@
     providers.cloudlab.username=myuser \\
     providers.cloudlab.hostname_suffix=myexp.cloudlab.us \\
     experiment_params.dataset.local_data_file=/path/to/hits.json \\
-    'experiment_params.query_groups[0].sql_file=/path/to/clickbench.sql'
+    'experiment_params.query_groups[0].sql_file={baseline:/path/to/clickbench.sql,sketchdb:/path/to/clickbench.sql}'
 
   Sketchdb mode runs when experiment_params.experiment contains an entry
   with mode: sketchdb (mirrors the Prometheus experiment config structure).
@@ -246,8 +246,8 @@ def _expand_groups(groups):
             sql_file = group.get("sql_file")
             if not sql_file or "queries" in group:
                 continue
-            # sql_file is per-mode (issue #491 precompute arms); label with
-            # baseline's text, matching the query text sketchdb also plans against.
+            # sql_file is per-mode; label with baseline's text, since every
+            # mode's query i answers baseline's query i.
             sql_file = sql_file.get(constants.BASELINE_EXPERIMENT_NAME)
             if not sql_file:
                 continue

asap-tools/experiments/experiment_utils/config.py

@@ -153,27 +153,58 @@ def _is_clickhouse_experiment(experiment_params: DictConfig) -> bool:
     return "dataset" in experiment_params
 
 
-def _resolve_sql_file(sql_file: Any, mode: str, idx: int, group_name: str) -> str:
-    """Resolve the sql_file path for one query group and mode.
-
-    ``sql_file`` is a dict keyed by mode name (e.g.
-    baseline/baseline_sketch/baseline_mv/...). sketchdb always resolves
-    through baseline's entry — ASAP's SQL-mode query engine must see the
-    exact same SQL text as baseline to pattern-match it against available
-    sketches, so it never gets its own dict entry.
+def _mode_sql_files(query_groups: Any, mode: str) -> List[Tuple[int, Any, str]]:
+    """(index, group, sql_file path) of each query group `mode` runs.
+
+    ``sql_file`` is a dict keyed by mode name (e.g. baseline/baseline_mv/
+    sketchdb). Every mode but sketchdb needs an entry in every group.
+    sketchdb skips the groups without one (e.g. queries ASAPQuery has no SQL
+    for), but only trailing groups: results are matched across modes by
+    global query index, so sketchdb's queries must be a prefix of baseline's,
+    and each file it runs must hold as many statements as baseline's.
     """
-    lookup_mode = (
-        constants.BASELINE_EXPERIMENT_NAME
-        if mode == constants.SKETCHDB_EXPERIMENT_NAME
-        else mode
-    )
-    path = sql_file.get(lookup_mode)
-    if not path:
-        raise ValueError(
-            f"Query group {idx} ({group_name!r}) missing sql_file entry for "
-            f"mode {lookup_mode!r}"
-        )
-    return path
+    runs = []
+    for idx, group in enumerate(query_groups):
+        name = group.get("name", str(idx))
+        sql_file = group.get("sql_file")
+        if not sql_file:
+            raise ValueError(f"Query group {idx} ({name!r}) missing 'sql_file'")
+        if not hasattr(sql_file, "get"):
+            raise ValueError(
+                f"Query group {idx} ({name!r}) sql_file must map mode name to "
+                f"SQL file, e.g. {{baseline: {sql_file}}}; got {sql_file!r}"
+            )
+        path = sql_file.get(mode)
+        if path:
+            if len(runs) != idx:
+                raise ValueError(
+                    f"Query group {idx} ({name!r}) has a sql_file for mode "
+                    f"{mode!r} after a group without one; a mode may skip "
+                    "only trailing groups, so its query indices match baseline's"
+                )
+            baseline_path = sql_file.get(constants.BASELINE_EXPERIMENT_NAME)
+            if (
+                mode != constants.BASELINE_EXPERIMENT_NAME
+                and baseline_path
+                and os.path.exists(path)
+                and os.path.exists(baseline_path)
+                and len(_load_sql_queries(path))
+                != len(_load_sql_queries(baseline_path))
+            ):
+                raise ValueError(
+                    f"Query group {idx} ({name!r}): mode {mode!r}'s {path!r} and "
+                    f"baseline's {baseline_path!r} hold different numbers of "
+                    "statements, so their query indices would not match"
+                )
+            runs.append((idx, group, path))
+        elif mode != constants.SKETCHDB_EXPERIMENT_NAME:
+            raise ValueError(
+                f"Query group {idx} ({name!r}) missing sql_file entry for "
+                f"mode {mode!r}"
+            )
+    if not runs:
+        raise ValueError(f"No query group has a sql_file entry for mode {mode!r}")
+    return runs
 
 
 def _validate_clickhouse_experiment_config(experiment_params: DictConfig) -> None:
@@ -239,7 +270,6 @@ def _validate_clickhouse_experiment_config(experiment_params: DictConfig) -> Non
             "At least one mode must be defined in experiment_params.experiment."
         )
     experiment_modes = {m["mode"] for m in experiment_params.experiment}
-    experiment_modes.discard(constants.SKETCHDB_EXPERIMENT_NAME)
 
     for i, group in enumerate(experiment_params.query_groups):
         sql_file = group.get("sql_file")
@@ -248,9 +278,8 @@ def _validate_clickhouse_experiment_config(experiment_params: DictConfig) -> Non
                 f"Query group {i} missing 'sql_file'. "
                 "Generate SQL files with benchmark/generate_queries.py first."
             )
-        group_name = group.get("name", str(i))
-        for mode in experiment_modes:
-            path = _resolve_sql_file(sql_file, mode, i, group_name)
+    for mode in sorted(experiment_modes):
+        for i, _, path in _mode_sql_files(experiment_params.query_groups, mode):
             if not os.path.exists(path):
                 raise ValueError(
                     f"Query group {i} sql_file for mode {mode!r} "
@@ -700,6 +729,22 @@ def _is_planned(query, planned):
     return re.fullmatch(r"[\s()+\-*/0-9.]*", query) is not None
 
 
+# A quoted time literal the query client renders, e.g.
+# '{{ end_datetime[:17] }}{{ "%02d" | format(end_time % 60 // 60 * 60) }}'.
+_SQL_TEMPLATED_TIME_LITERAL = re.compile(r"'\{\{[^']*\}\}[^']*'")
+
+
+def _comparable_sql(sql):
+    """`sql` without comment lines or a trailing ';', with each templated time
+    literal as NOW(): a saved plan holds the workload's SQL on NOW(), the
+    client config the same SQL on the literal the client fills in."""
+    # ponytail: textual; any templated literal reads as NOW(). The engine
+    # matches queries to the plan structurally, this only catches a wrong plan.
+    lines = [line for line in sql.splitlines() if not line.lstrip().startswith("--")]
+    sql = _SQL_TEMPLATED_TIME_LITERAL.sub("NOW()", "\n".join(lines))
+    return sql.strip().rstrip(";").strip()
+
+
 def _validate_sql_inference_config(inference_config):
     """Raise a clear error unless `inference_config` has the SQL schema shape."""
     if not isinstance(inference_config, dict):
@@ -797,11 +842,15 @@ def copy_prebuilt_controller_output(
                 "controller.prebuilt_output_dir inference_config.yaml has invalid "
                 "queries"
             )
-        planned = {q["query"] for q in inference_config["queries"]}
+        comparable = _comparable_sql if query_language == QUERY_LANGUAGE_SQL else str
+        planned = {comparable(q["query"]) for q in inference_config["queries"]}
         with open(controller_input_config) as f:
             groups = yaml.safe_load(f)["query_groups"]
         unplanned = [
-            q for g in groups for q in g["queries"] if not _is_planned(q, planned)
+            q
+            for g in groups
+            for q in g["queries"]
+            if not _is_planned(comparable(q), planned)
         ]
         if unplanned:
             raise ValueError(
@@ -1088,18 +1137,10 @@ def generate_clickhouse_client_configs(
     modes = list(mode_server_urls.keys())
     for mode, url in mode_server_urls.items():
         # Each mode may target a different sql_file (e.g. baseline_mv queries a
-        # materialized view, baseline_sketch swaps in a sketch function) — see
-        # _resolve_sql_file for the sketchdb-falls-back-to-baseline rule.
+        # materialized view) and sketchdb may skip trailing groups — see
+        # _mode_sql_files.
         built_groups = []
-        for idx, group in enumerate(query_groups_list):
-            sql_file = group.get("sql_file")
-            if not sql_file:
-                name = group.get("name", str(idx))
-                raise ValueError(f"Query group {idx!r} ({name!r}) missing 'sql_file'")
-            sql_file = _resolve_sql_file(
-                sql_file, mode, idx, group.get("name", str(idx))
-            )
-
+        for idx, group, sql_file in _mode_sql_files(query_groups_list, mode):
             queries = _load_sql_queries(sql_file)
             if not queries:
                 raise ValueError(f"No SQL statements found in {sql_file!r}")
@@ -1214,7 +1255,10 @@ def generate_sql_planner_input(
         groups_list = list(query_groups)
 
     planner_query_groups = []
-    for idx, group in enumerate(groups_list):
+    # The planner plans the SQL text the engine receives, i.e. sketchdb's.
+    for idx, group, sql_file in _mode_sql_files(
+        groups_list, constants.SKETCHDB_EXPERIMENT_NAME
+    ):
         ctrl_opts = dict(group.get("controller_options") or {})
         # Dropping an unknown key here would hide it from the planner's strict parse.
         unknown = set(ctrl_opts) - CONTROLLER_OPTION_KEYS
@@ -1224,16 +1268,6 @@ def generate_sql_planner_input(
                 f"{sorted(unknown)}; allowed: {sorted(CONTROLLER_OPTION_KEYS)}"
             )
 
-        sql_file = group.get("sql_file")
-        if not sql_file:
-            raise ValueError(f"query_groups[{idx}] missing 'sql_file'")
-        # sketchdb always plans against baseline's SQL text (see _resolve_sql_file).
-        sql_file = _resolve_sql_file(
-            sql_file,
-            constants.SKETCHDB_EXPERIMENT_NAME,
-            idx,
-            group.get("name", str(idx)),
-        )
         queries = _load_sql_queries(sql_file)
         if not queries:
             raise ValueError(f"No SQL statements found in {sql_file!r}")

asap-tools/experiments/post_experiment/single_experiment/calculate_fidelity.py

@@ -112,6 +112,15 @@ def get_timeseries_similarity_scores(
     similarity_functions,
     verbose: bool,
 ):
+    # The estimate mode may run only the first queries (sketchdb skips
+    # trailing query groups it has no SQL for); a gap among them still fails
+    # on the lookup below.
+    if len(estimate_results) < len(queries):
+        print(
+            f"Scoring the first {len(estimate_results)} of {len(queries)} "
+            "queries: the estimate mode ran no others"
+        )
+        queries = queries[: len(estimate_results)]
     similarity_scores = {
         f.__name__: {q: 0 for q in queries} for f in similarity_functions
     }

asap-tools/experiments/tests/test_calculate_fidelity.py

@@ -165,6 +165,32 @@ def test_sql_exact_matches_prometheus_json_estimate(self):
 
         self.assertAlmostEqual(scores["mape"][GROUPED_QUERY], 20.0)
 
+    def test_only_the_estimate_modes_queries_are_scored(self):
+        # sketchdb may skip trailing query groups; baseline ran them all.
+        exact = _sql_results("clickhouse", ["api\ta:1\t10\n"])
+        estimate = _sql_results("asap", ["api\ta:1\t11\n"])
+
+        scores = get_timeseries_similarity_scores(
+            {0: exact, 1: exact},
+            {0: estimate},
+            [GROUPED_QUERY, "SELECT 'skipped'"],
+            [mape],
+            verbose=False,
+        )
+
+        self.assertEqual(scores["mape"], {GROUPED_QUERY: 10.0})
+
+    def test_a_query_missing_among_the_estimate_modes_fails(self):
+        exact = _sql_results("clickhouse", ["api\ta:1\t10\n"])
+        with self.assertRaises(KeyError):
+            get_timeseries_similarity_scores(
+                {0: exact, 1: exact},
+                {1: exact},
+                [GROUPED_QUERY, GROUPED_QUERY + " "],
+                [mape],
+                verbose=False,
+            )
+
 
 if __name__ == "__main__":
     unittest.main()