fix(asap-tools): let long-window baseline queries finish

asap-tools/experiments/constants.py

@@ -41,6 +41,12 @@
 
 PROMETHEUS_CONFIG_DIR = "prometheus_config"
 PROMETHEUS_CONFIG_FILE = "prometheus.yml"
+# Long-window baseline queries can run for minutes; Prometheus would cancel
+# them after its 2-minute default.
+PROMETHEUS_QUERY_TIMEOUT = "24h"
+# The 50M default rejects 24h windows over a few hundred 1s-scraped series.
+# Peak query memory is roughly 16 B per sample (~8 GB at this cap).
+PROMETHEUS_QUERY_MAX_SAMPLES = 500_000_000
 PROMETHEUS_REMOTE_WRITE_ENDPOINT = "/api/v1/write"
 HTTP_READINESS_CURL_TIMEOUT_SECONDS = 5
 QUERY_ENGINE_RUNTIME_INFO_ENDPOINT = "/api/v1/status/runtimeinfo"

asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2

@@ -11,7 +11,14 @@ services:
     command:
       - "-storageDataPath=/victoria-metrics-data"
       - "-httpListenAddr=:8428"
-      - "-retentionPeriod=1d"
+      # Keep a 24 h window plus the run, and let long-window queries finish
+      # (the defaults are 30 s per query and 10 s waiting in the queue).
+      - "-retentionPeriod=3d"
+      - "-search.maxQueryDuration=24h"
+      - "-search.maxQueueDuration=24h"
+      # Raise the per-query caps so 24 h windows over many series aren't rejected.
+      - "-search.maxSamplesPerQuery=10000000000"
+      - "-search.maxPointsPerTimeseries=1000000"
     {% if cpu_limit or memory_limit -%}
     deploy:
       resources:

asap-tools/experiments/experiment_utils/services/docker_prometheus.py

@@ -6,7 +6,12 @@
 
 from .base import DockerServiceBase
 from experiment_utils.providers.base import InfrastructureProvider
-from constants import PROMETHEUS_CONFIG_DIR, PROMETHEUS_CONFIG_FILE
+from constants import (
+    PROMETHEUS_CONFIG_DIR,
+    PROMETHEUS_CONFIG_FILE,
+    PROMETHEUS_QUERY_MAX_SAMPLES,
+    PROMETHEUS_QUERY_TIMEOUT,
+)
 
 
 class DockerPrometheusService(DockerServiceBase):
@@ -98,6 +103,8 @@ def start(
                 "--web.console.libraries=/etc/prometheus/console_libraries",
                 "--web.console.templates=/etc/prometheus/consoles",
                 "--web.enable-lifecycle",
+                f"--query.timeout={PROMETHEUS_QUERY_TIMEOUT}",
+                f"--query.max-samples={PROMETHEUS_QUERY_MAX_SAMPLES}",
             ]
         )
 

asap-tools/experiments/experiment_utils/services/prometheus.py

@@ -8,7 +8,12 @@
 
 from .base import BaseService
 from experiment_utils.providers.base import InfrastructureProvider
-from constants import PROMETHEUS_CONFIG_DIR, PROMETHEUS_CONFIG_FILE
+from constants import (
+    PROMETHEUS_CONFIG_DIR,
+    PROMETHEUS_CONFIG_FILE,
+    PROMETHEUS_QUERY_MAX_SAMPLES,
+    PROMETHEUS_QUERY_TIMEOUT,
+)
 
 
 class PrometheusService(BaseService):
@@ -89,7 +94,12 @@ def _start_prometheus(self, experiment_output_dir: str) -> None:
             time.sleep(3)
 
         # Start prometheus
-        cmd = f"./prometheus --config.file={PROMETHEUS_CONFIG_FILE} > /dev/null 2>&1 < /dev/null &"
+        cmd = (
+            f"./prometheus --config.file={PROMETHEUS_CONFIG_FILE}"
+            f" --query.timeout={PROMETHEUS_QUERY_TIMEOUT}"
+            f" --query.max-samples={PROMETHEUS_QUERY_MAX_SAMPLES}"
+            " > /dev/null 2>&1 < /dev/null &"
+        )
         self.provider.execute_command(
             node_idx=self.node_offset,
             cmd=cmd,

asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py

@@ -1,5 +1,5 @@
 import requests
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
 from requests.adapters import HTTPAdapter
 
 from .query_client import QueryClient, QueryResponse
@@ -15,7 +15,9 @@ def __init__(
         database: str = "default",
         user: str = "default",
         password: str = "",
-        timeout: int = 30,
+        # (connect, read): long-window baseline queries can take minutes, but a
+        # dead server must still fail instead of hanging the querier.
+        timeout: Tuple[float, float] = (10, 24 * 60 * 60),
         **kwargs: Any,
     ):
         super().__init__(server_url, server_name)
@@ -96,7 +98,7 @@ def execute_query(
             return QueryResponse(
                 success=False,
                 data=None,
-                error_message=f"Request timed out after {self.timeout}s",
+                error_message=f"Request timed out (connect, read timeout: {self.timeout}s)",
             )
         except Exception as e:
             return QueryResponse(