Skip to content

Commit 2f70f7b

Browse files
feat(promql-compliance): let a suite supply fixed planner output
Add an optional suite-level planner_output_dir holding streaming_config.yaml and inference_config.yaml. When set, the runner validates both files at load time, skips generating the planner input, and adds a Compose override that parks the planner behind an inactive profile and mounts the directory where the query engine reads planner output. This lets the harness exercise engine configs the planner does not produce yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent b101a35 commit 2f70f7b

6 files changed

Lines changed: 316 additions & 5 deletions

File tree

‎promql-compliance/ARCHITECTURE.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ The runner performs these stages:
5454
are relative to this timestamp.
5555
3. Generate temporary planner and engine configuration.
5656
4. Start Prometheus and wait for its health check.
57-
5. Run the planner. Its output is placed in a named shared volume.
57+
5. Run the planner. Its output is placed in a named shared volume. A suite
58+
with `planner_output_dir` skips the planner and mounts that directory
59+
instead.
5860
6. Start the ASAPQuery query engine after the planner completes.
5961
7. Send the same encoded remote-write batches to Prometheus and ASAPQuery.
6062
8. Wait for both query APIs to be ready and for the first probe query to return

‎promql-compliance/HOW_TO.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,25 @@ To change the five-minute cap or one-second ingestion assumption, change the
128128
runner’s generated-config logic in `runner/run.go`. Do not edit the generated
129129
configuration: it is temporary and is deleted after the run.
130130

131+
## Use fixed planner output
132+
133+
By default the planner generates the engine's streaming and inference
134+
configs from the suite. To test configs the planner does not produce, such as
135+
roll-ups from a finer grouping, check the configs in next to the suite and
136+
point the suite at their directory:
137+
138+
```yaml
139+
name: rollup
140+
planner_output_dir: rollup-planner-output
141+
```
142+
143+
The path is relative to the suite file. The directory must contain
144+
`streaming_config.yaml` and `inference_config.yaml` in the planner's output
145+
format; the runner fails at load time if either is missing. The planner
146+
service then does not run, and the query engine reads the directory in place
147+
of the planner's output volume. With already-running targets the field has no
148+
effect. `suites/rollup.yaml` is an example.
149+
131150
## Configure tolerance
132151

133152
Set a suite-wide default:

‎promql-compliance/runner/config.go‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"math"
77
"os"
8+
"path/filepath"
89
"time"
910

1011
"gopkg.in/yaml.v3"
@@ -15,6 +16,10 @@ const (
1516
instantOrderDescending = "descending"
1617
orderGroupingBy = "by"
1718
orderGroupingWithout = "without"
19+
20+
// File names the planner writes and the query engine reads.
21+
plannerStreamingConfigFile = "streaming_config.yaml"
22+
plannerInferenceConfigFile = "inference_config.yaml"
1823
)
1924

2025
// Suite is a data-independent collection of PromQL cases. All timestamps are
@@ -23,6 +28,11 @@ type Suite struct {
2328
Name string `yaml:"name" json:"name"`
2429
ComparisonDefaults ComparisonPolicy `yaml:"comparison_defaults" json:"comparisonDefaults"`
2530
Queries []QueryCase `yaml:"queries" json:"queries"`
31+
// PlannerOutputDir, when set, holds fixed streaming and inference configs
32+
// that the query engine uses instead of running the planner. It is
33+
// relative to the suite file; LoadSuiteFile resolves it to an absolute
34+
// path.
35+
PlannerOutputDir string `yaml:"planner_output_dir" json:"plannerOutputDir,omitempty"`
2636
}
2737

2838
type QueryCase struct {
@@ -123,13 +133,47 @@ func LoadSuite(contents []byte) (Suite, error) {
123133
return suite, nil
124134
}
125135

126-
// LoadSuiteFile reads a query suite from disk.
136+
// LoadSuiteFile reads a query suite from disk and resolves its
137+
// planner_output_dir against the suite file's directory.
127138
func LoadSuiteFile(path string) (Suite, error) {
128139
contents, err := os.ReadFile(path)
129140
if err != nil {
130141
return Suite{}, fmt.Errorf("read query suite %q: %w", path, err)
131142
}
132-
return LoadSuite(contents)
143+
suite, err := LoadSuite(contents)
144+
if err != nil {
145+
return Suite{}, err
146+
}
147+
if suite.PlannerOutputDir != "" {
148+
resolved, err := resolvePlannerOutputDir(filepath.Dir(path), suite.PlannerOutputDir)
149+
if err != nil {
150+
return Suite{}, fmt.Errorf("query suite %q: %w", suite.Name, err)
151+
}
152+
suite.PlannerOutputDir = resolved
153+
}
154+
return suite, nil
155+
}
156+
157+
func resolvePlannerOutputDir(suiteDirectory, configured string) (string, error) {
158+
directory := configured
159+
if !filepath.IsAbs(directory) {
160+
directory = filepath.Join(suiteDirectory, directory)
161+
}
162+
directory, err := filepath.Abs(directory)
163+
if err != nil {
164+
return "", fmt.Errorf("resolve planner_output_dir %q: %w", configured, err)
165+
}
166+
for _, name := range []string{plannerStreamingConfigFile, plannerInferenceConfigFile} {
167+
file := filepath.Join(directory, name)
168+
info, err := os.Stat(file)
169+
if err != nil {
170+
return "", fmt.Errorf("planner_output_dir %q: %w", configured, err)
171+
}
172+
if !info.Mode().IsRegular() {
173+
return "", fmt.Errorf("planner_output_dir %q: %s is not a regular file", configured, file)
174+
}
175+
}
176+
return directory, nil
133177
}
134178

135179
func (q QueryCase) InstantTimes(base time.Time) []time.Time {

‎promql-compliance/runner/config_test.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package runner
22

33
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"strings"
48
"testing"
59
"time"
610
)
@@ -171,3 +175,118 @@ queries:
171175
t.Fatal("LoadSuite accepted an invalid instant vector order direction")
172176
}
173177
}
178+
179+
const plannerOutputSuite = `name: fixed-output
180+
planner_output_dir: %s
181+
queries:
182+
- name: up
183+
expr: up
184+
instant_offsets_seconds: [60]
185+
`
186+
187+
// writePlannerOutputSuite writes a suite under root/suites whose
188+
// planner_output_dir is configured, plus the named files under
189+
// root/suites/fixed.
190+
func writePlannerOutputSuite(t *testing.T, configured string, files ...string) (string, string) {
191+
t.Helper()
192+
root := t.TempDir()
193+
suiteDirectory := filepath.Join(root, "suites")
194+
outputDirectory := filepath.Join(suiteDirectory, "fixed")
195+
if err := os.MkdirAll(outputDirectory, 0o755); err != nil {
196+
t.Fatalf("create planner output directory: %v", err)
197+
}
198+
for _, name := range files {
199+
if err := os.WriteFile(filepath.Join(outputDirectory, name), []byte("{}\n"), 0o600); err != nil {
200+
t.Fatalf("write %s: %v", name, err)
201+
}
202+
}
203+
suitePath := filepath.Join(suiteDirectory, "suite.yaml")
204+
if err := os.WriteFile(suitePath, []byte(fmt.Sprintf(plannerOutputSuite, configured)), 0o600); err != nil {
205+
t.Fatalf("write suite: %v", err)
206+
}
207+
return suitePath, outputDirectory
208+
}
209+
210+
func TestLoadSuiteFileResolvesPlannerOutputDirAgainstSuiteFile(t *testing.T) {
211+
suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile, plannerInferenceConfigFile)
212+
213+
// Load through a relative suite path so resolution cannot depend on the
214+
// caller's working directory matching the suite's directory.
215+
workingDirectory, err := os.Getwd()
216+
if err != nil {
217+
t.Fatalf("Getwd: %v", err)
218+
}
219+
relativeSuitePath, err := filepath.Rel(workingDirectory, suitePath)
220+
if err != nil {
221+
t.Fatalf("Rel: %v", err)
222+
}
223+
suite, err := LoadSuiteFile(relativeSuitePath)
224+
if err != nil {
225+
t.Fatalf("LoadSuiteFile: %v", err)
226+
}
227+
if got, want := suite.PlannerOutputDir, outputDirectory; got != want {
228+
t.Fatalf("planner output dir = %q, want %q", got, want)
229+
}
230+
}
231+
232+
func TestLoadSuiteFileAcceptsAbsolutePlannerOutputDir(t *testing.T) {
233+
outputDirectory := t.TempDir()
234+
for _, name := range []string{plannerStreamingConfigFile, plannerInferenceConfigFile} {
235+
if err := os.WriteFile(filepath.Join(outputDirectory, name), []byte("{}\n"), 0o600); err != nil {
236+
t.Fatalf("write %s: %v", name, err)
237+
}
238+
}
239+
suitePath, _ := writePlannerOutputSuite(t, outputDirectory)
240+
241+
suite, err := LoadSuiteFile(suitePath)
242+
if err != nil {
243+
t.Fatalf("LoadSuiteFile: %v", err)
244+
}
245+
if got, want := suite.PlannerOutputDir, outputDirectory; got != want {
246+
t.Fatalf("planner output dir = %q, want %q", got, want)
247+
}
248+
}
249+
250+
func TestLoadSuiteFileLeavesPlannerOutputDirUnsetByDefault(t *testing.T) {
251+
suite, err := LoadSuiteFile("../suites/temporal.yaml")
252+
if err != nil {
253+
t.Fatalf("LoadSuiteFile: %v", err)
254+
}
255+
if suite.PlannerOutputDir != "" {
256+
t.Fatalf("planner output dir = %q, want empty", suite.PlannerOutputDir)
257+
}
258+
}
259+
260+
func TestLoadSuiteFileRejectsIncompletePlannerOutputDir(t *testing.T) {
261+
for _, test := range []struct {
262+
name string
263+
configured string
264+
files []string
265+
missing string
266+
}{
267+
{"missing inference config", "fixed", []string{plannerStreamingConfigFile}, plannerInferenceConfigFile},
268+
{"missing streaming config", "fixed", []string{plannerInferenceConfigFile}, plannerStreamingConfigFile},
269+
{"missing directory", "absent", nil, plannerStreamingConfigFile},
270+
} {
271+
t.Run(test.name, func(t *testing.T) {
272+
suitePath, _ := writePlannerOutputSuite(t, test.configured, test.files...)
273+
_, err := LoadSuiteFile(suitePath)
274+
if err == nil {
275+
t.Fatal("LoadSuiteFile accepted a planner_output_dir without both config files")
276+
}
277+
if !strings.Contains(err.Error(), test.missing) {
278+
t.Fatalf("error %q does not name the missing file %q", err, test.missing)
279+
}
280+
})
281+
}
282+
}
283+
284+
func TestLoadSuiteFileRejectsPlannerOutputDirectoryInPlaceOfFile(t *testing.T) {
285+
suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile)
286+
if err := os.Mkdir(filepath.Join(outputDirectory, plannerInferenceConfigFile), 0o755); err != nil {
287+
t.Fatalf("create directory: %v", err)
288+
}
289+
if _, err := LoadSuiteFile(suitePath); err == nil {
290+
t.Fatal("LoadSuiteFile accepted a directory as a planner config file")
291+
}
292+
}

‎promql-compliance/runner/run.go‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,27 @@ const (
2727
dataReadyTimeout = 3 * time.Minute
2828
readinessPollInterval = time.Second
2929
composeShutdownTimeout = time.Minute
30+
31+
plannerOutputMountPath = "/asap-planner-output"
32+
fixedPlannerOutputComposeFile = "docker-compose.fixed-planner-output.yml"
33+
fixedPlannerOutputDirEnv = "ASAP_FIXED_PLANNER_OUTPUT_DIR"
3034
)
3135

36+
// fixedPlannerOutputCompose overrides the base Compose file for suites with a
37+
// planner_output_dir: the planner sits behind a never-activated profile so it
38+
// does not run, and the query engine mounts the fixed directory where the
39+
// planner's output volume would be.
40+
const fixedPlannerOutputCompose = `services:
41+
planner:
42+
profiles: ["generated-planner-output"]
43+
queryengine:
44+
depends_on: !override
45+
prometheus:
46+
condition: service_healthy
47+
volumes:
48+
- "${` + fixedPlannerOutputDirEnv + `:?` + fixedPlannerOutputDirEnv + ` is required}:` + plannerOutputMountPath + `:ro"
49+
`
50+
3251
type RunOptions struct {
3352
DatasetPath string
3453
SuitePath string
@@ -79,6 +98,13 @@ func Run(ctx context.Context, options RunOptions) (Report, error) {
7998
project: options.ComposeProject,
8099
env: []string{"ASAP_TEST_CONFIG_DIR=" + runDirectory},
81100
}
101+
if suite.PlannerOutputDir != "" {
102+
lifecycle.files = append(
103+
append([]string(nil), options.ComposeFiles...),
104+
filepath.Join(runDirectory, fixedPlannerOutputComposeFile),
105+
)
106+
lifecycle.env = append(lifecycle.env, fixedPlannerOutputDirEnv+"="+suite.PlannerOutputDir)
107+
}
82108
if len(options.ComposeFiles) > 0 {
83109
if err := lifecycle.Start(ctx); err != nil {
84110
return Report{}, err
@@ -345,7 +371,24 @@ type plannerCleanup struct {
345371
Policy string `yaml:"policy"`
346372
}
347373

374+
// writeGeneratedConfigs writes the engine config plus either the planner input
375+
// or, for a suite with fixed planner output, the Compose override that skips
376+
// the planner. The planner input is not generated in the fixed case, so its
377+
// timing derivation cannot reject queries the planner never sees.
348378
func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite) error {
379+
if err := writeEngineConfig(directory); err != nil {
380+
return err
381+
}
382+
if suite.PlannerOutputDir != "" {
383+
if err := os.WriteFile(filepath.Join(directory, fixedPlannerOutputComposeFile), []byte(fixedPlannerOutputCompose), 0o600); err != nil {
384+
return fmt.Errorf("write fixed planner output compose override: %w", err)
385+
}
386+
return nil
387+
}
388+
return writePlannerConfig(directory, fixture, suite)
389+
}
390+
391+
func writePlannerConfig(directory string, fixture seeder.Fixture, suite Suite) error {
349392
metrics := make(map[string]map[string]struct{})
350393
for _, series := range fixture.Series {
351394
if metrics[series.Metric] == nil {
@@ -399,6 +442,10 @@ func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite
399442
if err := os.WriteFile(filepath.Join(directory, "controller-config.yaml"), contents, 0o600); err != nil {
400443
return fmt.Errorf("write generated planner config: %w", err)
401444
}
445+
return nil
446+
}
447+
448+
func writeEngineConfig(directory string) error {
402449
engineConfig := []byte(`output_dir: "/app/outputs"
403450
log_level: "INFO"
404451
data_ingestion_interval_ms: 1000
@@ -414,8 +461,8 @@ store:
414461
ingest:
415462
type: "http_remote_write"
416463
port: 9091
417-
inference_config: "/asap-planner-output/inference_config.yaml"
418-
streaming_config: "/asap-planner-output/streaming_config.yaml"
464+
inference_config: "` + plannerOutputMountPath + `/` + plannerInferenceConfigFile + `"
465+
streaming_config: "` + plannerOutputMountPath + `/` + plannerStreamingConfigFile + `"
419466
`)
420467
if err := os.WriteFile(filepath.Join(directory, "engine_config.yaml"), engineConfig, 0o600); err != nil {
421468
return fmt.Errorf("write generated engine config: %w", err)

0 commit comments

Comments
 (0)