diff --git a/pkg/server/clickhouse_param_format_test.go b/pkg/server/clickhouse_param_format_test.go new file mode 100644 index 00000000..3856eb63 --- /dev/null +++ b/pkg/server/clickhouse_param_format_test.go @@ -0,0 +1,63 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func decodeTestParams(t *testing.T, body string) map[string]any { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + decoded, err := decodeOperationRequest(req) + if err != nil { + t.Fatalf("decodeOperationRequest: %v", err) + } + return optionalMapArg(decoded.Args, "parameters") +} + +// Numeric ClickHouse parameters must render as plain integer literals that +// ClickHouse can parse, regardless of magnitude. Before the fix, values >= 1e6 +// rendered in scientific notation and values above 2^53 lost precision. +func TestClickHouseNumericParamsRenderAsIntegers(t *testing.T) { + params := decodeTestParams(t, `{"args":{"parameters":{"slot":7500000,"block":1000000,"ts":1700000000,"limit":42,"ids":[1000000,2000000]}}}`) + + cases := map[string]string{ + "slot": "7500000", + "block": "1000000", + "ts": "1700000000", + "limit": "42", + } + for key, want := range cases { + if got := formatClickHouseParamValue(params[key]); got != want { + t.Errorf("param %s = %q, want %q", key, got, want) + } + } + + if got := formatClickHouseParamValue(params["ids"]); got != "[1000000,2000000]" { + t.Errorf("array param = %q, want [1000000,2000000]", got) + } +} + +// Integers above 2^53 keep their exact value. +func TestClickHouseParamPreservesLargeIntegerPrecision(t *testing.T) { + params := decodeTestParams(t, `{"args":{"parameters":{"bigid":9007199254740993}}}`) + + if got := formatClickHouseParamValue(params["bigid"]); got != "9007199254740993" { + t.Errorf("bigid = %q, want 9007199254740993", got) + } +} + +// Non-integer numeric parameters render in plain decimal notation. +func TestClickHouseFloatParamRendersWithoutExponent(t *testing.T) { + params := decodeTestParams(t, `{"args":{"parameters":{"ratio":1500000.5}}}`) + + got := formatClickHouseParamValue(params["ratio"]) + if strings.Contains(got, "e") || strings.Contains(got, "E") { + t.Errorf("ratio = %q, want plain decimal without exponent", got) + } + if got != "1500000.5" { + t.Errorf("ratio = %q, want 1500000.5", got) + } +} diff --git a/pkg/server/operations_clickhouse.go b/pkg/server/operations_clickhouse.go index 3633c34f..e0eba0f0 100644 --- a/pkg/server/operations_clickhouse.go +++ b/pkg/server/operations_clickhouse.go @@ -1,10 +1,12 @@ package server import ( + "encoding/json" "fmt" "net/http" "net/url" "reflect" + "strconv" "strings" "github.com/ethpandaops/panda/pkg/operations" @@ -135,6 +137,10 @@ func formatClickHouseParamValue(value any) string { return "1" } return "0" + case json.Number: + return v.String() + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) default: return fmt.Sprint(v) } @@ -164,6 +170,10 @@ func formatClickHouseArrayLiteral(value any) string { return "0" case string: return quoteClickHouseStringLiteral(v) + case json.Number: + return v.String() + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) default: return fmt.Sprint(v) } diff --git a/pkg/server/operations_request.go b/pkg/server/operations_request.go index 044ae615..6b735797 100644 --- a/pkg/server/operations_request.go +++ b/pkg/server/operations_request.go @@ -16,7 +16,12 @@ func decodeOperationRequest(r *http.Request) (operations.Request, error) { defer func() { _ = r.Body.Close() }() var req operations.Request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + decoder := json.NewDecoder(r.Body) + // Decode numbers as json.Number so integer parameters keep their exact + // value. Without this they become float64, which loses precision above + // 2^53 and renders large values in scientific notation. + decoder.UseNumber() + if err := decoder.Decode(&req); err != nil { return operations.Request{}, fmt.Errorf("invalid request body: %w", err) } @@ -77,6 +82,11 @@ func optionalSliceArg(args map[string]any, key string) []any { func optionalIntArg(args map[string]any, key string, fallback int) int { switch value := args[key].(type) { + case json.Number: + if n, err := value.Int64(); err == nil { + return int(n) + } + return fallback case float64: return int(value) case int: