From 94f3f513e47a7a82199d6160d8fb6832cab0c7bf Mon Sep 17 00:00:00 2001 From: Damilola Edwards Date: Sat, 4 Jul 2026 17:35:44 +0100 Subject: [PATCH] Preserve integer precision in ClickHouse query parameters Operation request bodies were decoded without UseNumber, so JSON numbers became float64. The ClickHouse parameter formatters then fell through to fmt.Sprint, which renders float64 with %g: any integer of 1e6 or more came out in scientific notation (7.5e+06), which ClickHouse cannot parse as an integer type, and integers above 2^53 lost precision before they were ever formatted. This broke parameterized queries filtering by block, slot, timestamp, or gas. Decode numbers as json.Number so values keep their exact form, teach optionalIntArg to read json.Number, and format json.Number verbatim (with a plain-decimal fallback for float64). Add tests covering large integers, arrays, precision above 2^53, and non-integer values. --- pkg/server/clickhouse_param_format_test.go | 63 ++++++++++++++++++++++ pkg/server/operations_clickhouse.go | 10 ++++ pkg/server/operations_request.go | 12 ++++- 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 pkg/server/clickhouse_param_format_test.go diff --git a/pkg/server/clickhouse_param_format_test.go b/pkg/server/clickhouse_param_format_test.go new file mode 100644 index 00000000..3856eb63 --- /dev/null +++ b/pkg/server/clickhouse_param_format_test.go @@ -0,0 +1,63 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func decodeTestParams(t *testing.T, body string) map[string]any { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + decoded, err := decodeOperationRequest(req) + if err != nil { + t.Fatalf("decodeOperationRequest: %v", err) + } + return optionalMapArg(decoded.Args, "parameters") +} + +// Numeric ClickHouse parameters must render as plain integer literals that +// ClickHouse can parse, regardless of magnitude. Before the fix, values >= 1e6 +// rendered in scientific notation and values above 2^53 lost precision. +func TestClickHouseNumericParamsRenderAsIntegers(t *testing.T) { + params := decodeTestParams(t, `{"args":{"parameters":{"slot":7500000,"block":1000000,"ts":1700000000,"limit":42,"ids":[1000000,2000000]}}}`) + + cases := map[string]string{ + "slot": "7500000", + "block": "1000000", + "ts": "1700000000", + "limit": "42", + } + for key, want := range cases { + if got := formatClickHouseParamValue(params[key]); got != want { + t.Errorf("param %s = %q, want %q", key, got, want) + } + } + + if got := formatClickHouseParamValue(params["ids"]); got != "[1000000,2000000]" { + t.Errorf("array param = %q, want [1000000,2000000]", got) + } +} + +// Integers above 2^53 keep their exact value. +func TestClickHouseParamPreservesLargeIntegerPrecision(t *testing.T) { + params := decodeTestParams(t, `{"args":{"parameters":{"bigid":9007199254740993}}}`) + + if got := formatClickHouseParamValue(params["bigid"]); got != "9007199254740993" { + t.Errorf("bigid = %q, want 9007199254740993", got) + } +} + +// Non-integer numeric parameters render in plain decimal notation. +func TestClickHouseFloatParamRendersWithoutExponent(t *testing.T) { + params := decodeTestParams(t, `{"args":{"parameters":{"ratio":1500000.5}}}`) + + got := formatClickHouseParamValue(params["ratio"]) + if strings.Contains(got, "e") || strings.Contains(got, "E") { + t.Errorf("ratio = %q, want plain decimal without exponent", got) + } + if got != "1500000.5" { + t.Errorf("ratio = %q, want 1500000.5", got) + } +} diff --git a/pkg/server/operations_clickhouse.go b/pkg/server/operations_clickhouse.go index 3633c34f..e0eba0f0 100644 --- a/pkg/server/operations_clickhouse.go +++ b/pkg/server/operations_clickhouse.go @@ -1,10 +1,12 @@ package server import ( + "encoding/json" "fmt" "net/http" "net/url" "reflect" + "strconv" "strings" "github.com/ethpandaops/panda/pkg/operations" @@ -135,6 +137,10 @@ func formatClickHouseParamValue(value any) string { return "1" } return "0" + case json.Number: + return v.String() + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) default: return fmt.Sprint(v) } @@ -164,6 +170,10 @@ func formatClickHouseArrayLiteral(value any) string { return "0" case string: return quoteClickHouseStringLiteral(v) + case json.Number: + return v.String() + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) default: return fmt.Sprint(v) } diff --git a/pkg/server/operations_request.go b/pkg/server/operations_request.go index 044ae615..6b735797 100644 --- a/pkg/server/operations_request.go +++ b/pkg/server/operations_request.go @@ -16,7 +16,12 @@ func decodeOperationRequest(r *http.Request) (operations.Request, error) { defer func() { _ = r.Body.Close() }() var req operations.Request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + decoder := json.NewDecoder(r.Body) + // Decode numbers as json.Number so integer parameters keep their exact + // value. Without this they become float64, which loses precision above + // 2^53 and renders large values in scientific notation. + decoder.UseNumber() + if err := decoder.Decode(&req); err != nil { return operations.Request{}, fmt.Errorf("invalid request body: %w", err) } @@ -77,6 +82,11 @@ func optionalSliceArg(args map[string]any, key string) []any { func optionalIntArg(args map[string]any, key string, fallback int) int { switch value := args[key].(type) { + case json.Number: + if n, err := value.Int64(); err == nil { + return int(n) + } + return fallback case float64: return int(value) case int: