Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions pkg/server/clickhouse_param_format_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package server

import (
"net/http"
"net/http/httptest"
"strings"
"testing"
)

func decodeTestParams(t *testing.T, body string) map[string]any {
t.Helper()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
decoded, err := decodeOperationRequest(req)
if err != nil {
t.Fatalf("decodeOperationRequest: %v", err)
}
return optionalMapArg(decoded.Args, "parameters")
}

// Numeric ClickHouse parameters must render as plain integer literals that
// ClickHouse can parse, regardless of magnitude. Before the fix, values >= 1e6
// rendered in scientific notation and values above 2^53 lost precision.
func TestClickHouseNumericParamsRenderAsIntegers(t *testing.T) {
params := decodeTestParams(t, `{"args":{"parameters":{"slot":7500000,"block":1000000,"ts":1700000000,"limit":42,"ids":[1000000,2000000]}}}`)

cases := map[string]string{
"slot": "7500000",
"block": "1000000",
"ts": "1700000000",
"limit": "42",
}
for key, want := range cases {
if got := formatClickHouseParamValue(params[key]); got != want {
t.Errorf("param %s = %q, want %q", key, got, want)
}
}

if got := formatClickHouseParamValue(params["ids"]); got != "[1000000,2000000]" {
t.Errorf("array param = %q, want [1000000,2000000]", got)
}
}

// Integers above 2^53 keep their exact value.
func TestClickHouseParamPreservesLargeIntegerPrecision(t *testing.T) {
params := decodeTestParams(t, `{"args":{"parameters":{"bigid":9007199254740993}}}`)

if got := formatClickHouseParamValue(params["bigid"]); got != "9007199254740993" {
t.Errorf("bigid = %q, want 9007199254740993", got)
}
}

// Non-integer numeric parameters render in plain decimal notation.
func TestClickHouseFloatParamRendersWithoutExponent(t *testing.T) {
params := decodeTestParams(t, `{"args":{"parameters":{"ratio":1500000.5}}}`)

got := formatClickHouseParamValue(params["ratio"])
if strings.Contains(got, "e") || strings.Contains(got, "E") {
t.Errorf("ratio = %q, want plain decimal without exponent", got)
}
if got != "1500000.5" {
t.Errorf("ratio = %q, want 1500000.5", got)
}
}
10 changes: 10 additions & 0 deletions pkg/server/operations_clickhouse.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package server

import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"reflect"
"strconv"
"strings"

"github.com/ethpandaops/panda/pkg/operations"
Expand Down Expand Up @@ -135,6 +137,10 @@ func formatClickHouseParamValue(value any) string {
return "1"
}
return "0"
case json.Number:
return v.String()
case float64:
return strconv.FormatFloat(v, 'f', -1, 64)
default:
return fmt.Sprint(v)
}
Expand Down Expand Up @@ -164,6 +170,10 @@ func formatClickHouseArrayLiteral(value any) string {
return "0"
case string:
return quoteClickHouseStringLiteral(v)
case json.Number:
return v.String()
case float64:
return strconv.FormatFloat(v, 'f', -1, 64)
default:
return fmt.Sprint(v)
}
Expand Down
12 changes: 11 additions & 1 deletion pkg/server/operations_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,12 @@ func decodeOperationRequest(r *http.Request) (operations.Request, error) {
defer func() { _ = r.Body.Close() }()

var req operations.Request
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
decoder := json.NewDecoder(r.Body)
// Decode numbers as json.Number so integer parameters keep their exact
// value. Without this they become float64, which loses precision above
// 2^53 and renders large values in scientific notation.
decoder.UseNumber()
if err := decoder.Decode(&req); err != nil {
return operations.Request{}, fmt.Errorf("invalid request body: %w", err)
}

Expand Down Expand Up @@ -77,6 +82,11 @@ func optionalSliceArg(args map[string]any, key string) []any {

func optionalIntArg(args map[string]any, key string, fallback int) int {
switch value := args[key].(type) {
case json.Number:
if n, err := value.Int64(); err == nil {
return int(n)
}
return fallback
case float64:
return int(value)
case int:
Expand Down
Loading