From e5e99a2dc340459ae943e9312b6687d565123d30 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Fri, 7 Aug 2026 23:43:35 +0200 Subject: [PATCH 01/17] feat(compute): add cumulative sum kernels --- .../internal/kernels/vector_cumulative.go | 242 ++++++++++++++++++ arrow/compute/registry.go | 1 + arrow/compute/vector_cumulative.go | 64 +++++ arrow/compute/vector_cumulative_test.go | 134 ++++++++++ 4 files changed, 441 insertions(+) create mode 100644 arrow/compute/internal/kernels/vector_cumulative.go create mode 100644 arrow/compute/vector_cumulative.go create mode 100644 arrow/compute/vector_cumulative_test.go diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go new file mode 100644 index 000000000..a0dde8386 --- /dev/null +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 + +package kernels + +import ( + "fmt" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/bitutil" + "github.com/apache/arrow-go/v18/arrow/compute/exec" + "github.com/apache/arrow-go/v18/arrow/scalar" +) + +// CumulativeOptions controls cumulative operations. +type CumulativeOptions struct { + // Start is the initial value. A nil value uses the zero value for the + // input type. + Start scalar.Scalar + // SkipNulls controls whether a null input stops the cumulative operation. + // Null positions are still null in the output when this is true. + SkipNulls bool `compute:"skip_nulls"` +} + +func (CumulativeOptions) TypeName() string { return "CumulativeOptions" } + +type cumulativeSumState[T arrow.NumericType] struct { + current T + skipNulls bool + encounteredNull bool + checked bool +} + +func cumulativeStartValue[T arrow.NumericType](start scalar.Scalar, typ arrow.DataType) (T, error) { + var zero T + if start == nil { + return zero, nil + } + if !start.IsValid() { + return zero, fmt.Errorf("%w: cumulative sum start value must be valid", arrow.ErrInvalid) + } + + casted, err := start.CastTo(typ) + if err != nil { + return zero, fmt.Errorf("%w: cannot cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, err) + } + if releasable, ok := casted.(scalar.Releasable); ok { + defer releasable.Release() + } + + primitive, ok := casted.(scalar.PrimitiveScalar) + if !ok { + return zero, fmt.Errorf("%w: cumulative sum start value is not primitive", arrow.ErrInvalid) + } + + span := &exec.ArraySpan{Type: typ, Len: 1} + span.Buffers[1].Buf = primitive.Data() + return exec.GetSpanValues[T](span, 1)[0], nil +} + +func initCumulativeSum[T arrow.NumericType](checked bool) exec.KernelInitFn { + return func(_ *exec.KernelCtx, args exec.KernelInitArgs) (exec.KernelState, error) { + opts := &CumulativeOptions{} + if args.Options != nil { + var ok bool + opts, ok = args.Options.(*CumulativeOptions) + if !ok { + return nil, fmt.Errorf("%w: attempted to initialize cumulative sum from invalid function options", arrow.ErrInvalid) + } + } + + start, err := cumulativeStartValue[T](opts.Start, args.Inputs[0]) + if err != nil { + return nil, err + } + + return &cumulativeSumState[T]{ + current: start, + skipNulls: opts.SkipNulls, + checked: checked, + }, nil + } +} + +func checkedAdd[T arrow.NumericType](left, right T) (T, error) { + switch l := any(left).(type) { + case int8: + r := any(right).(int8) + if (r > 0 && l > int8(127)-r) || (r < 0 && l < int8(-128)-r) { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case int16: + r := any(right).(int16) + if (r > 0 && l > int16(32767)-r) || (r < 0 && l < int16(-32768)-r) { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case int32: + r := any(right).(int32) + if (r > 0 && l > int32(2147483647)-r) || (r < 0 && l < int32(-2147483648)-r) { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case int64: + r := any(right).(int64) + if (r > 0 && l > int64(9223372036854775807)-r) || (r < 0 && l < int64(-9223372036854775807-1)-r) { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case uint8: + r := any(right).(uint8) + if l > ^uint8(0)-r { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case uint16: + r := any(right).(uint16) + if l > ^uint16(0)-r { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case uint32: + r := any(right).(uint32) + if l > ^uint32(0)-r { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case uint64: + r := any(right).(uint64) + if l > ^uint64(0)-r { + return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + } + return T(l + r), nil + case float32: + return T(l + any(right).(float32)), nil + case float64: + return T(l + any(right).(float64)), nil + default: + panic("unsupported cumulative sum type") + } +} + +func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.ExecSpan, out *exec.ExecResult) error { + state := ctx.State.(*cumulativeSumState[T]) + input := &batch.Values[0].Array + + out.Len = input.Len + if input.Len == 0 { + return nil + } + + data := ctx.Allocate(int(input.Len) * arrow.GetDataType[T]().(arrow.FixedWidthDataType).Bytes()) + out.Buffers[1].WrapBuffer(data) + values := exec.GetSpanValues[T](out, 1) + + needsValidity := state.encounteredNull || input.MayHaveNulls() + if needsValidity { + validity := ctx.AllocateBitmap(input.Len) + validityBytes := validity.Bytes() + for i := range validityBytes { + validityBytes[i] = 0xFF + } + out.Buffers[0].WrapBuffer(validity) + } + + var nulls int64 + for i := int64(0); i < input.Len; i++ { + valid := len(input.Buffers[0].Buf) == 0 || bitutil.BitIsSet(input.Buffers[0].Buf, int(input.Offset+i)) + if !valid || state.encounteredNull { + if needsValidity { + bitutil.ClearBit(out.Buffers[0].Buf, int(i)) + } + nulls++ + if !valid && !state.skipNulls { + state.encounteredNull = true + } + continue + } + + current := state.current + value := exec.GetSpanValues[T](input, 1)[i] + var err error + if state.checked { + current, err = checkedAdd(current, value) + } else { + current += value + } + if err != nil { + out.Release() + return err + } + + state.current = current + values[i] = current + } + + out.Nulls = nulls + return nil +} + +func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked bool) exec.VectorKernel { + return exec.NewVectorKernel( + []exec.InputType{exec.NewExactInput(typ)}, + exec.NewOutputType(typ), + cumulativeSumExec[T], + initCumulativeSum[T](checked)) +} + +func cumulativeSumKernels(checked bool) []exec.VectorKernel { + return []exec.VectorKernel{ + newCumulativeSumKernel[int8](arrow.PrimitiveTypes.Int8, checked), + newCumulativeSumKernel[int16](arrow.PrimitiveTypes.Int16, checked), + newCumulativeSumKernel[int32](arrow.PrimitiveTypes.Int32, checked), + newCumulativeSumKernel[int64](arrow.PrimitiveTypes.Int64, checked), + newCumulativeSumKernel[uint8](arrow.PrimitiveTypes.Uint8, checked), + newCumulativeSumKernel[uint16](arrow.PrimitiveTypes.Uint16, checked), + newCumulativeSumKernel[uint32](arrow.PrimitiveTypes.Uint32, checked), + newCumulativeSumKernel[uint64](arrow.PrimitiveTypes.Uint64, checked), + newCumulativeSumKernel[float32](arrow.PrimitiveTypes.Float32, checked), + newCumulativeSumKernel[float64](arrow.PrimitiveTypes.Float64, checked), + } +} + +func GetVectorCumulativeKernels() (sum, checked []exec.VectorKernel) { + return cumulativeSumKernels(false), cumulativeSumKernels(true) +} diff --git a/arrow/compute/registry.go b/arrow/compute/registry.go index f1be3b910..bea370251 100644 --- a/arrow/compute/registry.go +++ b/arrow/compute/registry.go @@ -54,6 +54,7 @@ func GetFunctionRegistry() FunctionRegistry { RegisterScalarArithmetic(registry) RegisterScalarComparisons(registry) RegisterVectorHash(registry) + RegisterVectorCumulative(registry) RegisterVectorRunEndFuncs(registry) RegisterScalarSetLookup(registry) }) diff --git a/arrow/compute/vector_cumulative.go b/arrow/compute/vector_cumulative.go new file mode 100644 index 000000000..6ef45b626 --- /dev/null +++ b/arrow/compute/vector_cumulative.go @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 + +package compute + +import ( + "context" + + "github.com/apache/arrow-go/v18/arrow/compute/internal/kernels" +) + +var cumulativeSumDoc = FunctionDoc{ + Summary: "Compute the cumulative sum of an array", + Description: "Return the cumulative sum of the input array", + ArgNames: []string{"array"}, + OptionsType: "CumulativeOptions", +} + +type CumulativeOptions = kernels.CumulativeOptions + +func RegisterVectorCumulative(reg FunctionRegistry) { + sum, checked := kernels.GetVectorCumulativeKernels() + + sumFn := NewVectorFunction("cumulative_sum", Unary(), cumulativeSumDoc) + sumFn.SetDefaultOptions(&CumulativeOptions{}) + for _, k := range sum { + if err := sumFn.AddKernel(k); err != nil { + panic(err) + } + } + reg.AddFunction(sumFn, false) + + checkedFn := NewVectorFunction("cumulative_sum_checked", Unary(), cumulativeSumDoc) + checkedFn.SetDefaultOptions(&CumulativeOptions{}) + for _, k := range checked { + if err := checkedFn.AddKernel(k); err != nil { + panic(err) + } + } + reg.AddFunction(checkedFn, false) +} + +func CumulativeSum(ctx context.Context, opts CumulativeOptions, values Datum) (Datum, error) { + return CallFunction(ctx, "cumulative_sum", &opts, values) +} + +func CumulativeSumChecked(ctx context.Context, opts CumulativeOptions, values Datum) (Datum, error) { + return CallFunction(ctx, "cumulative_sum_checked", &opts, values) +} diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go new file mode 100644 index 000000000..c6ee72757 --- /dev/null +++ b/arrow/compute/vector_cumulative_test.go @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 + +package compute_test + +import ( + "context" + "strings" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/arrow/scalar" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func cumulativeInput(t *testing.T, mem memory.Allocator, typ arrow.DataType, values string) arrow.Array { + arr, _, err := array.FromJSON(mem, typ, strings.NewReader(values)) + require.NoError(t, err) + return arr +} + +func TestCumulativeSum(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 2, 3, 4]`) + defer input.Release() + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) + defer expected.Release() + + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + +} + +func TestCumulativeSumNullsAndStart(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, 2, null, 3]`) + defer input.Release() + + t.Run("propagate nulls", func(t *testing.T) { + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, null, null, null]`) + defer expected.Release() + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + + t.Run("skip nulls", func(t *testing.T) { + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, 3, null, 6]`) + defer expected.Release() + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{SkipNulls: true}, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + + t.Run("start value", func(t *testing.T) { + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[11, null, 13, null, 16]`) + defer expected.Release() + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + Start: scalar.NewInt64Scalar(10), + SkipNulls: true, + }, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + +} + +func TestCumulativeSumChunked(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + first := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 2]`) + second := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[3, 4]`) + input := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{first, second}) + defer input.Release() + defer first.Release() + defer second.Release() + + expectedFirst := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3]`) + expectedSecond := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[6, 10]`) + expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedFirst, expectedSecond}) + defer expected.Release() + defer expectedFirst.Release() + defer expectedSecond.Release() + + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) + +} + +func TestCumulativeSumChecked(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[127, 1]`) + defer input.Release() + + unchecked, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer unchecked.Release() + uncheckedArray := unchecked.(*compute.ArrayDatum).MakeArray() + defer uncheckedArray.Release() + assert.Equal(t, int8(-128), uncheckedArray.(*array.Int8).Value(1)) + + _, err = compute.CumulativeSumChecked(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + assert.ErrorIs(t, err, arrow.ErrInvalid) + +} From 6094144bd688b2a72531800bfbfbdb295108d38c Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 00:19:38 +0200 Subject: [PATCH 02/17] fix(compute): validate cumulative sum start values --- .../internal/kernels/vector_cumulative.go | 50 +++++++++++- arrow/compute/vector_cumulative.go | 22 +++-- arrow/compute/vector_cumulative_test.go | 80 +++++++++++++++++++ 3 files changed, 143 insertions(+), 9 deletions(-) diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index a0dde8386..f9c337875 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -46,6 +46,52 @@ type cumulativeSumState[T arrow.NumericType] struct { checked bool } +func safeNumericCastScalar(start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { + sourceID := start.DataType().ID() + targetID := typ.ID() + if !arrow.IsInteger(sourceID) && !arrow.IsFloating(sourceID) { + return nil, fmt.Errorf("%w: cumulative sum start value must be numeric, got %s", arrow.ErrType, start.DataType()) + } + if !arrow.IsInteger(targetID) && !arrow.IsFloating(targetID) { + return nil, fmt.Errorf("%w: cumulative sum input type must be numeric, got %s", arrow.ErrType, typ) + } + if sourceID == targetID { + return start, nil + } + + casted, err := start.CastTo(typ) + if err != nil { + return nil, fmt.Errorf("%w: cannot cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, err) + } + + // Floating-point to floating-point casts follow the existing compute cast + // behavior, which does not reject precision loss. The other numeric casts + // use the same safe checks as the compute cast kernels. + if arrow.IsFloating(sourceID) && arrow.IsFloating(targetID) { + return casted, nil + } + + var sourceSpan exec.ArraySpan + sourceSpan.FillFromScalar(start) + var checkErr error + switch { + case arrow.IsInteger(sourceID) && arrow.IsInteger(targetID): + checkErr = intsCanFit(&sourceSpan, targetID) + case arrow.IsInteger(sourceID) && arrow.IsFloating(targetID): + checkErr = checkIntToFloatTrunc(&sourceSpan, targetID) + case arrow.IsFloating(sourceID) && arrow.IsInteger(targetID): + var roundTrip scalar.Scalar + roundTrip, checkErr = casted.CastTo(start.DataType()) + if checkErr == nil && !scalar.Equals(start, roundTrip) { + checkErr = fmt.Errorf("%w: float value %s was truncated converting to %s", arrow.ErrInvalid, start, typ) + } + } + if checkErr != nil { + return nil, fmt.Errorf("%w: cannot safely cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, checkErr) + } + return casted, nil +} + func cumulativeStartValue[T arrow.NumericType](start scalar.Scalar, typ arrow.DataType) (T, error) { var zero T if start == nil { @@ -55,9 +101,9 @@ func cumulativeStartValue[T arrow.NumericType](start scalar.Scalar, typ arrow.Da return zero, fmt.Errorf("%w: cumulative sum start value must be valid", arrow.ErrInvalid) } - casted, err := start.CastTo(typ) + casted, err := safeNumericCastScalar(start, typ) if err != nil { - return zero, fmt.Errorf("%w: cannot cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, err) + return zero, err } if releasable, ok := casted.(scalar.Releasable); ok { defer releasable.Release() diff --git a/arrow/compute/vector_cumulative.go b/arrow/compute/vector_cumulative.go index 6ef45b626..60243be5a 100644 --- a/arrow/compute/vector_cumulative.go +++ b/arrow/compute/vector_cumulative.go @@ -24,12 +24,20 @@ import ( "github.com/apache/arrow-go/v18/arrow/compute/internal/kernels" ) -var cumulativeSumDoc = FunctionDoc{ - Summary: "Compute the cumulative sum of an array", - Description: "Return the cumulative sum of the input array", - ArgNames: []string{"array"}, - OptionsType: "CumulativeOptions", -} +var ( + cumulativeSumDoc = FunctionDoc{ + Summary: "Compute the cumulative sum of an array", + Description: "Return the cumulative sum of the input array", + ArgNames: []string{"array"}, + OptionsType: "CumulativeOptions", + } + cumulativeSumCheckedDoc = FunctionDoc{ + Summary: "Compute the cumulative sum of an array with overflow checking", + Description: "Return the cumulative sum of the input array and report integer overflow", + ArgNames: []string{"array"}, + OptionsType: "CumulativeOptions", + } +) type CumulativeOptions = kernels.CumulativeOptions @@ -45,7 +53,7 @@ func RegisterVectorCumulative(reg FunctionRegistry) { } reg.AddFunction(sumFn, false) - checkedFn := NewVectorFunction("cumulative_sum_checked", Unary(), cumulativeSumDoc) + checkedFn := NewVectorFunction("cumulative_sum_checked", Unary(), cumulativeSumCheckedDoc) checkedFn.SetDefaultOptions(&CumulativeOptions{}) for _, k := range checked { if err := checkedFn.AddKernel(k); err != nil { diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index c6ee72757..a9617636b 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -91,6 +91,48 @@ func TestCumulativeSumNullsAndStart(t *testing.T) { } +func TestCumulativeSumStartSafeCast(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + tests := []struct { + name string + typ arrow.DataType + start scalar.Scalar + }{ + {name: "signed integer overflow", typ: arrow.PrimitiveTypes.Int8, start: scalar.NewInt64Scalar(128)}, + {name: "signed integer underflow", typ: arrow.PrimitiveTypes.Int8, start: scalar.NewInt64Scalar(-129)}, + {name: "unsigned integer underflow", typ: arrow.PrimitiveTypes.Uint8, start: scalar.NewInt64Scalar(-1)}, + {name: "float truncation", typ: arrow.PrimitiveTypes.Int32, start: scalar.NewFloat64Scalar(1.5)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + input := cumulativeInput(t, mem, tc.typ, `[0]`) + defer input.Release() + + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + Start: tc.start, + }, &compute.ArrayDatum{Value: input.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) + }) + } + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[0]`) + defer input.Release() + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + Start: scalar.NewInt64Scalar(127), + }, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + actual := result.(*compute.ArrayDatum).MakeArray() + defer actual.Release() + assert.Equal(t, int8(127), actual.(*array.Int8).Value(0)) +} + func TestCumulativeSumChunked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -132,3 +174,41 @@ func TestCumulativeSumChecked(t *testing.T) { assert.ErrorIs(t, err, arrow.ErrInvalid) } + +func TestCumulativeSumCheckedIntegerOverflow(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + tests := []struct { + name string + typ arrow.DataType + values string + }{ + {name: "int8 positive", typ: arrow.PrimitiveTypes.Int8, values: `[127, 1]`}, + {name: "int8 negative", typ: arrow.PrimitiveTypes.Int8, values: `[-128, -1]`}, + {name: "int16 positive", typ: arrow.PrimitiveTypes.Int16, values: `[32767, 1]`}, + {name: "int16 negative", typ: arrow.PrimitiveTypes.Int16, values: `[-32768, -1]`}, + {name: "int32 positive", typ: arrow.PrimitiveTypes.Int32, values: `[2147483647, 1]`}, + {name: "int32 negative", typ: arrow.PrimitiveTypes.Int32, values: `[-2147483648, -1]`}, + {name: "int64 positive", typ: arrow.PrimitiveTypes.Int64, values: `[9223372036854775807, 1]`}, + {name: "int64 negative", typ: arrow.PrimitiveTypes.Int64, values: `[-9223372036854775808, -1]`}, + {name: "uint8", typ: arrow.PrimitiveTypes.Uint8, values: `[255, 1]`}, + {name: "uint16", typ: arrow.PrimitiveTypes.Uint16, values: `[65535, 1]`}, + {name: "uint32", typ: arrow.PrimitiveTypes.Uint32, values: `[4294967295, 1]`}, + {name: "uint64", typ: arrow.PrimitiveTypes.Uint64, values: `[18446744073709551615, 1]`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + input := cumulativeInput(t, mem, tc.typ, tc.values) + defer input.Release() + + result, err := compute.CumulativeSumChecked(context.Background(), compute.CumulativeOptions{}, + &compute.ArrayDatum{Value: input.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) + }) + } +} From 99fccb83787c713645a7393928f8fb734ae2fc94 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 00:53:27 +0200 Subject: [PATCH 03/17] fix(compute): harden cumulative sum semantics --- .../internal/kernels/vector_cumulative.go | 98 +++++----- arrow/compute/vector_cumulative.go | 11 +- arrow/compute/vector_cumulative_test.go | 176 ++++++++++++++++++ 3 files changed, 237 insertions(+), 48 deletions(-) diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index f9c337875..f71401a31 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -44,14 +44,12 @@ type cumulativeSumState[T arrow.NumericType] struct { skipNulls bool encounteredNull bool checked bool + add func(T, T) (T, error) } func safeNumericCastScalar(start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { sourceID := start.DataType().ID() targetID := typ.ID() - if !arrow.IsInteger(sourceID) && !arrow.IsFloating(sourceID) { - return nil, fmt.Errorf("%w: cumulative sum start value must be numeric, got %s", arrow.ErrType, start.DataType()) - } if !arrow.IsInteger(targetID) && !arrow.IsFloating(targetID) { return nil, fmt.Errorf("%w: cumulative sum input type must be numeric, got %s", arrow.ErrType, typ) } @@ -70,6 +68,13 @@ func safeNumericCastScalar(start scalar.Scalar, typ arrow.DataType) (scalar.Scal if arrow.IsFloating(sourceID) && arrow.IsFloating(targetID) { return casted, nil } + if !arrow.IsInteger(sourceID) && !arrow.IsFloating(sourceID) { + // Non-numeric scalar types such as strings and booleans validate their + // conversion while producing the target scalar. Keep that behavior + // aligned with scalar safe-cast dispatch instead of rejecting them based + // only on their source type. + return casted, nil + } var sourceSpan exec.ArraySpan sourceSpan.FillFromScalar(start) @@ -139,66 +144,70 @@ func initCumulativeSum[T arrow.NumericType](checked bool) exec.KernelInitFn { current: start, skipNulls: opts.SkipNulls, checked: checked, + add: checkedAdder[T](), }, nil } } -func checkedAdd[T arrow.NumericType](left, right T) (T, error) { - switch l := any(left).(type) { +func checkedAddSigned[T arrow.IntType](left, right T) (T, error) { + if (right > 0 && left > MaxOf[T]()-right) || (right < 0 && left < MinOf[T]()-right) { + return 0, errOverflow + } + return left + right, nil +} + +func checkedAddUnsigned[T arrow.UintType](left, right T) (T, error) { + if left > MaxOf[T]()-right { + return 0, errOverflow + } + return left + right, nil +} + +func checkedAdder[T arrow.NumericType]() func(T, T) (T, error) { + var zero T + switch any(zero).(type) { case int8: - r := any(right).(int8) - if (r > 0 && l > int8(127)-r) || (r < 0 && l < int8(-128)-r) { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddSigned(int8(left), int8(right)) + return T(value), err } - return T(l + r), nil case int16: - r := any(right).(int16) - if (r > 0 && l > int16(32767)-r) || (r < 0 && l < int16(-32768)-r) { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddSigned(int16(left), int16(right)) + return T(value), err } - return T(l + r), nil case int32: - r := any(right).(int32) - if (r > 0 && l > int32(2147483647)-r) || (r < 0 && l < int32(-2147483648)-r) { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddSigned(int32(left), int32(right)) + return T(value), err } - return T(l + r), nil case int64: - r := any(right).(int64) - if (r > 0 && l > int64(9223372036854775807)-r) || (r < 0 && l < int64(-9223372036854775807-1)-r) { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddSigned(int64(left), int64(right)) + return T(value), err } - return T(l + r), nil case uint8: - r := any(right).(uint8) - if l > ^uint8(0)-r { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddUnsigned(uint8(left), uint8(right)) + return T(value), err } - return T(l + r), nil case uint16: - r := any(right).(uint16) - if l > ^uint16(0)-r { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddUnsigned(uint16(left), uint16(right)) + return T(value), err } - return T(l + r), nil case uint32: - r := any(right).(uint32) - if l > ^uint32(0)-r { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddUnsigned(uint32(left), uint32(right)) + return T(value), err } - return T(l + r), nil case uint64: - r := any(right).(uint64) - if l > ^uint64(0)-r { - return 0, fmt.Errorf("%w: cumulative sum overflow", arrow.ErrInvalid) + return func(left, right T) (T, error) { + value, err := checkedAddUnsigned(uint64(left), uint64(right)) + return T(value), err } - return T(l + r), nil - case float32: - return T(l + any(right).(float32)), nil - case float64: - return T(l + any(right).(float64)), nil default: - panic("unsupported cumulative sum type") + return func(left, right T) (T, error) { return left + right, nil } } } @@ -214,6 +223,7 @@ func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.Exe data := ctx.Allocate(int(input.Len) * arrow.GetDataType[T]().(arrow.FixedWidthDataType).Bytes()) out.Buffers[1].WrapBuffer(data) values := exec.GetSpanValues[T](out, 1) + inputValues := exec.GetSpanValues[T](input, 1) needsValidity := state.encounteredNull || input.MayHaveNulls() if needsValidity { @@ -240,10 +250,10 @@ func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.Exe } current := state.current - value := exec.GetSpanValues[T](input, 1)[i] + value := inputValues[i] var err error if state.checked { - current, err = checkedAdd(current, value) + current, err = state.add(current, value) } else { current += value } diff --git a/arrow/compute/vector_cumulative.go b/arrow/compute/vector_cumulative.go index 60243be5a..895fa896f 100644 --- a/arrow/compute/vector_cumulative.go +++ b/arrow/compute/vector_cumulative.go @@ -26,14 +26,17 @@ import ( var ( cumulativeSumDoc = FunctionDoc{ - Summary: "Compute the cumulative sum of an array", - Description: "Return the cumulative sum of the input array", + Summary: "Compute the cumulative sum of an array", + Description: `Return the cumulative sum of the input array. Integer values +wrap on overflow; nulls stop the remaining output unless SkipNulls is enabled. +A nil Start uses zero.`, ArgNames: []string{"array"}, OptionsType: "CumulativeOptions", } cumulativeSumCheckedDoc = FunctionDoc{ - Summary: "Compute the cumulative sum of an array with overflow checking", - Description: "Return the cumulative sum of the input array and report integer overflow", + Summary: "Compute the cumulative sum of an array with overflow checking", + Description: `Return the cumulative sum of the input array and report +integer overflow. Null handling and Start follow CumulativeOptions.`, ArgNames: []string{"array"}, OptionsType: "CumulativeOptions", } diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index a9617636b..e27e21d3c 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -53,6 +53,56 @@ func TestCumulativeSum(t *testing.T) { } +func TestCumulativeSumAdditionalInputs(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + tests := []struct { + name string + typ arrow.DataType + in string + want string + }{ + {name: "empty", typ: arrow.PrimitiveTypes.Int32, in: `[]`, want: `[]`}, + {name: "all null", typ: arrow.PrimitiveTypes.Int32, in: `[null, null]`, want: `[null, null]`}, + {name: "uint8", typ: arrow.PrimitiveTypes.Uint8, in: `[1, 2, 3]`, want: `[1, 3, 6]`}, + {name: "float32", typ: arrow.PrimitiveTypes.Float32, in: `[1.5, 2.5]`, want: `[1.5, 4]`}, + {name: "float64", typ: arrow.PrimitiveTypes.Float64, in: `[1.5, 2.5]`, want: `[1.5, 4]`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + input := cumulativeInput(t, mem, tc.typ, tc.in) + defer input.Release() + expected := cumulativeInput(t, mem, tc.typ, tc.want) + defer expected.Release() + + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + } + + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[3]`) + defer expected.Release() + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, compute.NewDatum(int32(3))) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + + fullInput := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[0, 1, 2, 3]`) + defer fullInput.Release() + slicedInput := array.NewSlice(fullInput, 1, 3) + defer slicedInput.Release() + slicedExpected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3]`) + defer slicedExpected.Release() + result, err = compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: slicedInput.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: slicedExpected.Data()}, result, nil, nil) +} + func TestCumulativeSumNullsAndStart(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -133,6 +183,45 @@ func TestCumulativeSumStartSafeCast(t *testing.T) { assert.Equal(t, int8(127), actual.(*array.Int8).Value(0)) } +func TestCumulativeSumStartScalarConversions(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1]`) + defer input.Release() + + tests := []struct { + name string + start scalar.Scalar + want string + }{ + {name: "string", start: scalar.NewStringScalar("10"), want: `[11]`}, + {name: "boolean", start: scalar.NewBooleanScalar(true), want: `[2]`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.want) + defer expected.Release() + + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + Start: tc.start, + }, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + } + + result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + Start: scalar.NewStringScalar("not a number"), + }, &compute.ArrayDatum{Value: input.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) +} + func TestCumulativeSumChunked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -157,6 +246,93 @@ func TestCumulativeSumChunked(t *testing.T) { } +func TestCumulativeSumStateAcrossChunks(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + first := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null]`) + second := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[2, 3]`) + input := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{first, second}) + defer input.Release() + defer first.Release() + defer second.Release() + + for _, tc := range []struct { + name string + opts compute.CumulativeOptions + expected []string + }{ + {name: "propagate nulls", expected: []string{`[1, null]`, `[null, null]`}}, + {name: "skip nulls", opts: compute.CumulativeOptions{SkipNulls: true}, expected: []string{`[1, null]`, `[3, 6]`}}, + } { + t.Run(tc.name, func(t *testing.T) { + expectedFirst := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected[0]) + expectedSecond := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected[1]) + expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedFirst, expectedSecond}) + defer expected.Release() + defer expectedFirst.Release() + defer expectedSecond.Release() + + result, err := compute.CumulativeSum(context.Background(), tc.opts, &compute.ChunkedDatum{Value: input}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) + }) + } +} + +func TestCumulativeSumStateAcrossExecutorSpans(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, 2, 3]`) + defer input.Release() + + execCtx := compute.DefaultExecCtx() + execCtx.ChunkSize = 1 + ctx := compute.SetExecCtx(context.Background(), execCtx) + + tests := []struct { + name string + opts compute.CumulativeOptions + expected string + }{ + {name: "propagate nulls", expected: `[1, null, null, null]`}, + {name: "skip nulls", opts: compute.CumulativeOptions{SkipNulls: true}, expected: `[1, null, 3, 6]`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + expectedArray := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected) + defer expectedArray.Release() + expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedArray}) + defer expected.Release() + + result, err := compute.CumulativeSum(ctx, tc.opts, &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) + }) + } + + overflowInput := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[127, 1]`) + defer overflowInput.Release() + result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: overflowInput.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) + + startOverflowInput := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[1]`) + defer startOverflowInput.Release() + result, err = compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{ + Start: scalar.NewInt8Scalar(127), + }, &compute.ArrayDatum{Value: startOverflowInput.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) +} + func TestCumulativeSumChecked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) From 0917e19f33b80d5bd455555161872227d4f6d2bb Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 01:26:55 +0200 Subject: [PATCH 04/17] fix(compute): align cumulative sum option and cast semantics --- arrow/compute/expression.go | 1 + .../internal/kernels/vector_cumulative.go | 117 +++++++++++------- arrow/compute/vector_cumulative_test.go | 84 ++++++++++++- arrow/scalar/parse.go | 30 +++++ 4 files changed, 186 insertions(+), 46 deletions(-) diff --git a/arrow/compute/expression.go b/arrow/compute/expression.go index 2f1dc482e..af919eaa5 100644 --- a/arrow/compute/expression.go +++ b/arrow/compute/expression.go @@ -533,6 +533,7 @@ var ( funcOptsTypes = []FunctionOptions{ SetLookupOptions{}, ArithmeticOptions{}, CastOptions{}, FilterOptions{}, NullOptions{}, StrptimeOptions{}, MakeStructOptions{}, + CumulativeOptions{}, } ) diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index f71401a31..b0734e476 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -31,7 +31,7 @@ import ( type CumulativeOptions struct { // Start is the initial value. A nil value uses the zero value for the // input type. - Start scalar.Scalar + Start scalar.Scalar `compute:"start"` // SkipNulls controls whether a null input stops the cumulative operation. // Null positions are still null in the output when this is true. SkipNulls bool `compute:"skip_nulls"` @@ -47,57 +47,86 @@ type cumulativeSumState[T arrow.NumericType] struct { add func(T, T) (T, error) } -func safeNumericCastScalar(start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { - sourceID := start.DataType().ID() +func safeCastKernels(typ arrow.DataType) []exec.ScalarKernel { + switch typ.ID() { + case arrow.INT8: + return GetCastToInteger[int8](typ) + case arrow.INT16: + return GetCastToInteger[int16](typ) + case arrow.INT32: + return GetCastToInteger[int32](typ) + case arrow.INT64: + return GetCastToInteger[int64](typ) + case arrow.UINT8: + return GetCastToInteger[uint8](typ) + case arrow.UINT16: + return GetCastToInteger[uint16](typ) + case arrow.UINT32: + return GetCastToInteger[uint32](typ) + case arrow.UINT64: + return GetCastToInteger[uint64](typ) + case arrow.FLOAT32: + return GetCastToFloating[float32](typ) + case arrow.FLOAT64: + return GetCastToFloating[float64](typ) + default: + return nil + } +} + +func safeCastScalar(ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { + kernels := safeCastKernels(typ) + var castKernel *exec.ScalarKernel + for i := range kernels { + if kernels[i].GetSig().MatchesInputs([]arrow.DataType{start.DataType()}) { + castKernel = &kernels[i] + break + } + } + if castKernel == nil { + return nil, fmt.Errorf("%w: cannot safely cast cumulative sum start value from %s to %s", + arrow.ErrInvalid, start.DataType(), typ) + } + + input := exec.ArraySpan{} + input.FillFromScalar(start) + output := exec.ArraySpan{Type: typ, Len: 1} + output.Buffers[1].WrapBuffer(ctx.Allocate(typ.(arrow.FixedWidthDataType).Bytes())) + + castCtx := *ctx + castCtx.Kernel = castKernel + castCtx.State = CastOptions{ToType: typ} + batch := &exec.ExecSpan{ + Len: 1, + Values: []exec.ExecValue{{Array: input}}, + } + if err := castKernel.Exec(&castCtx, batch, &output); err != nil { + output.Release() + return nil, err + } + + arr := output.MakeArray() + defer arr.Release() + return scalar.GetScalar(arr, 0) +} + +func safeNumericCastScalar(ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { targetID := typ.ID() if !arrow.IsInteger(targetID) && !arrow.IsFloating(targetID) { return nil, fmt.Errorf("%w: cumulative sum input type must be numeric, got %s", arrow.ErrType, typ) } - if sourceID == targetID { + if arrow.TypeEqual(start.DataType(), typ) { return start, nil } - casted, err := start.CastTo(typ) + casted, err := safeCastScalar(ctx, start, typ) if err != nil { return nil, fmt.Errorf("%w: cannot cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, err) } - - // Floating-point to floating-point casts follow the existing compute cast - // behavior, which does not reject precision loss. The other numeric casts - // use the same safe checks as the compute cast kernels. - if arrow.IsFloating(sourceID) && arrow.IsFloating(targetID) { - return casted, nil - } - if !arrow.IsInteger(sourceID) && !arrow.IsFloating(sourceID) { - // Non-numeric scalar types such as strings and booleans validate their - // conversion while producing the target scalar. Keep that behavior - // aligned with scalar safe-cast dispatch instead of rejecting them based - // only on their source type. - return casted, nil - } - - var sourceSpan exec.ArraySpan - sourceSpan.FillFromScalar(start) - var checkErr error - switch { - case arrow.IsInteger(sourceID) && arrow.IsInteger(targetID): - checkErr = intsCanFit(&sourceSpan, targetID) - case arrow.IsInteger(sourceID) && arrow.IsFloating(targetID): - checkErr = checkIntToFloatTrunc(&sourceSpan, targetID) - case arrow.IsFloating(sourceID) && arrow.IsInteger(targetID): - var roundTrip scalar.Scalar - roundTrip, checkErr = casted.CastTo(start.DataType()) - if checkErr == nil && !scalar.Equals(start, roundTrip) { - checkErr = fmt.Errorf("%w: float value %s was truncated converting to %s", arrow.ErrInvalid, start, typ) - } - } - if checkErr != nil { - return nil, fmt.Errorf("%w: cannot safely cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, checkErr) - } return casted, nil } -func cumulativeStartValue[T arrow.NumericType](start scalar.Scalar, typ arrow.DataType) (T, error) { +func cumulativeStartValue[T arrow.NumericType](ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (T, error) { var zero T if start == nil { return zero, nil @@ -106,7 +135,7 @@ func cumulativeStartValue[T arrow.NumericType](start scalar.Scalar, typ arrow.Da return zero, fmt.Errorf("%w: cumulative sum start value must be valid", arrow.ErrInvalid) } - casted, err := safeNumericCastScalar(start, typ) + casted, err := safeNumericCastScalar(ctx, start, typ) if err != nil { return zero, err } @@ -125,7 +154,7 @@ func cumulativeStartValue[T arrow.NumericType](start scalar.Scalar, typ arrow.Da } func initCumulativeSum[T arrow.NumericType](checked bool) exec.KernelInitFn { - return func(_ *exec.KernelCtx, args exec.KernelInitArgs) (exec.KernelState, error) { + return func(ctx *exec.KernelCtx, args exec.KernelInitArgs) (exec.KernelState, error) { opts := &CumulativeOptions{} if args.Options != nil { var ok bool @@ -135,7 +164,7 @@ func initCumulativeSum[T arrow.NumericType](checked bool) exec.KernelInitFn { } } - start, err := cumulativeStartValue[T](opts.Start, args.Inputs[0]) + start, err := cumulativeStartValue[T](ctx, opts.Start, args.Inputs[0]) if err != nil { return nil, err } @@ -271,11 +300,13 @@ func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.Exe } func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked bool) exec.VectorKernel { - return exec.NewVectorKernel( + kernel := exec.NewVectorKernel( []exec.InputType{exec.NewExactInput(typ)}, exec.NewOutputType(typ), cumulativeSumExec[T], initCumulativeSum[T](checked)) + kernel.Parallelizable = false + return kernel } func cumulativeSumKernels(checked bool) []exec.VectorKernel { diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index e27e21d3c..9f09fc30e 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -186,6 +186,7 @@ func TestCumulativeSumStartSafeCast(t *testing.T) { func TestCumulativeSumStartScalarConversions(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1]`) defer input.Release() @@ -198,13 +199,25 @@ func TestCumulativeSumStartScalarConversions(t *testing.T) { {name: "string", start: scalar.NewStringScalar("10"), want: `[11]`}, {name: "boolean", start: scalar.NewBooleanScalar(true), want: `[2]`}, } + binaryBuffer := memory.NewBufferBytes([]byte("10")) + tests = append(tests, struct { + name string + start scalar.Scalar + want string + }{name: "binary", start: scalar.NewBinaryScalar(binaryBuffer, arrow.BinaryTypes.Binary), want: `[11]`}) + binaryBuffer.Release() + for _, tc := range tests { + if releasable, ok := tc.start.(scalar.Releasable); ok { + defer releasable.Release() + } + } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.want) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{ Start: tc.start, }, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) @@ -213,8 +226,10 @@ func TestCumulativeSumStartScalarConversions(t *testing.T) { }) } - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ - Start: scalar.NewStringScalar("not a number"), + invalidStart := scalar.NewStringScalar("not a number") + defer invalidStart.Release() + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{ + Start: invalidStart, }, &compute.ArrayDatum{Value: input.Data()}) if result != nil { result.Release() @@ -222,6 +237,69 @@ func TestCumulativeSumStartScalarConversions(t *testing.T) { assert.ErrorIs(t, err, arrow.ErrInvalid) } +func TestCumulativeSumDecimalStarts(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + + decimalType := &arrow.Decimal128Type{Precision: 10, Scale: 0} + decimalStart, err := scalar.ParseScalar(decimalType, "10") + require.NoError(t, err) + + intInput := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1]`) + defer intInput.Release() + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{ + Start: decimalStart, + }, &compute.ArrayDatum{Value: intInput.Data()}) + require.NoError(t, err) + defer result.Release() + actual := result.(*compute.ArrayDatum).MakeArray() + defer actual.Release() + assert.Equal(t, int32(11), actual.(*array.Int32).Value(0)) + + floatInput := cumulativeInput(t, mem, arrow.PrimitiveTypes.Float64, `[1.5]`) + defer floatInput.Release() + result, err = compute.CumulativeSum(ctx, compute.CumulativeOptions{ + Start: decimalStart, + }, &compute.ArrayDatum{Value: floatInput.Data()}) + require.NoError(t, err) + defer result.Release() + actual = result.(*compute.ArrayDatum).MakeArray() + defer actual.Release() + assert.Equal(t, 11.5, actual.(*array.Float64).Value(0)) + + fractionalStart, err := scalar.ParseScalar(&arrow.Decimal128Type{Precision: 10, Scale: 1}, "1.5") + require.NoError(t, err) + result, err = compute.CumulativeSum(ctx, compute.CumulativeOptions{ + Start: fractionalStart, + }, &compute.ArrayDatum{Value: intInput.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) +} + +func TestCumulativeOptionsSerialization(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + expr := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, + &compute.CumulativeOptions{ + Start: scalar.NewInt32Scalar(10), + SkipNulls: true, + }) + defer expr.Release() + + serialized, err := compute.SerializeExpr(expr, mem) + require.NoError(t, err) + defer serialized.Release() + + roundTripped, err := compute.DeserializeExpr(mem, serialized) + require.NoError(t, err) + defer roundTripped.Release() + assert.True(t, expr.Equals(roundTripped)) +} + func TestCumulativeSumChunked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) diff --git a/arrow/scalar/parse.go b/arrow/scalar/parse.go index 6d1e6a33c..e77289046 100644 --- a/arrow/scalar/parse.go +++ b/arrow/scalar/parse.go @@ -48,6 +48,7 @@ type hasTypename interface { var ( hasTypenameType = reflect.TypeOf((*hasTypename)(nil)).Elem() dataTypeType = reflect.TypeOf((*arrow.DataType)(nil)).Elem() + scalarType = reflect.TypeOf((*Scalar)(nil)).Elem() ) func FromScalar(sc *Struct, val interface{}) error { @@ -85,6 +86,20 @@ func FromScalar(sc *Struct, val interface{}) error { } func setFromScalar(s Scalar, v reflect.Value) error { + if v.Type() == scalarType { + if !s.IsValid() && s.DataType().ID() == arrow.NULL { + v.Set(reflect.Zero(v.Type())) + return nil + } + + clone, err := cloneScalar(s, memory.DefaultAllocator) + if err != nil { + return err + } + v.Set(reflect.ValueOf(clone)) + return nil + } + if v.Type() == dataTypeType { v.Set(reflect.ValueOf(s.DataType())) return nil @@ -122,11 +137,17 @@ func setFromScalar(s Scalar, v reflect.Value) error { } func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) { + if val == nil { + return ScalarNull, nil + } + switch v := val.(type) { case arrow.DataType: return MakeScalar(v), nil case TypeToScalar: return v.ToScalar() + case Scalar: + return cloneScalar(v, mem) } v := reflect.Indirect(reflect.ValueOf(val)) @@ -164,6 +185,15 @@ func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) { } } +func cloneScalar(val Scalar, mem memory.Allocator) (Scalar, error) { + arr, err := MakeArrayFromScalar(val, 1, mem) + if err != nil { + return nil, err + } + defer arr.Release() + return GetScalar(arr, 0) +} + func createListScalar(sliceval reflect.Value, mem memory.Allocator) (Scalar, error) { if sliceval.Kind() != reflect.Slice { return nil, fmt.Errorf("createListScalar only works for slices, not %s", sliceval.Kind()) From c9b802b96282898b03b9a70176eea0149ac5db53 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 02:26:30 +0200 Subject: [PATCH 05/17] fix(compute): harden cumulative sum lifecycles --- arrow/compute/exec.go | 1 + arrow/compute/executor.go | 30 +++- arrow/compute/expression.go | 42 +++++- .../internal/kernels/vector_cumulative.go | 120 +++++----------- arrow/compute/vector_cumulative.go | 40 +++++- arrow/compute/vector_cumulative_test.go | 135 ++++++++++++++---- arrow/scalar/parse.go | 38 ++++- arrow/scalar/scalar.go | 6 +- arrow/scalar/scalar_test.go | 57 ++++++++ 9 files changed, 340 insertions(+), 129 deletions(-) diff --git a/arrow/compute/exec.go b/arrow/compute/exec.go index 0afb3b13d..51633a04f 100644 --- a/arrow/compute/exec.go +++ b/arrow/compute/exec.go @@ -176,6 +176,7 @@ func execInternal(ctx context.Context, fn Function, opts FunctionOptions, passed if ctx.Err() == context.Canceled && result != nil { result.Release() + result = nil } return diff --git a/arrow/compute/executor.go b/arrow/compute/executor.go index 095a7e15d..b164e2327 100644 --- a/arrow/compute/executor.go +++ b/arrow/compute/executor.go @@ -523,6 +523,16 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC output Datum acc []arrow.Array ) + releasePartial := func() { + if output != nil { + output.Release() + output = nil + } + for _, c := range acc { + c.Release() + } + acc = nil + } toChunked := func() { acc = output.(ArrayLikeDatum).Chunks() @@ -535,6 +545,9 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC case <-ctx.Done(): return nil case output = <-out: + if output == nil { + return nil + } // if the inputs contained at least one chunked array // then we want to return chunked output if hasChunked { @@ -547,7 +560,8 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC case <-ctx.Done(): // context is done, either cancelled or a timeout. // either way, we end early and return what we've got so far. - return output + releasePartial() + return nil case o, ok := <-out: if !ok { // channel closed, wrap it up if output != nil { @@ -1002,6 +1016,16 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC output Datum acc []arrow.Array ) + releasePartial := func() { + if output != nil { + output.Release() + output = nil + } + for _, c := range acc { + c.Release() + } + acc = nil + } toChunked := func() { out := output.(ArrayLikeDatum).Chunks() @@ -1023,6 +1047,7 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC return nil case output = <-out: if output == nil || ctx.Err() != nil { + releasePartial() return nil } @@ -1038,7 +1063,8 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC case <-ctx.Done(): // context is done, either cancelled or a timeout. // either way, we end early and return what we've got so far. - return output + releasePartial() + return nil case o, ok := <-out: if !ok { // channel closed, wrap it up if output != nil { diff --git a/arrow/compute/expression.go b/arrow/compute/expression.go index af919eaa5..3dc168690 100644 --- a/arrow/compute/expression.go +++ b/arrow/compute/expression.go @@ -41,6 +41,8 @@ import ( var hashSeed = maphash.MakeSeed() +var scalarInterfaceType = reflect.TypeOf((*scalar.Scalar)(nil)).Elem() + // Expression is an interface for mapping one datum to another. An expression // is one of: // @@ -377,7 +379,43 @@ func (c *Call) Equals(other Expression) bool { if opt, ok := c.options.(FunctionOptionsEqual); ok { return opt.Equals(rhs.options) } - return reflect.DeepEqual(c.options, rhs.options) + return equalFunctionOptions(c.options, rhs.options) +} + +func equalFunctionOptions(lhs, rhs FunctionOptions) bool { + if lhs == nil || rhs == nil { + return lhs == nil && rhs == nil + } + if reflect.TypeOf(lhs) != reflect.TypeOf(rhs) { + return false + } + + left := reflect.Indirect(reflect.ValueOf(lhs)) + right := reflect.Indirect(reflect.ValueOf(rhs)) + if !left.IsValid() || left.Kind() != reflect.Struct { + return reflect.DeepEqual(lhs, rhs) + } + + for i := 0; i < left.NumField(); i++ { + leftField := left.Field(i) + rightField := right.Field(i) + if leftField.Type() == scalarInterfaceType { + leftScalar, leftOK := leftField.Interface().(scalar.Scalar) + rightScalar, rightOK := rightField.Interface().(scalar.Scalar) + if leftOK != rightOK { + return false + } + if leftOK && !scalar.Equals(leftScalar, rightScalar) { + return false + } + continue + } + + if !reflect.DeepEqual(leftField.Interface(), rightField.Interface()) { + return false + } + } + return true } func (c *Call) Release() { @@ -881,7 +919,7 @@ func DeserializeExpr(mem memory.Allocator, buf *memory.Buffer) (Expression, erro } optionsVal := reflect.New(funcOptionsMap[string(typname.(*scalar.Binary).Data())]).Interface() - if err := scalar.FromScalar(optsScalar.(*scalar.Struct), optionsVal); err != nil { + if err := scalar.FromScalarWithAllocator(optsScalar.(*scalar.Struct), optionsVal, mem); err != nil { return nil, err } opts = optionsVal.(FunctionOptions) diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index b0734e476..44f922460 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -39,6 +39,17 @@ type CumulativeOptions struct { func (CumulativeOptions) TypeName() string { return "CumulativeOptions" } +func (opts *CumulativeOptions) Release() { + if opts.Start == nil { + return + } + + if releasable, ok := opts.Start.(interface{ Release() }); ok { + releasable.Release() + } + opts.Start = nil +} + type cumulativeSumState[T arrow.NumericType] struct { current T skipNulls bool @@ -47,70 +58,9 @@ type cumulativeSumState[T arrow.NumericType] struct { add func(T, T) (T, error) } -func safeCastKernels(typ arrow.DataType) []exec.ScalarKernel { - switch typ.ID() { - case arrow.INT8: - return GetCastToInteger[int8](typ) - case arrow.INT16: - return GetCastToInteger[int16](typ) - case arrow.INT32: - return GetCastToInteger[int32](typ) - case arrow.INT64: - return GetCastToInteger[int64](typ) - case arrow.UINT8: - return GetCastToInteger[uint8](typ) - case arrow.UINT16: - return GetCastToInteger[uint16](typ) - case arrow.UINT32: - return GetCastToInteger[uint32](typ) - case arrow.UINT64: - return GetCastToInteger[uint64](typ) - case arrow.FLOAT32: - return GetCastToFloating[float32](typ) - case arrow.FLOAT64: - return GetCastToFloating[float64](typ) - default: - return nil - } -} - -func safeCastScalar(ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { - kernels := safeCastKernels(typ) - var castKernel *exec.ScalarKernel - for i := range kernels { - if kernels[i].GetSig().MatchesInputs([]arrow.DataType{start.DataType()}) { - castKernel = &kernels[i] - break - } - } - if castKernel == nil { - return nil, fmt.Errorf("%w: cannot safely cast cumulative sum start value from %s to %s", - arrow.ErrInvalid, start.DataType(), typ) - } - - input := exec.ArraySpan{} - input.FillFromScalar(start) - output := exec.ArraySpan{Type: typ, Len: 1} - output.Buffers[1].WrapBuffer(ctx.Allocate(typ.(arrow.FixedWidthDataType).Bytes())) - - castCtx := *ctx - castCtx.Kernel = castKernel - castCtx.State = CastOptions{ToType: typ} - batch := &exec.ExecSpan{ - Len: 1, - Values: []exec.ExecValue{{Array: input}}, - } - if err := castKernel.Exec(&castCtx, batch, &output); err != nil { - output.Release() - return nil, err - } - - arr := output.MakeArray() - defer arr.Release() - return scalar.GetScalar(arr, 0) -} +type ScalarCastFn func(*exec.KernelCtx, scalar.Scalar, arrow.DataType) (scalar.Scalar, error) -func safeNumericCastScalar(ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { +func safeNumericCastScalar(ctx *exec.KernelCtx, cast ScalarCastFn, start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { targetID := typ.ID() if !arrow.IsInteger(targetID) && !arrow.IsFloating(targetID) { return nil, fmt.Errorf("%w: cumulative sum input type must be numeric, got %s", arrow.ErrType, typ) @@ -119,14 +69,18 @@ func safeNumericCastScalar(ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.D return start, nil } - casted, err := safeCastScalar(ctx, start, typ) + if cast == nil { + return nil, fmt.Errorf("%w: cumulative sum start value caster is not configured", arrow.ErrInvalid) + } + + casted, err := cast(ctx, start, typ) if err != nil { return nil, fmt.Errorf("%w: cannot cast cumulative sum start value to %s: %v", arrow.ErrInvalid, typ, err) } return casted, nil } -func cumulativeStartValue[T arrow.NumericType](ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (T, error) { +func cumulativeStartValue[T arrow.NumericType](ctx *exec.KernelCtx, cast ScalarCastFn, start scalar.Scalar, typ arrow.DataType) (T, error) { var zero T if start == nil { return zero, nil @@ -135,7 +89,7 @@ func cumulativeStartValue[T arrow.NumericType](ctx *exec.KernelCtx, start scalar return zero, fmt.Errorf("%w: cumulative sum start value must be valid", arrow.ErrInvalid) } - casted, err := safeNumericCastScalar(ctx, start, typ) + casted, err := safeNumericCastScalar(ctx, cast, start, typ) if err != nil { return zero, err } @@ -153,7 +107,7 @@ func cumulativeStartValue[T arrow.NumericType](ctx *exec.KernelCtx, start scalar return exec.GetSpanValues[T](span, 1)[0], nil } -func initCumulativeSum[T arrow.NumericType](checked bool) exec.KernelInitFn { +func initCumulativeSum[T arrow.NumericType](checked bool, cast ScalarCastFn) exec.KernelInitFn { return func(ctx *exec.KernelCtx, args exec.KernelInitArgs) (exec.KernelState, error) { opts := &CumulativeOptions{} if args.Options != nil { @@ -164,7 +118,7 @@ func initCumulativeSum[T arrow.NumericType](checked bool) exec.KernelInitFn { } } - start, err := cumulativeStartValue[T](ctx, opts.Start, args.Inputs[0]) + start, err := cumulativeStartValue[T](ctx, cast, opts.Start, args.Inputs[0]) if err != nil { return nil, err } @@ -299,31 +253,31 @@ func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.Exe return nil } -func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked bool) exec.VectorKernel { +func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked bool, cast ScalarCastFn) exec.VectorKernel { kernel := exec.NewVectorKernel( []exec.InputType{exec.NewExactInput(typ)}, exec.NewOutputType(typ), cumulativeSumExec[T], - initCumulativeSum[T](checked)) + initCumulativeSum[T](checked, cast)) kernel.Parallelizable = false return kernel } -func cumulativeSumKernels(checked bool) []exec.VectorKernel { +func cumulativeSumKernels(checked bool, cast ScalarCastFn) []exec.VectorKernel { return []exec.VectorKernel{ - newCumulativeSumKernel[int8](arrow.PrimitiveTypes.Int8, checked), - newCumulativeSumKernel[int16](arrow.PrimitiveTypes.Int16, checked), - newCumulativeSumKernel[int32](arrow.PrimitiveTypes.Int32, checked), - newCumulativeSumKernel[int64](arrow.PrimitiveTypes.Int64, checked), - newCumulativeSumKernel[uint8](arrow.PrimitiveTypes.Uint8, checked), - newCumulativeSumKernel[uint16](arrow.PrimitiveTypes.Uint16, checked), - newCumulativeSumKernel[uint32](arrow.PrimitiveTypes.Uint32, checked), - newCumulativeSumKernel[uint64](arrow.PrimitiveTypes.Uint64, checked), - newCumulativeSumKernel[float32](arrow.PrimitiveTypes.Float32, checked), - newCumulativeSumKernel[float64](arrow.PrimitiveTypes.Float64, checked), + newCumulativeSumKernel[int8](arrow.PrimitiveTypes.Int8, checked, cast), + newCumulativeSumKernel[int16](arrow.PrimitiveTypes.Int16, checked, cast), + newCumulativeSumKernel[int32](arrow.PrimitiveTypes.Int32, checked, cast), + newCumulativeSumKernel[int64](arrow.PrimitiveTypes.Int64, checked, cast), + newCumulativeSumKernel[uint8](arrow.PrimitiveTypes.Uint8, checked, cast), + newCumulativeSumKernel[uint16](arrow.PrimitiveTypes.Uint16, checked, cast), + newCumulativeSumKernel[uint32](arrow.PrimitiveTypes.Uint32, checked, cast), + newCumulativeSumKernel[uint64](arrow.PrimitiveTypes.Uint64, checked, cast), + newCumulativeSumKernel[float32](arrow.PrimitiveTypes.Float32, checked, cast), + newCumulativeSumKernel[float64](arrow.PrimitiveTypes.Float64, checked, cast), } } -func GetVectorCumulativeKernels() (sum, checked []exec.VectorKernel) { - return cumulativeSumKernels(false), cumulativeSumKernels(true) +func GetVectorCumulativeKernels(cast ScalarCastFn) (sum, checked []exec.VectorKernel) { + return cumulativeSumKernels(false, cast), cumulativeSumKernels(true, cast) } diff --git a/arrow/compute/vector_cumulative.go b/arrow/compute/vector_cumulative.go index 895fa896f..d066afd6a 100644 --- a/arrow/compute/vector_cumulative.go +++ b/arrow/compute/vector_cumulative.go @@ -20,32 +20,58 @@ package compute import ( "context" + "fmt" + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/compute/exec" "github.com/apache/arrow-go/v18/arrow/compute/internal/kernels" + "github.com/apache/arrow-go/v18/arrow/scalar" ) var ( cumulativeSumDoc = FunctionDoc{ - Summary: "Compute the cumulative sum of an array", + Summary: "Compute the cumulative sum of numeric input", Description: `Return the cumulative sum of the input array. Integer values wrap on overflow; nulls stop the remaining output unless SkipNulls is enabled. -A nil Start uses zero.`, - ArgNames: []string{"array"}, +A nil Start uses zero. For chunked input, output chunks preserve the input +boundaries while accumulation continues across chunks.`, + ArgNames: []string{"values"}, OptionsType: "CumulativeOptions", } cumulativeSumCheckedDoc = FunctionDoc{ - Summary: "Compute the cumulative sum of an array with overflow checking", + Summary: "Compute cumulative sum of numeric input with overflow checking", Description: `Return the cumulative sum of the input array and report -integer overflow. Null handling and Start follow CumulativeOptions.`, - ArgNames: []string{"array"}, +integer overflow. Null handling and Start follow CumulativeOptions. For +chunked input, output chunks preserve input boundaries while accumulation +continues across chunks.`, + ArgNames: []string{"values"}, OptionsType: "CumulativeOptions", } ) type CumulativeOptions = kernels.CumulativeOptions +func safeCastScalar(ctx *exec.KernelCtx, start scalar.Scalar, typ arrow.DataType) (scalar.Scalar, error) { + input := NewDatumWithoutOwning(start) + result, err := CastDatum(ctx.Ctx, input, SafeCastOptions(typ)) + if err != nil { + return nil, err + } + + casted, ok := result.(*ScalarDatum) + if !ok { + result.Release() + return nil, fmt.Errorf("%w: safe cast of cumulative sum start value returned %T", arrow.ErrInvalid, result) + } + + value := casted.Value + casted.Value = nil + result.Release() + return value, nil +} + func RegisterVectorCumulative(reg FunctionRegistry) { - sum, checked := kernels.GetVectorCumulativeKernels() + sum, checked := kernels.GetVectorCumulativeKernels(safeCastScalar) sumFn := NewVectorFunction("cumulative_sum", Unary(), cumulativeSumDoc) sumFn.SetDefaultOptions(&CumulativeOptions{}) diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 9f09fc30e..5910bef6d 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -41,12 +41,13 @@ func cumulativeInput(t *testing.T, mem memory.Allocator, typ arrow.DataType, val func TestCumulativeSum(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 2, 3, 4]`) defer input.Release() expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) @@ -56,6 +57,7 @@ func TestCumulativeSum(t *testing.T) { func TestCumulativeSumAdditionalInputs(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) tests := []struct { name string @@ -77,7 +79,7 @@ func TestCumulativeSumAdditionalInputs(t *testing.T) { expected := cumulativeInput(t, mem, tc.typ, tc.want) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) @@ -86,7 +88,7 @@ func TestCumulativeSumAdditionalInputs(t *testing.T) { expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[3]`) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, compute.NewDatum(int32(3))) + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, compute.NewDatum(int32(3))) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) @@ -97,7 +99,7 @@ func TestCumulativeSumAdditionalInputs(t *testing.T) { defer slicedInput.Release() slicedExpected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3]`) defer slicedExpected.Release() - result, err = compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: slicedInput.Data()}) + result, err = compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: slicedInput.Data()}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: slicedExpected.Data()}, result, nil, nil) @@ -106,13 +108,14 @@ func TestCumulativeSumAdditionalInputs(t *testing.T) { func TestCumulativeSumNullsAndStart(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, 2, null, 3]`) defer input.Release() t.Run("propagate nulls", func(t *testing.T) { expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, null, null, null]`) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) @@ -121,7 +124,7 @@ func TestCumulativeSumNullsAndStart(t *testing.T) { t.Run("skip nulls", func(t *testing.T) { expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null, 3, null, 6]`) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{SkipNulls: true}, &compute.ArrayDatum{Value: input.Data()}) + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{SkipNulls: true}, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) @@ -130,7 +133,7 @@ func TestCumulativeSumNullsAndStart(t *testing.T) { t.Run("start value", func(t *testing.T) { expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[11, null, 13, null, 16]`) defer expected.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{ Start: scalar.NewInt64Scalar(10), SkipNulls: true, }, &compute.ArrayDatum{Value: input.Data()}) @@ -144,6 +147,7 @@ func TestCumulativeSumNullsAndStart(t *testing.T) { func TestCumulativeSumStartSafeCast(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) tests := []struct { name string @@ -161,7 +165,7 @@ func TestCumulativeSumStartSafeCast(t *testing.T) { input := cumulativeInput(t, mem, tc.typ, `[0]`) defer input.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{ Start: tc.start, }, &compute.ArrayDatum{Value: input.Data()}) if result != nil { @@ -173,7 +177,7 @@ func TestCumulativeSumStartSafeCast(t *testing.T) { input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[0]`) defer input.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{ + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{ Start: scalar.NewInt64Scalar(127), }, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) @@ -183,6 +187,40 @@ func TestCumulativeSumStartSafeCast(t *testing.T) { assert.Equal(t, int8(127), actual.(*array.Int8).Value(0)) } +func TestCumulativeSumStartSafeCastDictionary(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + + dictValues := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int64, `[10]`) + defer dictValues.Release() + dictIndices := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[0]`) + defer dictIndices.Release() + dictType := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: arrow.PrimitiveTypes.Int64, + } + dict := array.NewDictionaryArray(dictType, dictIndices, dictValues) + defer dict.Release() + + start, err := scalar.GetScalar(dict, 0) + require.NoError(t, err) + if releasable, ok := start.(scalar.Releasable); ok { + defer releasable.Release() + } + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1]`) + defer input.Release() + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[11]`) + defer expected.Release() + + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{Start: start}, + &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) +} + func TestCumulativeSumStartScalarConversions(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -283,26 +321,44 @@ func TestCumulativeOptionsSerialization(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) - expr := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, - &compute.CumulativeOptions{ - Start: scalar.NewInt32Scalar(10), - SkipNulls: true, - }) - defer expr.Release() + binaryBuffer := memory.NewBufferBytes([]byte("10")) + binaryStart := scalar.NewBinaryScalar(binaryBuffer, arrow.BinaryTypes.Binary) + binaryBuffer.Release() - serialized, err := compute.SerializeExpr(expr, mem) - require.NoError(t, err) - defer serialized.Release() + tests := []struct { + name string + start scalar.Scalar + }{ + {name: "nil", start: nil}, + {name: "int32", start: scalar.NewInt32Scalar(10)}, + {name: "string", start: scalar.NewStringScalar("10")}, + {name: "binary", start: binaryStart}, + {name: "typed null", start: scalar.MakeNullScalar(arrow.PrimitiveTypes.Int32)}, + } - roundTripped, err := compute.DeserializeExpr(mem, serialized) - require.NoError(t, err) - defer roundTripped.Release() - assert.True(t, expr.Equals(roundTripped)) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + expr := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, + &compute.CumulativeOptions{Start: tc.start, SkipNulls: true}) + + serialized, err := compute.SerializeExpr(expr, mem) + require.NoError(t, err) + roundTripped, err := compute.DeserializeExpr(mem, serialized) + serialized.Release() + require.NoError(t, err) + + assert.True(t, expr.Equals(roundTripped)) + assert.NotEmpty(t, roundTripped.String()) + roundTripped.Release() + expr.Release() + }) + } } func TestCumulativeSumChunked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) first := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 2]`) second := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[3, 4]`) input := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{first, second}) @@ -317,7 +373,7 @@ func TestCumulativeSumChunked(t *testing.T) { defer expectedFirst.Release() defer expectedSecond.Release() - result, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) @@ -327,6 +383,7 @@ func TestCumulativeSumChunked(t *testing.T) { func TestCumulativeSumStateAcrossChunks(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) first := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, null]`) second := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[2, 3]`) @@ -351,7 +408,7 @@ func TestCumulativeSumStateAcrossChunks(t *testing.T) { defer expectedFirst.Release() defer expectedSecond.Release() - result, err := compute.CumulativeSum(context.Background(), tc.opts, &compute.ChunkedDatum{Value: input}) + result, err := compute.CumulativeSum(ctx, tc.opts, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) @@ -368,7 +425,8 @@ func TestCumulativeSumStateAcrossExecutorSpans(t *testing.T) { execCtx := compute.DefaultExecCtx() execCtx.ChunkSize = 1 - ctx := compute.SetExecCtx(context.Background(), execCtx) + ctx := compute.WithAllocator(context.Background(), mem) + ctx = compute.SetExecCtx(ctx, execCtx) tests := []struct { name string @@ -411,20 +469,40 @@ func TestCumulativeSumStateAcrossExecutorSpans(t *testing.T) { assert.ErrorIs(t, err, arrow.ErrInvalid) } +func TestCumulativeSumCheckedOverflowAfterEmittedSpan(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + execCtx := compute.DefaultExecCtx() + execCtx.ChunkSize = 1 + execCtx.ExecChannelSize = 0 + ctx := compute.WithAllocator(context.Background(), mem) + ctx = compute.SetExecCtx(ctx, execCtx) + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[127, 1]`) + defer input.Release() + + result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, + &compute.ArrayDatum{Value: input.Data()}) + assert.Nil(t, result) + assert.ErrorIs(t, err, arrow.ErrInvalid) +} + func TestCumulativeSumChecked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[127, 1]`) defer input.Release() - unchecked, err := compute.CumulativeSum(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + unchecked, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) defer unchecked.Release() uncheckedArray := unchecked.(*compute.ArrayDatum).MakeArray() defer uncheckedArray.Release() assert.Equal(t, int8(-128), uncheckedArray.(*array.Int8).Value(1)) - _, err = compute.CumulativeSumChecked(context.Background(), compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) + _, err = compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) assert.ErrorIs(t, err, arrow.ErrInvalid) } @@ -432,6 +510,7 @@ func TestCumulativeSumChecked(t *testing.T) { func TestCumulativeSumCheckedIntegerOverflow(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) tests := []struct { name string @@ -457,7 +536,7 @@ func TestCumulativeSumCheckedIntegerOverflow(t *testing.T) { input := cumulativeInput(t, mem, tc.typ, tc.values) defer input.Release() - result, err := compute.CumulativeSumChecked(context.Background(), compute.CumulativeOptions{}, + result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, &compute.ArrayDatum{Value: input.Data()}) if result != nil { result.Release() diff --git a/arrow/scalar/parse.go b/arrow/scalar/parse.go index e77289046..7d715ee62 100644 --- a/arrow/scalar/parse.go +++ b/arrow/scalar/parse.go @@ -52,6 +52,12 @@ var ( ) func FromScalar(sc *Struct, val interface{}) error { + return FromScalarWithAllocator(sc, val, memory.DefaultAllocator) +} + +// FromScalarWithAllocator populates val from a Struct scalar, allocating any +// cloned scalar fields with mem. +func FromScalarWithAllocator(sc *Struct, val interface{}, mem memory.Allocator) error { if sc == nil || len(sc.Value) == 0 { return nil } @@ -77,7 +83,7 @@ func FromScalar(sc *Struct, val interface{}) error { if err != nil { return err } - if err := setFromScalar(fldVal, value.Field(i)); err != nil { + if err := setFromScalar(fldVal, value.Field(i), mem); err != nil { return err } } @@ -85,14 +91,14 @@ func FromScalar(sc *Struct, val interface{}) error { return nil } -func setFromScalar(s Scalar, v reflect.Value) error { +func setFromScalar(s Scalar, v reflect.Value, mem memory.Allocator) error { if v.Type() == scalarType { if !s.IsValid() && s.DataType().ID() == arrow.NULL { v.Set(reflect.Zero(v.Type())) return nil } - clone, err := cloneScalar(s, memory.DefaultAllocator) + clone, err := cloneScalar(s, mem) if err != nil { return err } @@ -125,7 +131,7 @@ func setFromScalar(s Scalar, v reflect.Value) error { case ListScalar: return fromListScalar(s, v) case *Struct: - return FromScalar(s, v.Interface()) + return FromScalarWithAllocator(s, v.Interface(), mem) default: if v.Type() == reflect.TypeOf(arrow.TimeUnit(0)) { v.Set(reflect.ValueOf(arrow.TimeUnit(s.value().(uint32)))) @@ -186,6 +192,30 @@ func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) { } func cloneScalar(val Scalar, mem memory.Allocator) (Scalar, error) { + if !val.IsValid() { + return MakeNullScalar(val.DataType()), nil + } + + if binary, ok := val.(BinaryScalar); ok { + data := mem.Allocate(len(binary.Data())) + copy(data, binary.Data()) + buf := memory.NewBufferWithAllocator(data, mem) + defer buf.Release() + + switch val.(type) { + case *String: + return NewStringScalarFromBuffer(buf), nil + case *LargeString: + return NewLargeStringScalarFromBuffer(buf), nil + case *LargeBinary: + return NewLargeBinaryScalar(buf), nil + case *FixedSizeBinary: + return NewFixedSizeBinaryScalar(buf, val.DataType()), nil + default: + return NewBinaryScalar(buf, val.DataType()), nil + } + } + arr, err := MakeArrayFromScalar(val, 1, mem) if err != nil { return nil, err diff --git a/arrow/scalar/scalar.go b/arrow/scalar/scalar.go index 8cb317648..6bdccb228 100644 --- a/arrow/scalar/scalar.go +++ b/arrow/scalar/scalar.go @@ -589,11 +589,11 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) { switch arr := arr.(type) { case *array.Binary: - buf := memory.NewBufferBytes(arr.Value(idx)) + buf := memory.NewBufferBytes(append([]byte(nil), arr.Value(idx)...)) defer buf.Release() return NewBinaryScalar(buf, arr.DataType()), nil case *array.LargeBinary: - buf := memory.NewBufferBytes(arr.Value(idx)) + buf := memory.NewBufferBytes(append([]byte(nil), arr.Value(idx)...)) defer buf.Release() return NewLargeBinaryScalar(buf), nil case *array.Boolean: @@ -617,7 +617,7 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) { } return NewExtensionScalar(storage, arr.DataType()), nil case *array.FixedSizeBinary: - buf := memory.NewBufferBytes(arr.Value(idx)) + buf := memory.NewBufferBytes(append([]byte(nil), arr.Value(idx)...)) defer buf.Release() return NewFixedSizeBinaryScalar(buf, arr.DataType()), nil case *array.FixedSizeList: diff --git a/arrow/scalar/scalar_test.go b/arrow/scalar/scalar_test.go index 5634864f7..0fd68ffc0 100644 --- a/arrow/scalar/scalar_test.go +++ b/arrow/scalar/scalar_test.go @@ -1291,6 +1291,63 @@ type OptionValTest struct { func (OptionValTest) TypeName() string { return "OptionValTest" } +type scalarFieldOption struct { + Value scalar.Scalar `compute:"value"` +} + +func (scalarFieldOption) TypeName() string { return "scalarFieldOption" } + +type zeroingAllocator struct{} + +func (*zeroingAllocator) Allocate(size int) []byte { return make([]byte, size) } + +func (*zeroingAllocator) Reallocate(size int, old []byte) []byte { + next := make([]byte, size) + copy(next, old) + clear(old) + return next +} + +func (*zeroingAllocator) Free(buf []byte) { clear(buf) } + +func TestScalarFieldCloneOwnsBinaryValue(t *testing.T) { + mem := memory.NewCheckedAllocator(&zeroingAllocator{}) + defer mem.AssertSize(t, 0) + + original := scalar.NewStringScalar("10") + encoded, err := scalar.ToScalar(scalarFieldOption{Value: original}, mem) + require.NoError(t, err) + original.Release() + + var decoded scalarFieldOption + require.NoError(t, scalar.FromScalarWithAllocator(encoded.(*scalar.Struct), &decoded, mem)) + encoded.(*scalar.Struct).Release() + + value, ok := decoded.Value.(scalar.BinaryScalar) + require.True(t, ok) + assert.Equal(t, "10", string(value.Data())) + value.Release() +} + +func TestGetScalarBinaryValueOwnsArrayBytes(t *testing.T) { + mem := memory.NewCheckedAllocator(&zeroingAllocator{}) + defer mem.AssertSize(t, 0) + + bldr := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary) + bldr.Append([]byte("10")) + arr := bldr.NewArray() + bldr.Release() + + value, err := scalar.GetScalar(arr, 0) + require.NoError(t, err) + arr.Release() + + binaryValue, ok := value.(scalar.BinaryScalar) + require.True(t, ok) + assert.Equal(t, "10", string(binaryValue.Data())) + binaryValue.Release() +} + func TestToScalar(t *testing.T) { ot := &OptionValTest{ToType: arrow.BinaryTypes.String, Allow: true} sc, err := scalar.ToScalar(ot, memory.DefaultAllocator) From 44d7107cb19e6e5ff697e11f0c5836eeea6338d5 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 03:20:35 +0200 Subject: [PATCH 06/17] fix(compute): release queued results on cancellation --- arrow/compute/executor.go | 34 +++++++++++++++++++++---- arrow/compute/vector_cumulative_test.go | 19 ++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/arrow/compute/executor.go b/arrow/compute/executor.go index b164e2327..988017440 100644 --- a/arrow/compute/executor.go +++ b/arrow/compute/executor.go @@ -414,6 +414,14 @@ type KernelExecutor interface { Clear() } +func releaseRemainingResults(out <-chan Datum) { + for datum := range out { + if datum != nil { + datum.Release() + } + } +} + // the base implementation for executing non-aggregate kernels. type nonAggExecImpl struct { ctx *exec.KernelCtx @@ -543,9 +551,11 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC // get first output select { case <-ctx.Done(): + releaseRemainingResults(out) return nil case output = <-out: if output == nil { + releaseRemainingResults(out) return nil } // if the inputs contained at least one chunked array @@ -558,9 +568,9 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC for { select { case <-ctx.Done(): - // context is done, either cancelled or a timeout. - // either way, we end early and return what we've got so far. + // Execution was cancelled, so release any partial results and return no output. releasePartial() + releaseRemainingResults(out) return nil case o, ok := <-out: if !ok { // channel closed, wrap it up @@ -996,8 +1006,13 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC var output Datum select { case <-ctx.Done(): + releaseRemainingResults(out) return nil case output = <-out: + if output == nil { + releaseRemainingResults(out) + return nil + } } // we got an output datum, but let's wait for the channel to @@ -1005,8 +1020,15 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC select { case <-ctx.Done(): output.Release() + releaseRemainingResults(out) return nil - case <-out: + case extra, ok := <-out: + if ok { + if extra != nil { + extra.Release() + } + releaseRemainingResults(out) + } return output } } @@ -1044,10 +1066,12 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC // get first output select { case <-ctx.Done(): + releaseRemainingResults(out) return nil case output = <-out: if output == nil || ctx.Err() != nil { releasePartial() + releaseRemainingResults(out) return nil } @@ -1061,9 +1085,9 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC for { select { case <-ctx.Done(): - // context is done, either cancelled or a timeout. - // either way, we end early and return what we've got so far. + // Execution was cancelled, so release any partial results and return no output. releasePartial() + releaseRemainingResults(out) return nil case o, ok := <-out: if !ok { // channel closed, wrap it up diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 5910bef6d..38b2c256d 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -488,6 +488,25 @@ func TestCumulativeSumCheckedOverflowAfterEmittedSpan(t *testing.T) { assert.ErrorIs(t, err, arrow.ErrInvalid) } +func TestCumulativeSumCheckedOverflowDrainsBufferedResults(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + execCtx := compute.DefaultExecCtx() + execCtx.ChunkSize = 1 + execCtx.ExecChannelSize = 1 + ctx := compute.WithAllocator(context.Background(), mem) + ctx = compute.SetExecCtx(ctx, execCtx) + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[1, 2, 127]`) + defer input.Release() + + result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, + &compute.ArrayDatum{Value: input.Data()}) + assert.Nil(t, result) + assert.ErrorIs(t, err, arrow.ErrInvalid) +} + func TestCumulativeSumChecked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) From 30f95108f14df45584691a1b9768081c01c9f2ba Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 03:20:35 +0200 Subject: [PATCH 07/17] fix(compute): preserve safe function option equality --- arrow/compute/expression.go | 54 ++++++++-------- arrow/compute/expression_test.go | 105 +++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 28 deletions(-) diff --git a/arrow/compute/expression.go b/arrow/compute/expression.go index 3dc168690..b06a65cd1 100644 --- a/arrow/compute/expression.go +++ b/arrow/compute/expression.go @@ -41,8 +41,6 @@ import ( var hashSeed = maphash.MakeSeed() -var scalarInterfaceType = reflect.TypeOf((*scalar.Scalar)(nil)).Elem() - // Expression is an interface for mapping one datum to another. An expression // is one of: // @@ -383,39 +381,39 @@ func (c *Call) Equals(other Expression) bool { } func equalFunctionOptions(lhs, rhs FunctionOptions) bool { + if left, ok := cumulativeOptions(lhs); ok { + right, ok := cumulativeOptions(rhs) + if !ok { + return false + } + if left == nil || right == nil { + return left == nil && right == nil + } + return left.SkipNulls == right.SkipNulls && equalOptionalScalar(left.Start, right.Start) + } + if lhs == nil || rhs == nil { return lhs == nil && rhs == nil } - if reflect.TypeOf(lhs) != reflect.TypeOf(rhs) { - return false - } + return reflect.DeepEqual(lhs, rhs) +} - left := reflect.Indirect(reflect.ValueOf(lhs)) - right := reflect.Indirect(reflect.ValueOf(rhs)) - if !left.IsValid() || left.Kind() != reflect.Struct { - return reflect.DeepEqual(lhs, rhs) +func cumulativeOptions(opts FunctionOptions) (*CumulativeOptions, bool) { + switch opts := opts.(type) { + case CumulativeOptions: + return &opts, true + case *CumulativeOptions: + return opts, true + default: + return nil, false } +} - for i := 0; i < left.NumField(); i++ { - leftField := left.Field(i) - rightField := right.Field(i) - if leftField.Type() == scalarInterfaceType { - leftScalar, leftOK := leftField.Interface().(scalar.Scalar) - rightScalar, rightOK := rightField.Interface().(scalar.Scalar) - if leftOK != rightOK { - return false - } - if leftOK && !scalar.Equals(leftScalar, rightScalar) { - return false - } - continue - } - - if !reflect.DeepEqual(leftField.Interface(), rightField.Interface()) { - return false - } +func equalOptionalScalar(lhs, rhs scalar.Scalar) bool { + if lhs == nil || rhs == nil { + return lhs == nil && rhs == nil } - return true + return scalar.Equals(lhs, rhs) } func (c *Call) Release() { diff --git a/arrow/compute/expression_test.go b/arrow/compute/expression_test.go index 42f64394f..6f6fcee27 100644 --- a/arrow/compute/expression_test.go +++ b/arrow/compute/expression_test.go @@ -30,6 +30,12 @@ import ( "github.com/stretchr/testify/assert" ) +type privateFunctionOptions struct { + value int +} + +func (privateFunctionOptions) TypeName() string { return "privateFunctionOptions" } + func TestExpressionToString(t *testing.T) { ts, _ := scalar.MakeScalar("1990-10-23 10:23:33.123456").CastTo(arrow.FixedWidthTypes.Timestamp_ns) @@ -115,6 +121,105 @@ func TestExpressionEquality(t *testing.T) { } } +func TestExpressionEqualityWithPrivateFunctionOptions(t *testing.T) { + left := compute.NewCall("test", nil, privateFunctionOptions{value: 1}) + right := compute.NewCall("test", nil, privateFunctionOptions{value: 1}) + different := compute.NewCall("test", nil, privateFunctionOptions{value: 2}) + defer left.Release() + defer right.Release() + defer different.Release() + + assert.NotPanics(t, func() { + assert.True(t, left.Equals(right)) + assert.False(t, left.Equals(different)) + }) +} + +func TestCumulativeOptionsEquality(t *testing.T) { + newBinaryStart := func() scalar.Scalar { + buf := memory.NewBufferBytes([]byte("10")) + defer buf.Release() + return scalar.NewBinaryScalar(buf, arrow.BinaryTypes.Binary) + } + + tests := []struct { + name string + leftStart, rightStart func() scalar.Scalar + leftSkip, rightSkip bool + want bool + }{ + { + name: "both nil", + leftStart: func() scalar.Scalar { return nil }, + rightStart: func() scalar.Scalar { return nil }, + want: true, + }, + { + name: "one nil", + leftStart: func() scalar.Scalar { return nil }, + rightStart: func() scalar.Scalar { return scalar.NewInt32Scalar(10) }, + want: false, + }, + { + name: "equal numeric scalars", + leftStart: func() scalar.Scalar { return scalar.NewInt32Scalar(10) }, + rightStart: func() scalar.Scalar { return scalar.NewInt32Scalar(10) }, + want: true, + }, + { + name: "equal string scalars", + leftStart: func() scalar.Scalar { return scalar.NewStringScalar("10") }, + rightStart: func() scalar.Scalar { return scalar.NewStringScalar("10") }, + want: true, + }, + { + name: "equal binary scalars", + leftStart: newBinaryStart, + rightStart: newBinaryStart, + want: true, + }, + { + name: "different scalar values", + leftStart: func() scalar.Scalar { return scalar.NewInt32Scalar(10) }, + rightStart: func() scalar.Scalar { return scalar.NewInt32Scalar(11) }, + want: false, + }, + { + name: "different scalar types", + leftStart: func() scalar.Scalar { return scalar.NewInt32Scalar(10) }, + rightStart: func() scalar.Scalar { return scalar.NewInt64Scalar(10) }, + want: false, + }, + { + name: "different skip nulls", + leftStart: func() scalar.Scalar { return nil }, + rightStart: func() scalar.Scalar { return nil }, + leftSkip: false, + rightSkip: true, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + left := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, + &compute.CumulativeOptions{Start: tc.leftStart(), SkipNulls: tc.leftSkip}) + right := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, + &compute.CumulativeOptions{Start: tc.rightStart(), SkipNulls: tc.rightSkip}) + defer left.Release() + defer right.Release() + + assert.Equal(t, tc.want, left.Equals(right)) + }) + } + + left := compute.NewCall("cumulative_sum", nil, compute.CumulativeOptions{Start: scalar.NewInt32Scalar(10)}) + right := compute.NewCall("cumulative_sum", nil, &compute.CumulativeOptions{Start: scalar.NewInt32Scalar(10)}) + defer left.Release() + defer right.Release() + assert.True(t, left.Equals(right)) +} + func TestExpressionHashing(t *testing.T) { set := make(map[uint64]compute.Expression) From 31735d615668336c77a7ae719a41b885a7809a7b Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 11:59:31 +0200 Subject: [PATCH 08/17] fix(compute): align cumulative sum with chunked execution --- arrow/compute/exec.go | 1 - arrow/compute/executor.go | 64 ++-------- arrow/compute/expression.go | 2 - arrow/compute/expression_test.go | 19 ++- .../internal/kernels/vector_cumulative.go | 119 ++++++++++++------ arrow/compute/vector_cumulative.go | 8 +- arrow/compute/vector_cumulative_test.go | 113 +++++++++-------- arrow/scalar/parse.go | 15 ++- arrow/scalar/scalar_test.go | 23 ++++ 9 files changed, 200 insertions(+), 164 deletions(-) diff --git a/arrow/compute/exec.go b/arrow/compute/exec.go index 51633a04f..0afb3b13d 100644 --- a/arrow/compute/exec.go +++ b/arrow/compute/exec.go @@ -176,7 +176,6 @@ func execInternal(ctx context.Context, fn Function, opts FunctionOptions, passed if ctx.Err() == context.Canceled && result != nil { result.Release() - result = nil } return diff --git a/arrow/compute/executor.go b/arrow/compute/executor.go index 988017440..095a7e15d 100644 --- a/arrow/compute/executor.go +++ b/arrow/compute/executor.go @@ -414,14 +414,6 @@ type KernelExecutor interface { Clear() } -func releaseRemainingResults(out <-chan Datum) { - for datum := range out { - if datum != nil { - datum.Release() - } - } -} - // the base implementation for executing non-aggregate kernels. type nonAggExecImpl struct { ctx *exec.KernelCtx @@ -531,16 +523,6 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC output Datum acc []arrow.Array ) - releasePartial := func() { - if output != nil { - output.Release() - output = nil - } - for _, c := range acc { - c.Release() - } - acc = nil - } toChunked := func() { acc = output.(ArrayLikeDatum).Chunks() @@ -551,13 +533,8 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC // get first output select { case <-ctx.Done(): - releaseRemainingResults(out) return nil case output = <-out: - if output == nil { - releaseRemainingResults(out) - return nil - } // if the inputs contained at least one chunked array // then we want to return chunked output if hasChunked { @@ -568,10 +545,9 @@ func (s *scalarExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC for { select { case <-ctx.Done(): - // Execution was cancelled, so release any partial results and return no output. - releasePartial() - releaseRemainingResults(out) - return nil + // context is done, either cancelled or a timeout. + // either way, we end early and return what we've got so far. + return output case o, ok := <-out: if !ok { // channel closed, wrap it up if output != nil { @@ -1006,13 +982,8 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC var output Datum select { case <-ctx.Done(): - releaseRemainingResults(out) return nil case output = <-out: - if output == nil { - releaseRemainingResults(out) - return nil - } } // we got an output datum, but let's wait for the channel to @@ -1020,15 +991,8 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC select { case <-ctx.Done(): output.Release() - releaseRemainingResults(out) return nil - case extra, ok := <-out: - if ok { - if extra != nil { - extra.Release() - } - releaseRemainingResults(out) - } + case <-out: return output } } @@ -1038,16 +1002,6 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC output Datum acc []arrow.Array ) - releasePartial := func() { - if output != nil { - output.Release() - output = nil - } - for _, c := range acc { - c.Release() - } - acc = nil - } toChunked := func() { out := output.(ArrayLikeDatum).Chunks() @@ -1066,12 +1020,9 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC // get first output select { case <-ctx.Done(): - releaseRemainingResults(out) return nil case output = <-out: if output == nil || ctx.Err() != nil { - releasePartial() - releaseRemainingResults(out) return nil } @@ -1085,10 +1036,9 @@ func (v *vectorExecutor) WrapResults(ctx context.Context, out <-chan Datum, hasC for { select { case <-ctx.Done(): - // Execution was cancelled, so release any partial results and return no output. - releasePartial() - releaseRemainingResults(out) - return nil + // context is done, either cancelled or a timeout. + // either way, we end early and return what we've got so far. + return output case o, ok := <-out: if !ok { // channel closed, wrap it up if output != nil { diff --git a/arrow/compute/expression.go b/arrow/compute/expression.go index b06a65cd1..8ec332a82 100644 --- a/arrow/compute/expression.go +++ b/arrow/compute/expression.go @@ -400,8 +400,6 @@ func equalFunctionOptions(lhs, rhs FunctionOptions) bool { func cumulativeOptions(opts FunctionOptions) (*CumulativeOptions, bool) { switch opts := opts.(type) { - case CumulativeOptions: - return &opts, true case *CumulativeOptions: return opts, true default: diff --git a/arrow/compute/expression_test.go b/arrow/compute/expression_test.go index 6f6fcee27..b132ae47c 100644 --- a/arrow/compute/expression_test.go +++ b/arrow/compute/expression_test.go @@ -213,11 +213,20 @@ func TestCumulativeOptionsEquality(t *testing.T) { }) } - left := compute.NewCall("cumulative_sum", nil, compute.CumulativeOptions{Start: scalar.NewInt32Scalar(10)}) - right := compute.NewCall("cumulative_sum", nil, &compute.CumulativeOptions{Start: scalar.NewInt32Scalar(10)}) - defer left.Release() - defer right.Release() - assert.True(t, left.Equals(right)) +} + +func TestCumulativeOptionsRelease(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + data := mem.Allocate(2) + copy(data, []byte("10")) + buffer := memory.NewBufferWithAllocator(data, mem) + start := scalar.NewBinaryScalar(buffer, arrow.BinaryTypes.Binary) + buffer.Release() + + expr := compute.NewCall("cumulative_sum", nil, &compute.CumulativeOptions{Start: start}) + expr.Release() } func TestExpressionHashing(t *testing.T) { diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index 44f922460..5eba4c0e3 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -194,65 +194,107 @@ func checkedAdder[T arrow.NumericType]() func(T, T) (T, error) { } } -func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.ExecSpan, out *exec.ExecResult) error { - state := ctx.State.(*cumulativeSumState[T]) - input := &batch.Values[0].Array - - out.Len = input.Len - if input.Len == 0 { - return nil +func prepareCumulativeOutput[T arrow.NumericType](ctx *exec.KernelCtx, out *exec.ExecResult, needsValidity bool) { + if out.Len == 0 { + return } - data := ctx.Allocate(int(input.Len) * arrow.GetDataType[T]().(arrow.FixedWidthDataType).Bytes()) + data := ctx.Allocate(int(out.Len) * arrow.GetDataType[T]().(arrow.FixedWidthDataType).Bytes()) out.Buffers[1].WrapBuffer(data) - values := exec.GetSpanValues[T](out, 1) - inputValues := exec.GetSpanValues[T](input, 1) - needsValidity := state.encounteredNull || input.MayHaveNulls() if needsValidity { - validity := ctx.AllocateBitmap(input.Len) + validity := ctx.AllocateBitmap(out.Len) validityBytes := validity.Bytes() for i := range validityBytes { validityBytes[i] = 0xFF } out.Buffers[0].WrapBuffer(validity) } +} - var nulls int64 - for i := int64(0); i < input.Len; i++ { - valid := len(input.Buffers[0].Buf) == 0 || bitutil.BitIsSet(input.Buffers[0].Buf, int(input.Offset+i)) - if !valid || state.encounteredNull { - if needsValidity { - bitutil.ClearBit(out.Buffers[0].Buf, int(i)) +func cumulativeSumSpans[T arrow.NumericType](ctx *exec.KernelCtx, state *cumulativeSumState[T], inputs []*exec.ArraySpan, out *exec.ExecResult, needsValidity bool) error { + prepareCumulativeOutput[T](ctx, out, needsValidity) + values := exec.GetSpanValues[T](out, 1) + + var ( + nulls int64 + outputOffset int64 + ) + for _, input := range inputs { + inputValues := exec.GetSpanValues[T](input, 1) + for i := int64(0); i < input.Len; i++ { + valid := len(input.Buffers[0].Buf) == 0 || bitutil.BitIsSet(input.Buffers[0].Buf, int(input.Offset+i)) + outputIndex := outputOffset + i + if !valid || state.encounteredNull { + if needsValidity { + bitutil.ClearBit(out.Buffers[0].Buf, int(outputIndex)) + } + nulls++ + if !valid && !state.skipNulls { + state.encounteredNull = true + } + continue } - nulls++ - if !valid && !state.skipNulls { - state.encounteredNull = true + + current := state.current + value := inputValues[i] + var err error + if state.checked { + current, err = state.add(current, value) + } else { + current += value + } + if err != nil { + out.Release() + return err } - continue - } - current := state.current - value := inputValues[i] - var err error - if state.checked { - current, err = state.add(current, value) - } else { - current += value + state.current = current + values[outputIndex] = current } - if err != nil { - out.Release() - return err - } - - state.current = current - values[i] = current + outputOffset += input.Len } out.Nulls = nulls return nil } +func cumulativeSumExec[T arrow.NumericType](ctx *exec.KernelCtx, batch *exec.ExecSpan, out *exec.ExecResult) error { + state := ctx.State.(*cumulativeSumState[T]) + input := &batch.Values[0].Array + + out.Len = input.Len + if input.Len == 0 { + return nil + } + + return cumulativeSumSpans(ctx, state, []*exec.ArraySpan{input}, out, state.encounteredNull || input.MayHaveNulls()) +} + +func cumulativeSumExecChunked[T arrow.NumericType](ctx *exec.KernelCtx, batch []*arrow.Chunked, out *exec.ExecResult) ([]*exec.ExecResult, error) { + state := ctx.State.(*cumulativeSumState[T]) + input := batch[0] + out.Len = int64(input.Len()) + if out.Len == 0 { + return []*exec.ExecResult{out}, nil + } + + chunks := input.Chunks() + spans := make([]exec.ArraySpan, len(chunks)) + inputs := make([]*exec.ArraySpan, len(chunks)) + needsValidity := state.encounteredNull || input.NullN() != 0 + for i, chunk := range chunks { + spans[i].SetMembers(chunk.Data()) + inputs[i] = &spans[i] + needsValidity = needsValidity || spans[i].MayHaveNulls() + } + + if err := cumulativeSumSpans(ctx, state, inputs, out, needsValidity); err != nil { + return nil, err + } + return []*exec.ExecResult{out}, nil +} + func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked bool, cast ScalarCastFn) exec.VectorKernel { kernel := exec.NewVectorKernel( []exec.InputType{exec.NewExactInput(typ)}, @@ -260,6 +302,9 @@ func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked boo cumulativeSumExec[T], initCumulativeSum[T](checked, cast)) kernel.Parallelizable = false + kernel.CanExecuteChunkWise = false + kernel.OutputChunked = false + kernel.ExecChunked = cumulativeSumExecChunked[T] return kernel } diff --git a/arrow/compute/vector_cumulative.go b/arrow/compute/vector_cumulative.go index d066afd6a..c7386c0c0 100644 --- a/arrow/compute/vector_cumulative.go +++ b/arrow/compute/vector_cumulative.go @@ -33,8 +33,8 @@ var ( Summary: "Compute the cumulative sum of numeric input", Description: `Return the cumulative sum of the input array. Integer values wrap on overflow; nulls stop the remaining output unless SkipNulls is enabled. -A nil Start uses zero. For chunked input, output chunks preserve the input -boundaries while accumulation continues across chunks.`, +A nil Start uses zero. For chunked input, accumulation continues across all +chunks and the result is returned as one array.`, ArgNames: []string{"values"}, OptionsType: "CumulativeOptions", } @@ -42,8 +42,8 @@ boundaries while accumulation continues across chunks.`, Summary: "Compute cumulative sum of numeric input with overflow checking", Description: `Return the cumulative sum of the input array and report integer overflow. Null handling and Start follow CumulativeOptions. For -chunked input, output chunks preserve input boundaries while accumulation -continues across chunks.`, +chunked input, accumulation continues across all chunks and the result is +returned as one array.`, ArgNames: []string{"values"}, OptionsType: "CumulativeOptions", } diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 38b2c256d..776393ac0 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -366,18 +366,42 @@ func TestCumulativeSumChunked(t *testing.T) { defer first.Release() defer second.Release() - expectedFirst := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3]`) - expectedSecond := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[6, 10]`) - expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedFirst, expectedSecond}) + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) defer expected.Release() - defer expectedFirst.Release() - defer expectedSecond.Release() result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() - assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) + require.Equal(t, compute.KindArray, result.Kind()) + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + +} + +func TestCumulativeSumChunkedOutputIgnoresChunkSizeAndEmptyChunks(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + emptyBefore := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[]`) + values := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 2, 3, 4]`) + emptyAfter := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[]`) + input := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{emptyBefore, values, emptyAfter}) + defer input.Release() + defer emptyBefore.Release() + defer values.Release() + defer emptyAfter.Release() + execCtx := compute.DefaultExecCtx() + execCtx.ChunkSize = 2 + ctx := compute.SetExecCtx(context.Background(), execCtx) + ctx = compute.WithAllocator(ctx, mem) + + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) + defer expected.Release() + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) + require.NoError(t, err) + defer result.Release() + require.Equal(t, compute.KindArray, result.Kind()) + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) } func TestCumulativeSumStateAcrossChunks(t *testing.T) { @@ -395,28 +419,42 @@ func TestCumulativeSumStateAcrossChunks(t *testing.T) { for _, tc := range []struct { name string opts compute.CumulativeOptions - expected []string + expected string }{ - {name: "propagate nulls", expected: []string{`[1, null]`, `[null, null]`}}, - {name: "skip nulls", opts: compute.CumulativeOptions{SkipNulls: true}, expected: []string{`[1, null]`, `[3, 6]`}}, + {name: "propagate nulls", expected: `[1, null, null, null]`}, + {name: "skip nulls", opts: compute.CumulativeOptions{SkipNulls: true}, expected: `[1, null, 3, 6]`}, } { t.Run(tc.name, func(t *testing.T) { - expectedFirst := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected[0]) - expectedSecond := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected[1]) - expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedFirst, expectedSecond}) + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected) defer expected.Release() - defer expectedFirst.Release() - defer expectedSecond.Release() result, err := compute.CumulativeSum(ctx, tc.opts, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() - assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) + require.Equal(t, compute.KindArray, result.Kind()) + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) }) } } -func TestCumulativeSumStateAcrossExecutorSpans(t *testing.T) { +func TestCumulativeSumCheckedChunkedOverflow(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + first := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[127]`) + second := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[1]`) + input := arrow.NewChunked(arrow.PrimitiveTypes.Int8, []arrow.Array{first, second}) + defer input.Release() + defer first.Release() + defer second.Release() + + ctx := compute.WithAllocator(context.Background(), mem) + result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) + assert.Nil(t, result) + assert.ErrorIs(t, err, arrow.ErrInvalid) +} + +func TestCumulativeSumIgnoresExecutorChunkSize(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -440,13 +478,12 @@ func TestCumulativeSumStateAcrossExecutorSpans(t *testing.T) { t.Run(tc.name, func(t *testing.T) { expectedArray := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected) defer expectedArray.Release() - expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedArray}) - defer expected.Release() result, err := compute.CumulativeSum(ctx, tc.opts, &compute.ArrayDatum{Value: input.Data()}) require.NoError(t, err) defer result.Release() - assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) + require.Equal(t, compute.KindArray, result.Kind()) + assertDatumsEqual(t, &compute.ArrayDatum{Value: expectedArray.Data()}, result, nil, nil) }) } @@ -469,44 +506,6 @@ func TestCumulativeSumStateAcrossExecutorSpans(t *testing.T) { assert.ErrorIs(t, err, arrow.ErrInvalid) } -func TestCumulativeSumCheckedOverflowAfterEmittedSpan(t *testing.T) { - mem := memory.NewCheckedAllocator(memory.DefaultAllocator) - defer mem.AssertSize(t, 0) - - execCtx := compute.DefaultExecCtx() - execCtx.ChunkSize = 1 - execCtx.ExecChannelSize = 0 - ctx := compute.WithAllocator(context.Background(), mem) - ctx = compute.SetExecCtx(ctx, execCtx) - - input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[127, 1]`) - defer input.Release() - - result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, - &compute.ArrayDatum{Value: input.Data()}) - assert.Nil(t, result) - assert.ErrorIs(t, err, arrow.ErrInvalid) -} - -func TestCumulativeSumCheckedOverflowDrainsBufferedResults(t *testing.T) { - mem := memory.NewCheckedAllocator(memory.DefaultAllocator) - defer mem.AssertSize(t, 0) - - execCtx := compute.DefaultExecCtx() - execCtx.ChunkSize = 1 - execCtx.ExecChannelSize = 1 - ctx := compute.WithAllocator(context.Background(), mem) - ctx = compute.SetExecCtx(ctx, execCtx) - - input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[1, 2, 127]`) - defer input.Release() - - result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, - &compute.ArrayDatum{Value: input.Data()}) - assert.Nil(t, result) - assert.ErrorIs(t, err, arrow.ErrInvalid) -} - func TestCumulativeSumChecked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) diff --git a/arrow/scalar/parse.go b/arrow/scalar/parse.go index 7d715ee62..4f9de7e38 100644 --- a/arrow/scalar/parse.go +++ b/arrow/scalar/parse.go @@ -161,6 +161,13 @@ func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) { case reflect.Struct: scalars := make([]Scalar, 0, v.Type().NumField()) fields := make([]string, 0, v.Type().NumField()) + releaseScalars := func() { + for _, s := range scalars { + if releasable, ok := s.(Releasable); ok { + releasable.Release() + } + } + } for i := 0; i < v.Type().NumField(); i++ { fld := v.Type().Field(i) tag := fld.Tag.Get("compute") @@ -171,6 +178,7 @@ func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) { fldVal := v.Field(i) s, err := ToScalar(fldVal.Interface(), mem) if err != nil { + releaseScalars() return nil, err } scalars = append(scalars, s) @@ -183,7 +191,12 @@ func ToScalar(val interface{}, mem memory.Allocator) (Scalar, error) { fields = append(fields, "_type_name") } - return NewStructScalarWithNames(scalars, fields) + result, err := NewStructScalarWithNames(scalars, fields) + if err != nil { + releaseScalars() + return nil, err + } + return result, nil case reflect.Slice: return createListScalar(v, mem) default: diff --git a/arrow/scalar/scalar_test.go b/arrow/scalar/scalar_test.go index 0fd68ffc0..bf2406766 100644 --- a/arrow/scalar/scalar_test.go +++ b/arrow/scalar/scalar_test.go @@ -1297,6 +1297,11 @@ type scalarFieldOption struct { func (scalarFieldOption) TypeName() string { return "scalarFieldOption" } +type scalarFieldWithUnsupportedSlice struct { + Value scalar.Scalar `compute:"value"` + Unsupported []float64 `compute:"unsupported"` +} + type zeroingAllocator struct{} func (*zeroingAllocator) Allocate(size int) []byte { return make([]byte, size) } @@ -1329,6 +1334,24 @@ func TestScalarFieldCloneOwnsBinaryValue(t *testing.T) { value.Release() } +func TestToScalarReleasesFieldsWhenLaterFieldFails(t *testing.T) { + mem := memory.NewCheckedAllocator(&zeroingAllocator{}) + defer mem.AssertSize(t, 0) + + data := mem.Allocate(2) + copy(data, []byte("10")) + buffer := memory.NewBufferWithAllocator(data, mem) + original := scalar.NewBinaryScalar(buffer, arrow.BinaryTypes.Binary) + buffer.Release() + + _, err := scalar.ToScalar(scalarFieldWithUnsupportedSlice{ + Value: original, + Unsupported: []float64{1}, + }, mem) + require.Error(t, err) + original.Release() +} + func TestGetScalarBinaryValueOwnsArrayBytes(t *testing.T) { mem := memory.NewCheckedAllocator(&zeroingAllocator{}) defer mem.AssertSize(t, 0) From 69b5679c14c0a22bade9100190ac927df0bc0b07 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 16:45:55 +0200 Subject: [PATCH 09/17] fix(compute): preserve chunked cumulative output --- .../internal/kernels/vector_cumulative.go | 1 - arrow/compute/vector_cumulative.go | 4 +- arrow/compute/vector_cumulative_test.go | 56 ++++++++++++++++--- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index 5eba4c0e3..18e66c088 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -303,7 +303,6 @@ func newCumulativeSumKernel[T arrow.NumericType](typ arrow.DataType, checked boo initCumulativeSum[T](checked, cast)) kernel.Parallelizable = false kernel.CanExecuteChunkWise = false - kernel.OutputChunked = false kernel.ExecChunked = cumulativeSumExecChunked[T] return kernel } diff --git a/arrow/compute/vector_cumulative.go b/arrow/compute/vector_cumulative.go index c7386c0c0..cc7d28534 100644 --- a/arrow/compute/vector_cumulative.go +++ b/arrow/compute/vector_cumulative.go @@ -34,7 +34,7 @@ var ( Description: `Return the cumulative sum of the input array. Integer values wrap on overflow; nulls stop the remaining output unless SkipNulls is enabled. A nil Start uses zero. For chunked input, accumulation continues across all -chunks and the result is returned as one array.`, +chunks and the result is returned as a chunked array.`, ArgNames: []string{"values"}, OptionsType: "CumulativeOptions", } @@ -43,7 +43,7 @@ chunks and the result is returned as one array.`, Description: `Return the cumulative sum of the input array and report integer overflow. Null handling and Start follow CumulativeOptions. For chunked input, accumulation continues across all chunks and the result is -returned as one array.`, +returned as a chunked array.`, ArgNames: []string{"values"}, OptionsType: "CumulativeOptions", } diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 776393ac0..557cc020f 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -355,6 +355,38 @@ func TestCumulativeOptionsSerialization(t *testing.T) { } } +func TestCumulativeOptionsDictionarySerialization(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + dictValues := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int64, `[10]`) + defer dictValues.Release() + dictIndices := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int8, `[0]`) + defer dictIndices.Release() + dictType := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: arrow.PrimitiveTypes.Int64, + } + dict := array.NewDictionaryArray(dictType, dictIndices, dictValues) + defer dict.Release() + + start, err := scalar.GetScalar(dict, 0) + require.NoError(t, err) + expr := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, + &compute.CumulativeOptions{Start: start}) + defer expr.Release() + + serialized, err := compute.SerializeExpr(expr, mem) + require.NoError(t, err) + defer serialized.Release() + + roundTripped, err := compute.DeserializeExpr(mem, serialized) + require.NoError(t, err) + defer roundTripped.Release() + + assert.True(t, expr.Equals(roundTripped)) +} + func TestCumulativeSumChunked(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -366,14 +398,16 @@ func TestCumulativeSumChunked(t *testing.T) { defer first.Release() defer second.Release() - expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) + expectedArray := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) + defer expectedArray.Release() + expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedArray}) defer expected.Release() result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() - require.Equal(t, compute.KindArray, result.Kind()) - assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + require.Equal(t, compute.KindChunked, result.Kind()) + assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) } @@ -395,13 +429,15 @@ func TestCumulativeSumChunkedOutputIgnoresChunkSizeAndEmptyChunks(t *testing.T) ctx := compute.SetExecCtx(context.Background(), execCtx) ctx = compute.WithAllocator(ctx, mem) - expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) + expectedArray := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6, 10]`) + defer expectedArray.Release() + expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedArray}) defer expected.Release() result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{}, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() - require.Equal(t, compute.KindArray, result.Kind()) - assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + require.Equal(t, compute.KindChunked, result.Kind()) + assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) } func TestCumulativeSumStateAcrossChunks(t *testing.T) { @@ -425,14 +461,16 @@ func TestCumulativeSumStateAcrossChunks(t *testing.T) { {name: "skip nulls", opts: compute.CumulativeOptions{SkipNulls: true}, expected: `[1, null, 3, 6]`}, } { t.Run(tc.name, func(t *testing.T) { - expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected) + expectedArray := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.expected) + defer expectedArray.Release() + expected := arrow.NewChunked(arrow.PrimitiveTypes.Int32, []arrow.Array{expectedArray}) defer expected.Release() result, err := compute.CumulativeSum(ctx, tc.opts, &compute.ChunkedDatum{Value: input}) require.NoError(t, err) defer result.Release() - require.Equal(t, compute.KindArray, result.Kind()) - assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + require.Equal(t, compute.KindChunked, result.Kind()) + assertDatumsEqual(t, &compute.ChunkedDatum{Value: expected}, result, nil, nil) }) } } From aa2bd9552f781dd327b23a6643934c48e4d3d278 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 16:46:21 +0200 Subject: [PATCH 10/17] fix(arrow/scalar): roll back partial scalar decoding --- arrow/scalar/parse.go | 23 +++++++++++-- arrow/scalar/scalar_test.go | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/arrow/scalar/parse.go b/arrow/scalar/parse.go index 4f9de7e38..248eeb0a9 100644 --- a/arrow/scalar/parse.go +++ b/arrow/scalar/parse.go @@ -58,6 +58,17 @@ func FromScalar(sc *Struct, val interface{}) error { // FromScalarWithAllocator populates val from a Struct scalar, allocating any // cloned scalar fields with mem. func FromScalarWithAllocator(sc *Struct, val interface{}, mem memory.Allocator) error { + var rollbacks []func() + err := fromScalarWithAllocator(sc, val, mem, &rollbacks) + if err != nil { + for i := len(rollbacks) - 1; i >= 0; i-- { + rollbacks[i]() + } + } + return err +} + +func fromScalarWithAllocator(sc *Struct, val interface{}, mem memory.Allocator, rollbacks *[]func()) error { if sc == nil || len(sc.Value) == 0 { return nil } @@ -83,7 +94,7 @@ func FromScalarWithAllocator(sc *Struct, val interface{}, mem memory.Allocator) if err != nil { return err } - if err := setFromScalar(fldVal, value.Field(i), mem); err != nil { + if err := setFromScalar(fldVal, value.Field(i), mem, rollbacks); err != nil { return err } } @@ -91,7 +102,7 @@ func FromScalarWithAllocator(sc *Struct, val interface{}, mem memory.Allocator) return nil } -func setFromScalar(s Scalar, v reflect.Value, mem memory.Allocator) error { +func setFromScalar(s Scalar, v reflect.Value, mem memory.Allocator, rollbacks *[]func()) error { if v.Type() == scalarType { if !s.IsValid() && s.DataType().ID() == arrow.NULL { v.Set(reflect.Zero(v.Type())) @@ -103,6 +114,12 @@ func setFromScalar(s Scalar, v reflect.Value, mem memory.Allocator) error { return err } v.Set(reflect.ValueOf(clone)) + *rollbacks = append(*rollbacks, func() { + if releasable, ok := clone.(interface{ Release() }); ok { + releasable.Release() + } + v.Set(reflect.Zero(v.Type())) + }) return nil } @@ -131,7 +148,7 @@ func setFromScalar(s Scalar, v reflect.Value, mem memory.Allocator) error { case ListScalar: return fromListScalar(s, v) case *Struct: - return FromScalarWithAllocator(s, v.Interface(), mem) + return fromScalarWithAllocator(s, v.Interface(), mem, rollbacks) default: if v.Type() == reflect.TypeOf(arrow.TimeUnit(0)) { v.Set(reflect.ValueOf(arrow.TimeUnit(s.value().(uint32)))) diff --git a/arrow/scalar/scalar_test.go b/arrow/scalar/scalar_test.go index bf2406766..051ce45f2 100644 --- a/arrow/scalar/scalar_test.go +++ b/arrow/scalar/scalar_test.go @@ -1253,6 +1253,43 @@ func TestMakeArrayFromScalar(t *testing.T) { } } +func TestMakeArrayFromDictionaryScalar(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + + dictValues, _, err := array.FromJSON(mem, arrow.PrimitiveTypes.Int64, strings.NewReader(`[10, 20]`)) + require.NoError(t, err) + defer dictValues.Release() + dictIndices, _, err := array.FromJSON(mem, arrow.PrimitiveTypes.Int8, strings.NewReader(`[0, 1]`)) + require.NoError(t, err) + defer dictIndices.Release() + dictType := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: arrow.PrimitiveTypes.Int64, + } + dictArray := array.NewDictionaryArray(dictType, dictIndices, dictValues) + defer dictArray.Release() + + dictScalar, err := scalar.GetScalar(dictArray, 0) + require.NoError(t, err) + + result, err := scalar.MakeArrayFromScalar(dictScalar, 2, mem) + require.NoError(t, err) + defer result.Release() + + actual, err := scalar.GetScalar(result, 1) + require.NoError(t, err) + defer actual.(scalar.Releasable).Release() + assert.True(t, scalar.Equals(dictScalar, actual)) + + structScalar, err := scalar.NewStructScalarWithNames([]scalar.Scalar{dictScalar}, []string{"value"}) + require.NoError(t, err) + defer structScalar.Release() + structArray, err := scalar.MakeArrayFromScalar(structScalar, 2, mem) + require.NoError(t, err) + defer structArray.Release() +} + func TestMakeArrayFromScalarRejectsNegativeLength(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) @@ -1302,6 +1339,11 @@ type scalarFieldWithUnsupportedSlice struct { Unsupported []float64 `compute:"unsupported"` } +type scalarFieldWithLaterListFailure struct { + Value scalar.Scalar `compute:"value"` + Invalid bool `compute:"invalid"` +} + type zeroingAllocator struct{} func (*zeroingAllocator) Allocate(size int) []byte { return make([]byte, size) } @@ -1352,6 +1394,31 @@ func TestToScalarReleasesFieldsWhenLaterFieldFails(t *testing.T) { original.Release() } +func TestFromScalarWithAllocatorReleasesFieldsWhenLaterFieldFails(t *testing.T) { + mem := memory.NewCheckedAllocator(&zeroingAllocator{}) + defer mem.AssertSize(t, 0) + + data := mem.Allocate(2) + copy(data, []byte("10")) + buffer := memory.NewBufferWithAllocator(data, mem) + value := scalar.NewBinaryScalar(buffer, arrow.BinaryTypes.Binary) + buffer.Release() + + listValues, _, err := array.FromJSON(mem, arrow.PrimitiveTypes.Int32, strings.NewReader(`[1]`)) + require.NoError(t, err) + defer listValues.Release() + invalid := scalar.NewListScalar(listValues) + encoded, err := scalar.NewStructScalarWithNames( + []scalar.Scalar{value, invalid}, []string{"value", "invalid"}) + require.NoError(t, err) + defer encoded.Release() + + var decoded scalarFieldWithLaterListFailure + err = scalar.FromScalarWithAllocator(encoded, &decoded, mem) + require.Error(t, err) + assert.Nil(t, decoded.Value) +} + func TestGetScalarBinaryValueOwnsArrayBytes(t *testing.T) { mem := memory.NewCheckedAllocator(&zeroingAllocator{}) defer mem.AssertSize(t, 0) From 3e3bf8b3b579abe69336c544b0a11e84ff73393d Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 16:46:29 +0200 Subject: [PATCH 11/17] fix(compute): harden cumulative option serialization --- arrow/compute/expression.go | 2 ++ arrow/compute/expression_test.go | 12 ++++++++++++ arrow/ipc/file_reader.go | 7 +++++-- arrow/ipc/file_writer.go | 9 +++++++++ arrow/scalar/scalar.go | 20 ++++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/arrow/compute/expression.go b/arrow/compute/expression.go index 8ec332a82..b06a65cd1 100644 --- a/arrow/compute/expression.go +++ b/arrow/compute/expression.go @@ -400,6 +400,8 @@ func equalFunctionOptions(lhs, rhs FunctionOptions) bool { func cumulativeOptions(opts FunctionOptions) (*CumulativeOptions, bool) { switch opts := opts.(type) { + case CumulativeOptions: + return &opts, true case *CumulativeOptions: return opts, true default: diff --git a/arrow/compute/expression_test.go b/arrow/compute/expression_test.go index b132ae47c..7f3f85b56 100644 --- a/arrow/compute/expression_test.go +++ b/arrow/compute/expression_test.go @@ -215,6 +215,18 @@ func TestCumulativeOptionsEquality(t *testing.T) { } +func TestCumulativeOptionsValueAndPointerEquality(t *testing.T) { + value := compute.CumulativeOptions{Start: scalar.NewInt32Scalar(10)} + pointer := &compute.CumulativeOptions{Start: scalar.NewInt32Scalar(10)} + + left := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, value) + right := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, pointer) + defer left.Release() + defer right.Release() + + assert.True(t, left.Equals(right)) +} + func TestCumulativeOptionsRelease(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) diff --git a/arrow/ipc/file_reader.go b/arrow/ipc/file_reader.go index 3128a150e..4c543b916 100644 --- a/arrow/ipc/file_reader.go +++ b/arrow/ipc/file_reader.go @@ -330,8 +330,10 @@ func (f *FileReader) readSchema(ensureNativeEndian bool) error { if err != nil { return err } - - kind, err = readDictionary(&f.memo, msg.meta, msg.body, f.swapEndianness, f.mem) + kind, err = func() (dictutils.Kind, error) { + defer msg.Release() + return readDictionary(&f.memo, msg.meta, msg.body, f.swapEndianness, f.mem) + }() if err != nil { return err } @@ -408,6 +410,7 @@ func (f *FileReader) Close() error { f.record.Release() f.record = nil } + f.memo.Clear() return nil } diff --git a/arrow/ipc/file_writer.go b/arrow/ipc/file_writer.go index 0d970ba5a..9bbd8556a 100644 --- a/arrow/ipc/file_writer.go +++ b/arrow/ipc/file_writer.go @@ -285,6 +285,8 @@ func NewFileWriter(w io.Writer, opts ...Option) (*FileWriter, error) { } func (f *FileWriter) Close() error { + defer f.releaseDictionaries() + err := f.checkStarted() if err != nil { return fmt.Errorf("arrow/ipc: could not write empty file: %w", err) @@ -303,6 +305,13 @@ func (f *FileWriter) Close() error { return nil } +func (f *FileWriter) releaseDictionaries() { + for _, d := range f.lastWrittenDicts { + d.Release() + } + f.lastWrittenDicts = nil +} + func (f *FileWriter) Write(rec arrow.RecordBatch) error { schema := rec.Schema() if schema == nil || !schema.Equal(f.schema) { diff --git a/arrow/scalar/scalar.go b/arrow/scalar/scalar.go index 6bdccb228..8cc0533fe 100644 --- a/arrow/scalar/scalar.go +++ b/arrow/scalar/scalar.go @@ -864,6 +864,26 @@ func MakeArrayFromScalar(sc Scalar, length int, mem memory.Allocator) (arrow.Arr data := finishFixedWidth(arrow.Decimal256Traits.CastToBytes([]decimal256.Num{s.Value})) defer data.Release() return array.MakeFromData(data), nil + case *Dictionary: + if err := s.Validate(); err != nil { + return nil, err + } + + indices, err := MakeArrayFromScalar(s.Value.Index, length, mem) + if err != nil { + return nil, err + } + defer indices.Release() + + // Copy the dictionary values as well as the indices so the resulting + // scalar does not retain memory owned by the source dictionary. + dict, err := array.Concatenate([]arrow.Array{s.Value.Dict}, mem) + if err != nil { + return nil, err + } + defer dict.Release() + + return array.NewDictionaryArray(s.DataType(), indices, dict), nil case PrimitiveScalar: data := finishFixedWidth(s.Data()) defer data.Release() From d9ebb2bda8e6dac02fd0da47a5358fa270a0d975 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 21:59:48 +0200 Subject: [PATCH 12/17] fix(compute): harden cumulative option ownership --- arrow/compute/expression_test.go | 27 +++++++++++++------ .../internal/kernels/vector_cumulative.go | 3 +-- arrow/compute/vector_cumulative_test.go | 20 ++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/arrow/compute/expression_test.go b/arrow/compute/expression_test.go index 7f3f85b56..d3e695923 100644 --- a/arrow/compute/expression_test.go +++ b/arrow/compute/expression_test.go @@ -231,14 +231,25 @@ func TestCumulativeOptionsRelease(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) - data := mem.Allocate(2) - copy(data, []byte("10")) - buffer := memory.NewBufferWithAllocator(data, mem) - start := scalar.NewBinaryScalar(buffer, arrow.BinaryTypes.Binary) - buffer.Release() - - expr := compute.NewCall("cumulative_sum", nil, &compute.CumulativeOptions{Start: start}) - expr.Release() + newStart := func() scalar.Scalar { + data := mem.Allocate(2) + copy(data, []byte("10")) + buffer := memory.NewBufferWithAllocator(data, mem) + start := scalar.NewBinaryScalar(buffer, arrow.BinaryTypes.Binary) + buffer.Release() + return start + } + + t.Run("pointer options", func(t *testing.T) { + expr := compute.NewCall("cumulative_sum", nil, + &compute.CumulativeOptions{Start: newStart()}) + expr.Release() + }) + t.Run("value options", func(t *testing.T) { + expr := compute.NewCall("cumulative_sum", nil, + compute.CumulativeOptions{Start: newStart()}) + expr.Release() + }) } func TestExpressionHashing(t *testing.T) { diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index 18e66c088..92561d38c 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -39,7 +39,7 @@ type CumulativeOptions struct { func (CumulativeOptions) TypeName() string { return "CumulativeOptions" } -func (opts *CumulativeOptions) Release() { +func (opts CumulativeOptions) Release() { if opts.Start == nil { return } @@ -47,7 +47,6 @@ func (opts *CumulativeOptions) Release() { if releasable, ok := opts.Start.(interface{ Release() }); ok { releasable.Release() } - opts.Start = nil } type cumulativeSumState[T arrow.NumericType] struct { diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 557cc020f..c44c6f719 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -411,6 +411,26 @@ func TestCumulativeSumChunked(t *testing.T) { } +func TestCumulativeSumEmptyChunkedInput(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + input := arrow.NewChunked(arrow.PrimitiveTypes.Int32, nil) + defer input.Release() + + result, err := compute.CumulativeSum( + context.Background(), + compute.CumulativeOptions{}, + &compute.ChunkedDatum{Value: input}, + ) + require.NoError(t, err) + defer result.Release() + + chunked, ok := result.(*compute.ChunkedDatum) + require.True(t, ok) + assert.Empty(t, chunked.Value.Chunks()) + assert.Equal(t, int64(0), result.Len()) +} + func TestCumulativeSumChunkedOutputIgnoresChunkSizeAndEmptyChunks(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) From a00fcb897c939e57dad3f86c360a5a71eed520da Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 8 Aug 2026 22:22:40 +0200 Subject: [PATCH 13/17] test(compute): cover sliced cumulative sum nulls --- arrow/compute/vector_cumulative_test.go | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index c44c6f719..d55a4983d 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -103,6 +103,33 @@ func TestCumulativeSumAdditionalInputs(t *testing.T) { require.NoError(t, err) defer result.Release() assertDatumsEqual(t, &compute.ArrayDatum{Value: slicedExpected.Data()}, result, nil, nil) + + t.Run("sliced nulls", func(t *testing.T) { + fullInput := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[9, null, 2, 3, 99]`) + defer fullInput.Release() + slicedInput := array.NewSlice(fullInput, 1, 4) + defer slicedInput.Release() + + tests := []struct { + name string + opts compute.CumulativeOptions + want string + }{ + {name: "propagate nulls", want: `[null, null, null]`}, + {name: "skip nulls", opts: compute.CumulativeOptions{SkipNulls: true}, want: `[null, 2, 5]`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, tc.want) + defer expected.Release() + + result, err := compute.CumulativeSum(ctx, tc.opts, &compute.ArrayDatum{Value: slicedInput.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + } + }) } func TestCumulativeSumNullsAndStart(t *testing.T) { From 4f5fa333f3d95f7c65ba36e4580c275bfaf40d4a Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 9 Aug 2026 01:05:00 +0200 Subject: [PATCH 14/17] perf(compute): specialize cumulative sum hot paths --- .../internal/kernels/vector_cumulative.go | 113 ++++++++++++++--- arrow/compute/vector_cumulative_bench_test.go | 119 ++++++++++++++++++ arrow/compute/vector_cumulative_test.go | 57 +++++++++ 3 files changed, 273 insertions(+), 16 deletions(-) create mode 100644 arrow/compute/vector_cumulative_bench_test.go diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index 92561d38c..c94eb2582 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -211,23 +211,81 @@ func prepareCumulativeOutput[T arrow.NumericType](ctx *exec.KernelCtx, out *exec } } -func cumulativeSumSpans[T arrow.NumericType](ctx *exec.KernelCtx, state *cumulativeSumState[T], inputs []*exec.ArraySpan, out *exec.ExecResult, needsValidity bool) error { - prepareCumulativeOutput[T](ctx, out, needsValidity) - values := exec.GetSpanValues[T](out, 1) +func cumulativeSumNoNulls[T arrow.NumericType](state *cumulativeSumState[T], inputs []*exec.ArraySpan, values []T) { + var outputOffset int64 + current := state.current + for _, input := range inputs { + inputValues := exec.GetSpanValues[T](input, 1) + for i := int64(0); i < input.Len; i++ { + current += inputValues[i] + values[outputOffset+i] = current + } + outputOffset += input.Len + } + state.current = current +} +func cumulativeSumNoNullsChecked[T arrow.NumericType](state *cumulativeSumState[T], inputs []*exec.ArraySpan, values []T) error { + var outputOffset int64 + current := state.current + for _, input := range inputs { + inputValues := exec.GetSpanValues[T](input, 1) + for i := int64(0); i < input.Len; i++ { + var err error + current, err = state.add(current, inputValues[i]) + if err != nil { + return err + } + values[outputOffset+i] = current + } + outputOffset += input.Len + } + state.current = current + return nil +} + +func cumulativeSumWithNulls[T arrow.NumericType](state *cumulativeSumState[T], inputs []*exec.ArraySpan, values []T, validity []byte) int64 { var ( nulls int64 outputOffset int64 ) + current := state.current for _, input := range inputs { inputValues := exec.GetSpanValues[T](input, 1) for i := int64(0); i < input.Len; i++ { valid := len(input.Buffers[0].Buf) == 0 || bitutil.BitIsSet(input.Buffers[0].Buf, int(input.Offset+i)) outputIndex := outputOffset + i if !valid || state.encounteredNull { - if needsValidity { - bitutil.ClearBit(out.Buffers[0].Buf, int(outputIndex)) + bitutil.ClearBit(validity, int(outputIndex)) + nulls++ + if !valid && !state.skipNulls { + state.encounteredNull = true } + continue + } + + current += inputValues[i] + values[outputIndex] = current + } + outputOffset += input.Len + } + state.current = current + return nulls +} + +func cumulativeSumWithNullsChecked[T arrow.NumericType](state *cumulativeSumState[T], inputs []*exec.ArraySpan, values []T, validity []byte) (int64, error) { + var ( + nulls int64 + outputOffset int64 + ) + current := state.current + for _, input := range inputs { + inputValues := exec.GetSpanValues[T](input, 1) + for i := int64(0); i < input.Len; i++ { + valid := len(input.Buffers[0].Buf) == 0 || bitutil.BitIsSet(input.Buffers[0].Buf, int(input.Offset+i)) + outputIndex := outputOffset + i + if !valid || state.encounteredNull { + bitutil.ClearBit(validity, int(outputIndex)) nulls++ if !valid && !state.skipNulls { state.encounteredNull = true @@ -235,25 +293,48 @@ func cumulativeSumSpans[T arrow.NumericType](ctx *exec.KernelCtx, state *cumulat continue } - current := state.current - value := inputValues[i] var err error - if state.checked { - current, err = state.add(current, value) - } else { - current += value - } + current, err = state.add(current, inputValues[i]) if err != nil { - out.Release() - return err + return nulls, err } - - state.current = current values[outputIndex] = current } outputOffset += input.Len } + state.current = current + return nulls, nil +} +func cumulativeSumSpans[T arrow.NumericType](ctx *exec.KernelCtx, state *cumulativeSumState[T], inputs []*exec.ArraySpan, out *exec.ExecResult, needsValidity bool) error { + prepareCumulativeOutput[T](ctx, out, needsValidity) + values := exec.GetSpanValues[T](out, 1) + + if !needsValidity && !state.encounteredNull { + if state.checked { + if err := cumulativeSumNoNullsChecked(state, inputs, values); err != nil { + out.Release() + return err + } + } else { + cumulativeSumNoNulls(state, inputs, values) + } + return nil + } + + var ( + nulls int64 + err error + ) + if state.checked { + nulls, err = cumulativeSumWithNullsChecked(state, inputs, values, out.Buffers[0].Buf) + } else { + nulls = cumulativeSumWithNulls(state, inputs, values, out.Buffers[0].Buf) + } + if err != nil { + out.Release() + return err + } out.Nulls = nulls return nil } diff --git a/arrow/compute/vector_cumulative_bench_test.go b/arrow/compute/vector_cumulative_bench_test.go new file mode 100644 index 000000000..206baa6bc --- /dev/null +++ b/arrow/compute/vector_cumulative_bench_test.go @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 + +package compute_test + +import ( + "context" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/arrow-go/v18/arrow/memory" +) + +func benchmarkInt64Array(b *testing.B, length int, withNulls bool) arrow.Array { + b.Helper() + builder := array.NewInt64Builder(memory.DefaultAllocator) + builder.Reserve(length) + for i := 0; i < length; i++ { + if withNulls && i%100 == 0 { + builder.AppendNull() + } else { + builder.Append(1) + } + } + result := builder.NewArray() + builder.Release() + return result +} + +func benchmarkFloat64Array(b *testing.B, length int) arrow.Array { + b.Helper() + builder := array.NewFloat64Builder(memory.DefaultAllocator) + builder.Reserve(length) + for i := 0; i < length; i++ { + builder.Append(1) + } + result := builder.NewArray() + builder.Release() + return result +} + +func benchmarkInt64Chunked(b *testing.B, chunks, length int) *arrow.Chunked { + b.Helper() + values := make([]arrow.Array, chunks) + for i := range values { + values[i] = benchmarkInt64Array(b, length, false) + } + result := arrow.NewChunked(arrow.PrimitiveTypes.Int64, values) + for _, value := range values { + value.Release() + } + return result +} + +func BenchmarkCumulativeSum(b *testing.B) { + const length = 10_000_000 + + intInput := benchmarkInt64Array(b, length, false) + defer intInput.Release() + nullInput := benchmarkInt64Array(b, length, true) + defer nullInput.Release() + floatInput := benchmarkFloat64Array(b, length) + defer floatInput.Release() + chunkedInput := benchmarkInt64Chunked(b, 100, length/100) + defer chunkedInput.Release() + + ctx := context.Background() + tests := []struct { + name string + input compute.Datum + opts compute.CumulativeOptions + checked bool + }{ + {name: "int64", input: &compute.ArrayDatum{Value: intInput.Data()}}, + {name: "int64_checked", input: &compute.ArrayDatum{Value: intInput.Data()}, checked: true}, + {name: "int64_1pct_nulls_skip", input: &compute.ArrayDatum{Value: nullInput.Data()}, opts: compute.CumulativeOptions{SkipNulls: true}}, + {name: "float64", input: &compute.ArrayDatum{Value: floatInput.Data()}}, + {name: "int64_chunked", input: &compute.ChunkedDatum{Value: chunkedInput}}, + } + + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var ( + result compute.Datum + err error + ) + if tc.checked { + result, err = compute.CumulativeSumChecked(ctx, tc.opts, tc.input) + } else { + result, err = compute.CumulativeSum(ctx, tc.opts, tc.input) + } + if err != nil { + b.Fatal(err) + } + result.Release() + } + }) + } +} diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index d55a4983d..6c6389c47 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -302,6 +302,29 @@ func TestCumulativeSumStartScalarConversions(t *testing.T) { assert.ErrorIs(t, err, arrow.ErrInvalid) } +func TestCumulativeSumDoesNotTakeStartOwnership(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + + data := mem.Allocate(2) + copy(data, "10") + buffer := memory.NewBufferWithAllocator(data, mem) + start := scalar.NewBinaryScalar(buffer, arrow.BinaryTypes.Binary) + buffer.Release() + + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1]`) + defer input.Release() + + result, err := compute.CumulativeSum(ctx, compute.CumulativeOptions{Start: start}, + &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + result.Release() + + assert.Equal(t, "10", string(start.Data())) + start.Release() +} + func TestCumulativeSumDecimalStarts(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -648,3 +671,37 @@ func TestCumulativeSumCheckedIntegerOverflow(t *testing.T) { }) } } + +func TestCumulativeSumCheckedIntegerBoundaries(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + + tests := []struct { + name string + typ arrow.DataType + input string + expected string + }{ + {name: "int8 positive", typ: arrow.PrimitiveTypes.Int8, input: `[126, 1]`, expected: `[126, 127]`}, + {name: "int8 negative", typ: arrow.PrimitiveTypes.Int8, input: `[-127, -1]`, expected: `[-127, -128]`}, + {name: "int64 negative", typ: arrow.PrimitiveTypes.Int64, input: `[-9223372036854775807, -1]`, expected: `[-9223372036854775807, -9223372036854775808]`}, + {name: "uint8", typ: arrow.PrimitiveTypes.Uint8, input: `[254, 1]`, expected: `[254, 255]`}, + {name: "uint64", typ: arrow.PrimitiveTypes.Uint64, input: `[18446744073709551614, 1]`, expected: `[18446744073709551614, 18446744073709551615]`}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + input := cumulativeInput(t, mem, tc.typ, tc.input) + defer input.Release() + expected := cumulativeInput(t, mem, tc.typ, tc.expected) + defer expected.Release() + + result, err := compute.CumulativeSumChecked(ctx, compute.CumulativeOptions{}, + &compute.ArrayDatum{Value: input.Data()}) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + } +} From 937be428045b26998de4656ff52a23f9ec2522d0 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 9 Aug 2026 01:05:05 +0200 Subject: [PATCH 15/17] fix(arrow): retain scalar and reader resources --- arrow/ipc/file_reader.go | 2 ++ arrow/ipc/file_reader_internal_test.go | 32 +++++++++++++++++ arrow/scalar/scalar.go | 14 ++++++-- arrow/scalar/scalar_test.go | 49 ++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 3 deletions(-) diff --git a/arrow/ipc/file_reader.go b/arrow/ipc/file_reader.go index 4c543b916..a7602e88c 100644 --- a/arrow/ipc/file_reader.go +++ b/arrow/ipc/file_reader.go @@ -239,6 +239,7 @@ func NewMappedFileReader(data []byte, opts ...Option) (*FileReader, error) { ) if err := f.init(cfg); err != nil { + _ = f.Close() return nil, err } return &f, nil @@ -260,6 +261,7 @@ func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error) { ) if err := f.init(cfg); err != nil { + _ = f.Close() return nil, err } return &f, nil diff --git a/arrow/ipc/file_reader_internal_test.go b/arrow/ipc/file_reader_internal_test.go index 93ab1c39b..35ff72d4c 100644 --- a/arrow/ipc/file_reader_internal_test.go +++ b/arrow/ipc/file_reader_internal_test.go @@ -18,14 +18,46 @@ package ipc import ( + "bytes" "testing" "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/arrow-go/v18/arrow/internal/dictutils" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/stretchr/testify/require" ) +func TestFileReaderInitFailureReleasesDictionaries(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + schema := arrow.NewSchema([]arrow.Field{{ + Name: "value", + Type: &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: arrow.BinaryTypes.String, + }, + }}, nil) + builder := array.NewRecordBuilder(mem, schema) + column := builder.Field(0).(*array.BinaryDictionaryBuilder) + column.Append([]byte("value")) + record := builder.NewRecordBatch() + defer record.Release() + defer builder.Release() + + var buf bytes.Buffer + writer, err := NewFileWriter(&buf, WithAllocator(mem), WithSchema(schema)) + require.NoError(t, err) + require.NoError(t, writer.Write(record)) + require.NoError(t, writer.Close()) + + wrongSchema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int32}}, nil) + reader, err := NewFileReader(bytes.NewReader(buf.Bytes()), WithAllocator(mem), WithSchema(wrongSchema)) + require.Error(t, err) + require.Nil(t, reader) +} + func TestLoadRecordBatchReturnsMalformedMetadataErrors(t *testing.T) { meta := memory.NewBufferBytes([]byte{0}) defer meta.Release() diff --git a/arrow/scalar/scalar.go b/arrow/scalar/scalar.go index 8cc0533fe..514939974 100644 --- a/arrow/scalar/scalar.go +++ b/arrow/scalar/scalar.go @@ -589,11 +589,11 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) { switch arr := arr.(type) { case *array.Binary: - buf := memory.NewBufferBytes(append([]byte(nil), arr.Value(idx)...)) + buf := scalarValueBuffer(arr.Data().Buffers()[2], arr.ValueOffset(idx), arr.ValueLen(idx)) defer buf.Release() return NewBinaryScalar(buf, arr.DataType()), nil case *array.LargeBinary: - buf := memory.NewBufferBytes(append([]byte(nil), arr.Value(idx)...)) + buf := scalarValueBuffer(arr.Data().Buffers()[2], int(arr.ValueOffset(idx)), arr.ValueLen(idx)) defer buf.Release() return NewLargeBinaryScalar(buf), nil case *array.Boolean: @@ -617,7 +617,8 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) { } return NewExtensionScalar(storage, arr.DataType()), nil case *array.FixedSizeBinary: - buf := memory.NewBufferBytes(append([]byte(nil), arr.Value(idx)...)) + width := arr.DataType().(*arrow.FixedSizeBinaryType).ByteWidth + buf := scalarValueBuffer(arr.Data().Buffers()[1], (arr.Data().Offset()+idx)*width, width) defer buf.Release() return NewFixedSizeBinaryScalar(buf, arr.DataType()), nil case *array.FixedSizeList: @@ -752,6 +753,13 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) { return nil, fmt.Errorf("cannot create scalar from array of type %s", arr.DataType()) } +func scalarValueBuffer(values *memory.Buffer, offset, length int) *memory.Buffer { + if values == nil { + return memory.NewBufferBytes(nil) + } + return memory.SliceBuffer(values, offset, length) +} + // MakeArrayOfNull creates an array of size length which is all null of the given data type. // // Deprecated: Use array.MakeArrayOfNull diff --git a/arrow/scalar/scalar_test.go b/arrow/scalar/scalar_test.go index 051ce45f2..612fa3691 100644 --- a/arrow/scalar/scalar_test.go +++ b/arrow/scalar/scalar_test.go @@ -1438,6 +1438,55 @@ func TestGetScalarBinaryValueOwnsArrayBytes(t *testing.T) { binaryValue.Release() } +func TestGetScalarBinaryValueOwnsAllBinaryArrayBytes(t *testing.T) { + mem := memory.NewCheckedAllocator(&zeroingAllocator{}) + defer mem.AssertSize(t, 0) + + tests := []struct { + name string + build func() arrow.Array + want string + }{ + { + name: "large binary", + build: func() arrow.Array { + bldr := array.NewBinaryBuilder(mem, arrow.BinaryTypes.LargeBinary) + bldr.Append([]byte("large")) + arr := bldr.NewArray() + bldr.Release() + return arr + }, + want: "large", + }, + { + name: "fixed size binary", + build: func() arrow.Array { + typ := &arrow.FixedSizeBinaryType{ByteWidth: 5} + bldr := array.NewFixedSizeBinaryBuilder(mem, typ) + bldr.Append([]byte("fixed")) + arr := bldr.NewArray() + bldr.Release() + return arr + }, + want: "fixed", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + arr := tc.build() + value, err := scalar.GetScalar(arr, 0) + require.NoError(t, err) + arr.Release() + + binaryValue, ok := value.(scalar.BinaryScalar) + require.True(t, ok) + assert.Equal(t, tc.want, string(binaryValue.Data())) + binaryValue.Release() + }) + } +} + func TestToScalar(t *testing.T) { ot := &OptionValTest{ToType: arrow.BinaryTypes.String, Allow: true} sc, err := scalar.ToScalar(ot, memory.DefaultAllocator) From 67489e1dc3364bcd1043d60cc5f62f2d5f7e302c Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 9 Aug 2026 02:45:29 +0200 Subject: [PATCH 16/17] fix(compute): retain cumulative expression starts --- arrow/compute/expression.go | 37 ++++++- arrow/compute/expression_test.go | 10 +- .../internal/kernels/vector_cumulative.go | 2 +- arrow/compute/vector_cumulative_bench_test.go | 5 + arrow/compute/vector_cumulative_test.go | 96 +++++++++++++++++++ 5 files changed, 144 insertions(+), 6 deletions(-) diff --git a/arrow/compute/expression.go b/arrow/compute/expression.go index b06a65cd1..0d7c23adf 100644 --- a/arrow/compute/expression.go +++ b/arrow/compute/expression.go @@ -602,9 +602,34 @@ func NewFieldRef(field string) Expression { } // NewCall constructs an expression that represents a specific function call with -// the given arguments and options. +// the given arguments and options. Cumulative start scalars are retained for +// the lifetime of the expression. func NewCall(name string, args []Expression, opts FunctionOptions) Expression { - return &Call{funcName: name, args: args, options: opts} + return &Call{funcName: name, args: args, options: cloneExpressionOptions(opts)} +} + +func cloneExpressionOptions(opts FunctionOptions) FunctionOptions { + switch opts := opts.(type) { + case CumulativeOptions: + opts.Start = retainExpressionScalar(opts.Start) + return opts + case *CumulativeOptions: + if opts == nil { + return nil + } + cloned := *opts + cloned.Start = retainExpressionScalar(cloned.Start) + return cloned + default: + return opts + } +} + +func retainExpressionScalar(value scalar.Scalar) scalar.Scalar { + if releasable, ok := value.(scalar.Releasable); ok { + releasable.Retain() + } + return value } // Project is shorthand for `make_struct` to produce a record batch output @@ -923,7 +948,13 @@ func DeserializeExpr(mem memory.Allocator, buf *memory.Buffer) (Expression, erro opts = optionsVal.(FunctionOptions) } index += 2 - return NewCall(val, args, opts), nil + expr := NewCall(val, args, opts) + if _, ok := cumulativeOptions(opts); ok { + if r, ok := opts.(releasable); ok { + r.Release() + } + } + return expr, nil } arg, err := getone() diff --git a/arrow/compute/expression_test.go b/arrow/compute/expression_test.go index d3e695923..9f4d720e1 100644 --- a/arrow/compute/expression_test.go +++ b/arrow/compute/expression_test.go @@ -241,14 +241,20 @@ func TestCumulativeOptionsRelease(t *testing.T) { } t.Run("pointer options", func(t *testing.T) { + start := newStart() expr := compute.NewCall("cumulative_sum", nil, - &compute.CumulativeOptions{Start: newStart()}) + &compute.CumulativeOptions{Start: start}) expr.Release() + assert.Equal(t, "10", string(start.(scalar.BinaryScalar).Data())) + start.(scalar.Releasable).Release() }) t.Run("value options", func(t *testing.T) { + start := newStart() expr := compute.NewCall("cumulative_sum", nil, - compute.CumulativeOptions{Start: newStart()}) + compute.CumulativeOptions{Start: start}) expr.Release() + assert.Equal(t, "10", string(start.(scalar.BinaryScalar).Data())) + start.(scalar.Releasable).Release() }) } diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index c94eb2582..258ef8b97 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -30,7 +30,7 @@ import ( // CumulativeOptions controls cumulative operations. type CumulativeOptions struct { // Start is the initial value. A nil value uses the zero value for the - // input type. + // input type. A non-nil null scalar is invalid. Start scalar.Scalar `compute:"start"` // SkipNulls controls whether a null input stops the cumulative operation. // Null positions are still null in the output when this is true. diff --git a/arrow/compute/vector_cumulative_bench_test.go b/arrow/compute/vector_cumulative_bench_test.go index 206baa6bc..9e8f00cdc 100644 --- a/arrow/compute/vector_cumulative_bench_test.go +++ b/arrow/compute/vector_cumulative_bench_test.go @@ -97,6 +97,11 @@ func BenchmarkCumulativeSum(b *testing.B) { for _, tc := range tests { b.Run(tc.name, func(b *testing.B) { + if input, ok := tc.input.(compute.ArrayLikeDatum); ok { + if typ, ok := input.Type().(arrow.FixedWidthDataType); ok { + b.SetBytes(int64(input.Len()) * int64(typ.Bytes())) + } + } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 6c6389c47..3c5d4791a 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -132,6 +132,72 @@ func TestCumulativeSumAdditionalInputs(t *testing.T) { }) } +func TestCumulativeSumNullScalarInput(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + + types := []arrow.DataType{ + arrow.PrimitiveTypes.Int8, + arrow.PrimitiveTypes.Int16, + arrow.PrimitiveTypes.Int32, + arrow.PrimitiveTypes.Int64, + arrow.PrimitiveTypes.Uint8, + arrow.PrimitiveTypes.Uint16, + arrow.PrimitiveTypes.Uint32, + arrow.PrimitiveTypes.Uint64, + arrow.PrimitiveTypes.Float32, + arrow.PrimitiveTypes.Float64, + } + functions := []struct { + name string + run func(context.Context, compute.CumulativeOptions, compute.Datum) (compute.Datum, error) + }{ + {name: "unchecked", run: compute.CumulativeSum}, + {name: "checked", run: compute.CumulativeSumChecked}, + } + options := []struct { + name string + start bool + skip bool + }{ + {name: "no_start_no_skip"}, + {name: "no_start_skip", skip: true}, + {name: "start_no_skip", start: true}, + {name: "start_skip", start: true, skip: true}, + } + + for _, typ := range types { + start, err := scalar.ParseScalar(typ, "10") + require.NoError(t, err) + if releasable, ok := start.(scalar.Releasable); ok { + defer releasable.Release() + } + + for _, fn := range functions { + for _, tc := range options { + t.Run(typ.Name()+"/"+fn.name+"/"+tc.name, func(t *testing.T) { + input := compute.NewDatum(scalar.MakeNullScalar(typ)) + defer input.Release() + + opts := compute.CumulativeOptions{SkipNulls: tc.skip} + if tc.start { + opts.Start = start + } + + result, err := fn.run(ctx, opts, input) + require.NoError(t, err) + defer result.Release() + + expected := cumulativeInput(t, mem, typ, `[null]`) + defer expected.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) + }) + } + } + } +} + func TestCumulativeSumNullsAndStart(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -171,6 +237,32 @@ func TestCumulativeSumNullsAndStart(t *testing.T) { } +func TestCumulativeSumRejectsTypedNullStart(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1]`) + defer input.Release() + + for _, tc := range []struct { + name string + run func(context.Context, compute.CumulativeOptions, compute.Datum) (compute.Datum, error) + }{ + {name: "unchecked", run: compute.CumulativeSum}, + {name: "checked", run: compute.CumulativeSumChecked}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := tc.run(ctx, compute.CumulativeOptions{ + Start: scalar.MakeNullScalar(arrow.PrimitiveTypes.Int32), + }, &compute.ArrayDatum{Value: input.Data()}) + if result != nil { + result.Release() + } + assert.ErrorIs(t, err, arrow.ErrInvalid) + }) + } +} + func TestCumulativeSumStartSafeCast(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -401,6 +493,9 @@ func TestCumulativeOptionsSerialization(t *testing.T) { assert.NotEmpty(t, roundTripped.String()) roundTripped.Release() expr.Release() + if releasable, ok := tc.start.(scalar.Releasable); ok { + releasable.Release() + } }) } } @@ -422,6 +517,7 @@ func TestCumulativeOptionsDictionarySerialization(t *testing.T) { start, err := scalar.GetScalar(dict, 0) require.NoError(t, err) + defer start.(scalar.Releasable).Release() expr := compute.NewCall("cumulative_sum", []compute.Expression{compute.NewFieldRef("values")}, &compute.CumulativeOptions{Start: start}) defer expr.Release() From 67f77c62c6d64e90e2f2d01f99cc963a836dd6d3 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 9 Aug 2026 03:00:13 +0200 Subject: [PATCH 17/17] fix(compute): accept value cumulative options --- .../internal/kernels/vector_cumulative.go | 16 +++++++++------ arrow/compute/vector_cumulative_test.go | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/arrow/compute/internal/kernels/vector_cumulative.go b/arrow/compute/internal/kernels/vector_cumulative.go index 258ef8b97..006a1261e 100644 --- a/arrow/compute/internal/kernels/vector_cumulative.go +++ b/arrow/compute/internal/kernels/vector_cumulative.go @@ -108,13 +108,17 @@ func cumulativeStartValue[T arrow.NumericType](ctx *exec.KernelCtx, cast ScalarC func initCumulativeSum[T arrow.NumericType](checked bool, cast ScalarCastFn) exec.KernelInitFn { return func(ctx *exec.KernelCtx, args exec.KernelInitArgs) (exec.KernelState, error) { - opts := &CumulativeOptions{} - if args.Options != nil { - var ok bool - opts, ok = args.Options.(*CumulativeOptions) - if !ok { - return nil, fmt.Errorf("%w: attempted to initialize cumulative sum from invalid function options", arrow.ErrInvalid) + opts := CumulativeOptions{} + switch value := args.Options.(type) { + case nil: + case CumulativeOptions: + opts = value + case *CumulativeOptions: + if value != nil { + opts = *value } + default: + return nil, fmt.Errorf("%w: attempted to initialize cumulative sum from invalid function options", arrow.ErrInvalid) } start, err := cumulativeStartValue[T](ctx, cast, opts.Start, args.Inputs[0]) diff --git a/arrow/compute/vector_cumulative_test.go b/arrow/compute/vector_cumulative_test.go index 3c5d4791a..8bbc853ef 100644 --- a/arrow/compute/vector_cumulative_test.go +++ b/arrow/compute/vector_cumulative_test.go @@ -54,6 +54,26 @@ func TestCumulativeSum(t *testing.T) { } +func TestCumulativeSumValueOptions(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(context.Background(), mem) + input := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 2, 3]`) + defer input.Release() + expected := cumulativeInput(t, mem, arrow.PrimitiveTypes.Int32, `[1, 3, 6]`) + defer expected.Release() + + result, err := compute.CallFunction( + ctx, + "cumulative_sum", + compute.CumulativeOptions{}, + &compute.ArrayDatum{Value: input.Data()}, + ) + require.NoError(t, err) + defer result.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, result, nil, nil) +} + func TestCumulativeSumAdditionalInputs(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0)