diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index ec80e27ba5..af105ff3fa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -992,12 +992,12 @@ public void testSumGroupByNullValue() throws IOException { verifySchema(response, schema("a", null, "bigint"), schema("age", null, "int")); verifyDataRows( response, - rows(isPushdownDisabled() ? null : 0, null), + rows(null, null), rows(32838, 28), rows(39225, 32), rows(4180, 33), rows(48086, 34), - rows(isPushdownDisabled() ? null : 0, 36)); + rows(null, 36)); } @Test @@ -1061,7 +1061,7 @@ public void testSumNull() throws IOException { + " ],\n" + " \"datarows\": [\n" + " [\n" - + (isPushdownDisabled() ? " null\n" : " 0\n") + + " null\n" + " ]\n" + " ],\n" + " \"total\": 1,\n" diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index b339f3f802..225f0b9a39 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -505,11 +505,7 @@ public void testSumWithNull() throws IOException { "source=%s | where age = 36 | stats sum(balance)", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("sum(balance)", null, "bigint")); - // TODO: Fix -- temporary workaround for the pushdown issue: - // The current pushdown implementation will return 0 for sum when getting null values as input. - // Returning null should be the expected behavior. - Integer expectedValue = isPushdownDisabled() ? null : 0; - verifyDataRows(response, rows(expectedValue)); + verifyDataRows(response, rows((Integer) null)); } @Test diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml index ae4cade06e..a1460bbbcd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($68)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"stats":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml index ef93b63ee8..64b35f5fb0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject(AdvEngineID=[$19], ResolutionWidth=[$80]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"stats":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index de1f6d31f0..74a8808480 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -6,4 +6,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):BIGINT], expr#3=[+($t0, $t2)], expr#4=[2], expr#5=[*($t1, $t4)], expr#6=[+($t0, $t5)], expr#7=[3], expr#8=[*($t1, $t7)], expr#9=[+($t0, $t8)], expr#10=[4], expr#11=[*($t1, $t10)], expr#12=[+($t0, $t11)], expr#13=[5], expr#14=[*($t1, $t13)], expr#15=[+($t0, $t14)], expr#16=[6], expr#17=[*($t1, $t16)], expr#18=[+($t0, $t17)], expr#19=[7], expr#20=[*($t1, $t19)], expr#21=[+($t0, $t20)], expr#22=[8], expr#23=[*($t1, $t22)], expr#24=[+($t0, $t23)], expr#25=[9], expr#26=[*($t1, $t25)], expr#27=[+($t0, $t26)], expr#28=[10], expr#29=[*($t1, $t28)], expr#30=[+($t0, $t29)], expr#31=[11], expr#32=[*($t1, $t31)], expr#33=[+($t0, $t32)], expr#34=[12], expr#35=[*($t1, $t34)], expr#36=[+($t0, $t35)], expr#37=[13], expr#38=[*($t1, $t37)], expr#39=[+($t0, $t38)], expr#40=[14], expr#41=[*($t1, $t40)], expr#42=[+($t0, $t41)], expr#43=[15], expr#44=[*($t1, $t43)], expr#45=[+($t0, $t44)], expr#46=[16], expr#47=[*($t1, $t46)], expr#48=[+($t0, $t47)], expr#49=[17], expr#50=[*($t1, $t49)], expr#51=[+($t0, $t50)], expr#52=[18], expr#53=[*($t1, $t52)], expr#54=[+($t0, $t53)], expr#55=[19], expr#56=[*($t1, $t55)], expr#57=[+($t0, $t56)], expr#58=[20], expr#59=[*($t1, $t58)], expr#60=[+($t0, $t59)], expr#61=[21], expr#62=[*($t1, $t61)], expr#63=[+($t0, $t62)], expr#64=[22], expr#65=[*($t1, $t64)], expr#66=[+($t0, $t65)], expr#67=[23], expr#68=[*($t1, $t67)], expr#69=[+($t0, $t68)], expr#70=[24], expr#71=[*($t1, $t70)], expr#72=[+($t0, $t71)], expr#73=[25], expr#74=[*($t1, $t73)], expr#75=[+($t0, $t74)], expr#76=[26], expr#77=[*($t1, $t76)], expr#78=[+($t0, $t77)], expr#79=[27], expr#80=[*($t1, $t79)], expr#81=[+($t0, $t80)], expr#82=[28], expr#83=[*($t1, $t82)], expr#84=[+($t0, $t83)], expr#85=[29], expr#86=[*($t1, $t85)], expr#87=[+($t0, $t86)], expr#88=[30], expr#89=[*($t1, $t88)], expr#90=[+($t0, $t89)], expr#91=[31], expr#92=[*($t1, $t91)], expr#93=[+($t0, $t92)], expr#94=[32], expr#95=[*($t1, $t94)], expr#96=[+($t0, $t95)], expr#97=[33], expr#98=[*($t1, $t97)], expr#99=[+($t0, $t98)], expr#100=[34], expr#101=[*($t1, $t100)], expr#102=[+($t0, $t101)], expr#103=[35], expr#104=[*($t1, $t103)], expr#105=[+($t0, $t104)], expr#106=[36], expr#107=[*($t1, $t106)], expr#108=[+($t0, $t107)], expr#109=[37], expr#110=[*($t1, $t109)], expr#111=[+($t0, $t110)], expr#112=[38], expr#113=[*($t1, $t112)], expr#114=[+($t0, $t113)], expr#115=[39], expr#116=[*($t1, $t115)], expr#117=[+($t0, $t116)], expr#118=[40], expr#119=[*($t1, $t118)], expr#120=[+($t0, $t119)], expr#121=[41], expr#122=[*($t1, $t121)], expr#123=[+($t0, $t122)], expr#124=[42], expr#125=[*($t1, $t124)], expr#126=[+($t0, $t125)], expr#127=[43], expr#128=[*($t1, $t127)], expr#129=[+($t0, $t128)], expr#130=[44], expr#131=[*($t1, $t130)], expr#132=[+($t0, $t131)], expr#133=[45], expr#134=[*($t1, $t133)], expr#135=[+($t0, $t134)], expr#136=[46], expr#137=[*($t1, $t136)], expr#138=[+($t0, $t137)], expr#139=[47], expr#140=[*($t1, $t139)], expr#141=[+($t0, $t140)], expr#142=[48], expr#143=[*($t1, $t142)], expr#144=[+($t0, $t143)], expr#145=[49], expr#146=[*($t1, $t145)], expr#147=[+($t0, $t146)], expr#148=[50], expr#149=[*($t1, $t148)], expr#150=[+($t0, $t149)], expr#151=[51], expr#152=[*($t1, $t151)], expr#153=[+($t0, $t152)], expr#154=[52], expr#155=[*($t1, $t154)], expr#156=[+($t0, $t155)], expr#157=[53], expr#158=[*($t1, $t157)], expr#159=[+($t0, $t158)], expr#160=[54], expr#161=[*($t1, $t160)], expr#162=[+($t0, $t161)], expr#163=[55], expr#164=[*($t1, $t163)], expr#165=[+($t0, $t164)], expr#166=[56], expr#167=[*($t1, $t166)], expr#168=[+($t0, $t167)], expr#169=[57], expr#170=[*($t1, $t169)], expr#171=[+($t0, $t170)], expr#172=[58], expr#173=[*($t1, $t172)], expr#174=[+($t0, $t173)], expr#175=[59], expr#176=[*($t1, $t175)], expr#177=[+($t0, $t176)], expr#178=[60], expr#179=[*($t1, $t178)], expr#180=[+($t0, $t179)], expr#181=[61], expr#182=[*($t1, $t181)], expr#183=[+($t0, $t182)], expr#184=[62], expr#185=[*($t1, $t184)], expr#186=[+($t0, $t185)], expr#187=[63], expr#188=[*($t1, $t187)], expr#189=[+($t0, $t188)], expr#190=[64], expr#191=[*($t1, $t190)], expr#192=[+($t0, $t191)], expr#193=[65], expr#194=[*($t1, $t193)], expr#195=[+($t0, $t194)], expr#196=[66], expr#197=[*($t1, $t196)], expr#198=[+($t0, $t197)], expr#199=[67], expr#200=[*($t1, $t199)], expr#201=[+($t0, $t200)], expr#202=[68], expr#203=[*($t1, $t202)], expr#204=[+($t0, $t203)], expr#205=[69], expr#206=[*($t1, $t205)], expr#207=[+($t0, $t206)], expr#208=[70], expr#209=[*($t1, $t208)], expr#210=[+($t0, $t209)], expr#211=[71], expr#212=[*($t1, $t211)], expr#213=[+($t0, $t212)], expr#214=[72], expr#215=[*($t1, $t214)], expr#216=[+($t0, $t215)], expr#217=[73], expr#218=[*($t1, $t217)], expr#219=[+($t0, $t218)], expr#220=[74], expr#221=[*($t1, $t220)], expr#222=[+($t0, $t221)], expr#223=[75], expr#224=[*($t1, $t223)], expr#225=[+($t0, $t224)], expr#226=[76], expr#227=[*($t1, $t226)], expr#228=[+($t0, $t227)], expr#229=[77], expr#230=[*($t1, $t229)], expr#231=[+($t0, $t230)], expr#232=[78], expr#233=[*($t1, $t232)], expr#234=[+($t0, $t233)], expr#235=[79], expr#236=[*($t1, $t235)], expr#237=[+($t0, $t236)], expr#238=[80], expr#239=[*($t1, $t238)], expr#240=[+($t0, $t239)], expr#241=[81], expr#242=[*($t1, $t241)], expr#243=[+($t0, $t242)], expr#244=[82], expr#245=[*($t1, $t244)], expr#246=[+($t0, $t245)], expr#247=[83], expr#248=[*($t1, $t247)], expr#249=[+($t0, $t248)], expr#250=[84], expr#251=[*($t1, $t250)], expr#252=[+($t0, $t251)], expr#253=[85], expr#254=[*($t1, $t253)], expr#255=[+($t0, $t254)], expr#256=[86], expr#257=[*($t1, $t256)], expr#258=[+($t0, $t257)], expr#259=[87], expr#260=[*($t1, $t259)], expr#261=[+($t0, $t260)], expr#262=[88], expr#263=[*($t1, $t262)], expr#264=[+($t0, $t263)], expr#265=[89], expr#266=[*($t1, $t265)], expr#267=[+($t0, $t266)], sum(ResolutionWidth)=[$t0], sum(ResolutionWidth+1)=[$t3], sum(ResolutionWidth+2)=[$t6], sum(ResolutionWidth+3)=[$t9], sum(ResolutionWidth+4)=[$t12], sum(ResolutionWidth+5)=[$t15], sum(ResolutionWidth+6)=[$t18], sum(ResolutionWidth+7)=[$t21], sum(ResolutionWidth+8)=[$t24], sum(ResolutionWidth+9)=[$t27], sum(ResolutionWidth+10)=[$t30], sum(ResolutionWidth+11)=[$t33], sum(ResolutionWidth+12)=[$t36], sum(ResolutionWidth+13)=[$t39], sum(ResolutionWidth+14)=[$t42], sum(ResolutionWidth+15)=[$t45], sum(ResolutionWidth+16)=[$t48], sum(ResolutionWidth+17)=[$t51], sum(ResolutionWidth+18)=[$t54], sum(ResolutionWidth+19)=[$t57], sum(ResolutionWidth+20)=[$t60], sum(ResolutionWidth+21)=[$t63], sum(ResolutionWidth+22)=[$t66], sum(ResolutionWidth+23)=[$t69], sum(ResolutionWidth+24)=[$t72], sum(ResolutionWidth+25)=[$t75], sum(ResolutionWidth+26)=[$t78], sum(ResolutionWidth+27)=[$t81], sum(ResolutionWidth+28)=[$t84], sum(ResolutionWidth+29)=[$t87], sum(ResolutionWidth+30)=[$t90], sum(ResolutionWidth+31)=[$t93], sum(ResolutionWidth+32)=[$t96], sum(ResolutionWidth+33)=[$t99], sum(ResolutionWidth+34)=[$t102], sum(ResolutionWidth+35)=[$t105], sum(ResolutionWidth+36)=[$t108], sum(ResolutionWidth+37)=[$t111], sum(ResolutionWidth+38)=[$t114], sum(ResolutionWidth+39)=[$t117], sum(ResolutionWidth+40)=[$t120], sum(ResolutionWidth+41)=[$t123], sum(ResolutionWidth+42)=[$t126], sum(ResolutionWidth+43)=[$t129], sum(ResolutionWidth+44)=[$t132], sum(ResolutionWidth+45)=[$t135], sum(ResolutionWidth+46)=[$t138], sum(ResolutionWidth+47)=[$t141], sum(ResolutionWidth+48)=[$t144], sum(ResolutionWidth+49)=[$t147], sum(ResolutionWidth+50)=[$t150], sum(ResolutionWidth+51)=[$t153], sum(ResolutionWidth+52)=[$t156], sum(ResolutionWidth+53)=[$t159], sum(ResolutionWidth+54)=[$t162], sum(ResolutionWidth+55)=[$t165], sum(ResolutionWidth+56)=[$t168], sum(ResolutionWidth+57)=[$t171], sum(ResolutionWidth+58)=[$t174], sum(ResolutionWidth+59)=[$t177], sum(ResolutionWidth+60)=[$t180], sum(ResolutionWidth+61)=[$t183], sum(ResolutionWidth+62)=[$t186], sum(ResolutionWidth+63)=[$t189], sum(ResolutionWidth+64)=[$t192], sum(ResolutionWidth+65)=[$t195], sum(ResolutionWidth+66)=[$t198], sum(ResolutionWidth+67)=[$t201], sum(ResolutionWidth+68)=[$t204], sum(ResolutionWidth+69)=[$t207], sum(ResolutionWidth+70)=[$t210], sum(ResolutionWidth+71)=[$t213], sum(ResolutionWidth+72)=[$t216], sum(ResolutionWidth+73)=[$t219], sum(ResolutionWidth+74)=[$t222], sum(ResolutionWidth+75)=[$t225], sum(ResolutionWidth+76)=[$t228], sum(ResolutionWidth+77)=[$t231], sum(ResolutionWidth+78)=[$t234], sum(ResolutionWidth+79)=[$t237], sum(ResolutionWidth+80)=[$t240], sum(ResolutionWidth+81)=[$t243], sum(ResolutionWidth+82)=[$t246], sum(ResolutionWidth+83)=[$t249], sum(ResolutionWidth+84)=[$t252], sum(ResolutionWidth+85)=[$t255], sum(ResolutionWidth+86)=[$t258], sum(ResolutionWidth+87)=[$t261], sum(ResolutionWidth+88)=[$t264], sum(ResolutionWidth+89)=[$t267]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(ResolutionWidth)=SUM($0),sum(ResolutionWidth+1)_COUNT=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(ResolutionWidth)":{"sum":{"field":"ResolutionWidth"}},"sum(ResolutionWidth+1)_COUNT":{"value_count":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(ResolutionWidth)=SUM($0),sum(ResolutionWidth+1)_COUNT=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(ResolutionWidth)":{"stats":{"field":"ResolutionWidth"}},"sum(ResolutionWidth+1)_COUNT":{"value_count":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml index a8ac7eaf9b..043eb92a8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml @@ -9,4 +9,4 @@ calcite: LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"stats":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml index 5cf8f54b25..fe898f93a3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml @@ -9,4 +9,4 @@ calcite: LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"stats":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml index e9b5c203f2..13cec363a0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=SUM($1),avg(ResolutionWidth)=AVG($3)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=SUM($1),avg(ResolutionWidth)=AVG($3)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"stats":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml index be021c55e2..5ec94b6d4f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum.sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"stats":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml index 57132615c4..fbccfc2b16 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -9,4 +9,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum(balance), span(age,5)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), SORT_AGG_METRICS->[1 DESC LAST], PROJECT->[sum(balance), span(age,5)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance).sum":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":0},"aggregations":{"sum(balance)":{"stats":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml index 3215115297..95de7d0797 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[sum(balance), c, dc(employer), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[sum(balance), c, dc(employer), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"stats":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml index 211aa979ce..97180d28a8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), SORT_AGG_METRICS->[4 DESC LAST], PROJECT->[sum(balance), count(), d, gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), SORT_AGG_METRICS->[4 DESC LAST], PROJECT->[sum(balance), count(), d, gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"stats":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml index 68cb12a49d..d87eecbfbb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml @@ -9,4 +9,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"stats":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index c543431c51..84835dcd3d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"stats":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index c17bd10e18..578b6ed80f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"stats":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"stats":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml index 3767a38b3a..3144e3ee07 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml @@ -9,4 +9,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"stats":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml index 520f729b7f..40f52ff40b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml @@ -9,4 +9,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"stats":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 7d8cb8826c..84523327d6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -99,6 +99,7 @@ import org.opensearch.sql.opensearch.response.agg.SinglePercentileParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; +import org.opensearch.sql.opensearch.response.agg.SumStatsParser; import org.opensearch.sql.opensearch.response.agg.TopHitsParser; import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.CompositeAggregationBuilder; import org.opensearch.sql.utils.Utils; @@ -476,10 +477,11 @@ private static Pair createRegularAggregation( new SingleValueParser(aggName)); // 1. Only case SUM, skip SUM0 / COUNT since calling avg() in DSL should be faster. // 2. To align with databases, SUM0 is not preferred now. + // Use stats aggregation instead of sum to detect all-null input (count == 0 -> null). case SUM -> Pair.of( - helper.build(args.getFirst().getKey(), AggregationBuilders.sum(aggName)), - new SingleValueParser(aggName)); + helper.build(args.getFirst().getKey(), AggregationBuilders.stats(aggName)), + new SumStatsParser(aggName)); case COUNT -> Pair.of( helper.build( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SumStatsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SumStatsParser.java new file mode 100644 index 0000000000..fe493aa8a3 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SumStatsParser.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.metrics.Stats; + +/** + * Parser for SUM aggregation that uses {@link Stats} to correctly return null when all values are + * null. Per the SQL standard, SUM of all nulls should return null, not 0. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class SumStatsParser implements MetricParser { + + @Getter private final String name; + + @Override + public List> parse(Aggregation agg) { + Stats stats = (Stats) agg; + Object value = stats.getCount() == 0 ? null : stats.getSum(); + return Collections.singletonList(new HashMap<>(Collections.singletonMap(agg.getName(), value))); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java index 7c15586d14..2d161d3b67 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java @@ -32,6 +32,7 @@ import org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.StatsAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -124,25 +125,26 @@ public void rePushDownSortAggMeasure( } if (builder instanceof CompositeAggregationBuilder composite) { boolean asc = collations.get(0).getDirection() == RelFieldCollation.Direction.ASCENDING; - String path = getAggregationPath(collations, fieldNames, composite); + String aggName = getAggregationPath(collations, fieldNames, composite); + String orderPath = resolveOrderPath(aggName, composite); BucketOrder bucketOrder = composite.getSubAggregations().isEmpty() ? BucketOrder.count(asc) - : BucketOrder.aggregation(path, asc); + : BucketOrder.aggregation(orderPath, asc); AggregationBuilder aggregationBuilder = null; if (composite.sources().size() == 1) { if (composite.sources().get(0) instanceof TermsValuesSourceBuilder terms && !terms.missingBucket()) { aggregationBuilder = buildTermsAggregationBuilder(terms, bucketOrder, composite.size()); - attachSubAggregations(composite.getSubAggregations(), path, aggregationBuilder); + attachSubAggregations(composite.getSubAggregations(), aggName, aggregationBuilder); } else if (composite.sources().get(0) instanceof DateHistogramValuesSourceBuilder dateHisto) { aggregationBuilder = buildDateHistogramAggregationBuilder(dateHisto, bucketOrder); - attachSubAggregations(composite.getSubAggregations(), path, aggregationBuilder); + attachSubAggregations(composite.getSubAggregations(), aggName, aggregationBuilder); } else if (composite.sources().get(0) instanceof HistogramValuesSourceBuilder histo && !histo.missingBucket()) { aggregationBuilder = buildHistogramAggregationBuilder(histo, bucketOrder); - attachSubAggregations(composite.getSubAggregations(), path, aggregationBuilder); + attachSubAggregations(composite.getSubAggregations(), aggName, aggregationBuilder); } else { throw new OpenSearchRequestBuilder.PushDownUnSupportedException( "Cannot pushdown sort aggregate measure"); @@ -153,7 +155,7 @@ public void rePushDownSortAggMeasure( src -> src instanceof TermsValuesSourceBuilder terms && !terms.missingBucket())) { // multi-term agg aggregationBuilder = buildMultiTermsAggregationBuilder(composite, bucketOrder); - attachSubAggregations(composite.getSubAggregations(), path, aggregationBuilder); + attachSubAggregations(composite.getSubAggregations(), aggName, aggregationBuilder); } else { throw new OpenSearchRequestBuilder.PushDownUnSupportedException( "Cannot pushdown sort aggregate measure"); @@ -336,6 +338,23 @@ private String getAggregationPath( return fieldNames.get(collations.get(0).getFieldIndex()); } + /** + * Resolve the order path for a bucket order. Stats aggregation is a multi-value metrics + * aggregation, so when ordering on it, the sort path must specify the sub-metric name (e.g., + * "sum(balance).sum"). + */ + private String resolveOrderPath(String aggName, CompositeAggregationBuilder composite) { + AggregationBuilder sortMetric = + composite.getSubAggregations().stream() + .filter(sub -> sub.getName().equals(aggName)) + .findFirst() + .orElse(null); + if (sortMetric instanceof StatsAggregationBuilder) { + return aggName + ".sum"; + } + return aggName; + } + private AggregationBuilder attachSubAggregations( Collection subAggregations, String path, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index 2ab392875f..d20cc1615c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -95,11 +95,7 @@ public Pair visitNamedAggregator( new SingleValueParser(name)); case "sum": return make( - AggregationBuilders.sum(name), - expression, - condition, - name, - new SingleValueParser(name)); + AggregationBuilders.stats(name), expression, condition, name, new SumStatsParser(name)); case "count": return make( AggregationBuilders.count(name), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index 4779332aba..165882d804 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -53,6 +53,7 @@ import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; +import org.opensearch.sql.opensearch.response.agg.SumStatsParser; import org.opensearch.sql.opensearch.response.agg.TopHitsParser; class AggregateAnalyzerTest { @@ -159,7 +160,7 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { assertEquals( "[{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}," + " {\"avg\":{\"avg\":{\"field\":\"a\"}}}," - + " {\"sum\":{\"sum\":{\"field\":\"a\"}}}," + + " {\"sum\":{\"stats\":{\"field\":\"a\"}}}," + " {\"min\":{\"min\":{\"field\":\"a\"}}}," + " {\"max\":{\"max\":{\"field\":\"a\"}}}]", result.getLeft().toString()); @@ -172,7 +173,11 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { .forEach( (k, v) -> { assertTrue(outputFields.contains(k)); - assertInstanceOf(SingleValueParser.class, v); + if (k.equals("sum")) { + assertInstanceOf(SumStatsParser.class, v); + } else { + assertInstanceOf(SingleValueParser.class, v); + } }); } @@ -558,7 +563,7 @@ void analyze_aggCall_multipleWithFilter() throws ExpressionNotAnalyzableExceptio "[{\"filter_avg\":{\"filter\":{\"term\":{\"a\":{\"value\":10,\"boost\":1.0}}}," + "\"aggregations\":{\"filter_avg\":{\"avg\":{\"field\":\"a\"}}}}}," + " {\"filter_sum\":{\"filter\":{\"term\":{\"a\":{\"value\":20,\"boost\":1.0}}}," - + "\"aggregations\":{\"filter_sum\":{\"sum\":{\"field\":\"a\"}}}}}," + + "\"aggregations\":{\"filter_sum\":{\"stats\":{\"field\":\"a\"}}}}}," + " {\"filter_min\":{\"filter\":{\"term\":{\"b.keyword\":{\"value\":\"test1\",\"boost\":1.0}}}," + "\"aggregations\":{\"filter_min\":{\"min\":{\"field\":\"a\"}}}}}," + " {\"filter_max\":{\"filter\":{\"term\":{\"b.keyword\":{\"value\":\"test2\",\"boost\":1.0}}}," @@ -572,7 +577,7 @@ void analyze_aggCall_multipleWithFilter() throws ExpressionNotAnalyzableExceptio .build(), FilterParser.builder() .name("filter_sum") - .metricsParser(new SingleValueParser("filter_sum")) + .metricsParser(new SumStatsParser("filter_sum")) .build(), FilterParser.builder() .name("filter_min") diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java index f230bae5a8..db55a4e865 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java @@ -42,11 +42,13 @@ import org.opensearch.search.aggregations.metrics.ParsedExtendedStats; import org.opensearch.search.aggregations.metrics.ParsedMax; import org.opensearch.search.aggregations.metrics.ParsedMin; +import org.opensearch.search.aggregations.metrics.ParsedStats; import org.opensearch.search.aggregations.metrics.ParsedSum; import org.opensearch.search.aggregations.metrics.ParsedTDigestPercentiles; import org.opensearch.search.aggregations.metrics.ParsedTopHits; import org.opensearch.search.aggregations.metrics.ParsedValueCount; import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.opensearch.search.aggregations.metrics.StatsAggregationBuilder; import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; @@ -59,6 +61,7 @@ public class AggregationResponseUtils { .put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c)) .put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c)) .put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c)) + .put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c)) .put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c)) .put( PercentilesAggregationBuilder.NAME, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 7ba64eaa47..e198aa7a4b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -31,6 +31,7 @@ import org.opensearch.sql.opensearch.response.agg.SinglePercentileParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; +import org.opensearch.sql.opensearch.response.agg.SumStatsParser; import org.opensearch.sql.opensearch.response.agg.TopHitsParser; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -570,6 +571,44 @@ void two_bucket_percentiles_should_pass() { ImmutableMap.of("percentiles", List.of(21.0, 27.0, 30.0, 35.0, 55.0, 58.0, 60.0)))); } + /** SELECT SUM(num_field) as total_sum FROM accounts -- all values are null. */ + @Test + void no_bucket_sum_all_nulls_should_return_null() { + String response = + "{\n" + + " \"stats#total_sum\": {\n" + + " \"count\": 0,\n" + + " \"min\": null,\n" + + " \"max\": null,\n" + + " \"avg\": null,\n" + + " \"sum\": 0.0\n" + + " }\n" + + "}"; + NoBucketAggregationParser parser = + new NoBucketAggregationParser(new SumStatsParser("total_sum")); + List> result = parse(parser, response); + assertEquals(1, result.size()); + assertNull(result.get(0).get("total_sum")); + } + + /** SELECT SUM(num_field) as total_sum FROM accounts -- some values are non-null. */ + @Test + void no_bucket_sum_with_values_should_return_sum() { + String response = + "{\n" + + " \"stats#total_sum\": {\n" + + " \"count\": 3,\n" + + " \"min\": 10.0,\n" + + " \"max\": 30.0,\n" + + " \"avg\": 20.0,\n" + + " \"sum\": 60.0\n" + + " }\n" + + "}"; + NoBucketAggregationParser parser = + new NoBucketAggregationParser(new SumStatsParser("total_sum")); + assertThat(parse(parser, response), contains(entry("total_sum", 60d))); + } + public List> parse(OpenSearchAggregationResponseParser parser, String json) { return parser.parse(fromJson(json)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 64ae7b187c..1ecb75db8f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -84,7 +84,7 @@ void should_build_sum_aggregation() { format( "{%n" + " \"sum(age)\" : {%n" - + " \"sum\" : {%n" + + " \"stats\" : {%n" + " \"field\" : \"age\"%n" + " }%n" + " }%n"