diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java index 0429168a8fd5..4130252d6f1a 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java @@ -26,6 +26,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.apache.calcite.linq4j.tree.Primitive; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.InvalidRelException; @@ -38,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import java.lang.reflect.Method; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; @@ -265,10 +267,15 @@ public EnumerableAggregate(RelOptCluster cluster, RelTraitSet traitSet, resultBlock.add(physType.record(results)); if (getGroupType() != Group.SIMPLE) { final List list = new ArrayList<>(); + boolean hasEmptyGroup = false; for (ImmutableBitSet set : groupSets) { - list.add( + final Expression keySelector = inputPhysType.generateSelector(parameter, groupSet.asList(), - set.asList(), keyPhysType.getFormat())); + set.asList(), keyPhysType.getFormat()); + list.add(keySelector); + if (set.isEmpty()) { + hasEmptyGroup = true; + } } final Expression keySelectors_ = builder.append("keySelectors", @@ -280,20 +287,72 @@ public EnumerableAggregate(RelOptCluster cluster, RelTraitSet traitSet, resultBlock.toBlock(), requireNonNull(key_, "key_"), acc_)); + final Expressions.FluentList arguments = + Expressions.list(childExp, + keySelectors_, + Expressions.call(lambdaFactory, + BuiltInMethod.AGG_LAMBDA_FACTORY_ACC_INITIALIZER.method), + Expressions.call(lambdaFactory, + BuiltInMethod.AGG_LAMBDA_FACTORY_ACC_ADDER.method), + Expressions.call(lambdaFactory, + BuiltInMethod.AGG_LAMBDA_FACTORY_ACC_RESULT_SELECTOR.method, + resultSelector)); + final Method groupByMultipleMethod; + if (hasEmptyGroup) { + // The key of the empty grouping set is a constant: each grouping + // column is defaulted (null, or the default value of its primitive + // type) and each indicator is true. It is built directly, mirroring + // the key that the selector generated for that grouping set + // computes for any input row. Passing it to groupByMultiple lets + // the empty grouping set produce its single row when the input is + // empty, just as a query with no GROUP BY produces a row over an + // empty input. + // + // For example, for the query + // + // SELECT deptno, COUNT(*) AS c + // FROM emp WHERE FALSE + // GROUP BY ROLLUP(deptno) + // + // where deptno is an INT column, the generated code ends in a call + // to groupByMultiple whose last argument is the constant key of the + // empty grouping set (the key selectors, the lambda factory and the + // result selector are anonymous classes, elided here): + // + // return EnumerableDefaults.groupByMultiple( + // Linq4j.asEnumerable(new Integer[] {}), + // Arrays.asList(keySelector0, keySelector1), + // lambdaFactory.accumulatorInitializer(), + // lambdaFactory.accumulatorAdder(), + // lambdaFactory.resultSelector(resultSelector), + // FlatLists.of(0, true)); + // + // Here, '0' is the default value of the deptno column, and 'true' + // is its indicator, true because deptno is not in the empty + // grouping set. + final List keyExpressions = new ArrayList<>(); + for (int j = 0; j < groupCount; j++) { + final Primitive primitive = + Primitive.of(keyPhysType.fieldClass(j)); + keyExpressions.add( + Expressions.constant( + primitive != null ? primitive.defaultValue : null)); + } + for (int j = 0; j < groupCount; j++) { + keyExpressions.add(Expressions.constant(true)); + } + final Expression emptyGroupKey = + builder.append("emptyGroupKey", + keyPhysType.record(keyExpressions)); + arguments.add(emptyGroupKey); + groupByMultipleMethod = BuiltInMethod.GROUP_BY_MULTIPLE2.method; + } else { + groupByMultipleMethod = BuiltInMethod.GROUP_BY_MULTIPLE.method; + } builder.add( Expressions.return_(null, - Expressions.call( - BuiltInMethod.GROUP_BY_MULTIPLE.method, - Expressions.list(childExp, - keySelectors_, - Expressions.call(lambdaFactory, - BuiltInMethod.AGG_LAMBDA_FACTORY_ACC_INITIALIZER.method), - Expressions.call(lambdaFactory, - BuiltInMethod.AGG_LAMBDA_FACTORY_ACC_ADDER.method), - Expressions.call(lambdaFactory, - BuiltInMethod.AGG_LAMBDA_FACTORY_ACC_RESULT_SELECTOR.method, - resultSelector)) - .appendIfNotNull(keyPhysType.comparer())))); + Expressions.call(groupByMultipleMethod, + arguments.appendIfNotNull(keyPhysType.comparer())))); } else if (groupCount == 0) { final Expression resultSelector = builder.append( diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java index 1e4a97040cd0..10b6a823ed84 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java @@ -395,6 +395,15 @@ public void send(Row row) { } public void end(Sink sink) throws InterruptedException { + if (accumulators.isEmpty() && grouping.isEmpty()) { + // The empty grouping set (the grand total) produces a row even when + // the input is empty, just as a query with no GROUP BY does. + final AccumulatorList list = new AccumulatorList(); + for (AccumulatorFactory factory : accumulatorFactories) { + list.add(factory.get()); + } + accumulators.put(Row.of(), list); + } for (Map.Entry e : accumulators.entrySet()) { final Row key = e.getKey(); final AccumulatorList list = e.getValue(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java index 95331c69a414..2cf5c1140af6 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/PruneEmptyRules.java @@ -395,7 +395,7 @@ public interface RemoveEmptySingleRuleConfig extends PruneEmptyRule.Config { .withOperandFor(Sort.class, singleRel -> true); RemoveEmptySingleRuleConfig AGGREGATE = ImmutableRemoveEmptySingleRuleConfig.of() .withDescription("PruneEmptyAggregate") - .withOperandFor(Aggregate.class, Aggregate::isNotGrandTotal); + .withOperandFor(Aggregate.class, aggregate -> !aggregate.hasEmptyGroup()); RemoveEmptySingleRuleConfig WINDOW = ImmutableRemoveEmptySingleRuleConfig.of() .withDescription("PruneEmptyWindow") .withOperandFor(Window.class, singleRel -> true); diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 5fd9d9a71541..76579631be34 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -296,6 +296,9 @@ public enum BuiltInMethod { GROUP_BY_MULTIPLE(EnumerableDefaults.class, "groupByMultiple", Enumerable.class, List.class, Function0.class, Function2.class, Function2.class), + GROUP_BY_MULTIPLE2(EnumerableDefaults.class, "groupByMultiple", + Enumerable.class, List.class, Function0.class, Function2.class, + Function2.class, Object.class), AGGREGATE(ExtendedEnumerable.class, "aggregate", Object.class, Function2.class, Function1.class), ORDER_BY(ExtendedEnumerable.class, "orderBy", Function1.class, diff --git a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java index c138056e727a..ade1182ef171 100644 --- a/core/src/test/java/org/apache/calcite/test/InterpreterTest.java +++ b/core/src/test/java/org/apache/calcite/test/InterpreterTest.java @@ -355,6 +355,22 @@ private static void assertRows(Interpreter interpreter, "[Ringo, 1]"); } + /** Tests ROLLUP, which is GROUPING SETS with an empty grouping set. */ + @Test void testAggregateRollup() { + rootSchema().add("beatles", new ScannableTableTest.BeatlesTable()); + sql("select \"j\", count(*) from \"beatles\" group by rollup(\"j\")") + .returnsRowsUnordered("[John, 1]", "[Paul, 1]", "[Ringo, 1]", + "[George, 1]", "[null, 4]"); + } + + /** As {@link #testAggregateRollup()}, but the input is empty. The empty + * grouping set yields one row, as a query with no GROUP BY does. */ + @Test void testAggregateRollupOnEmptyInput() { + rootSchema().add("beatles", new ScannableTableTest.BeatlesTable()); + sql("select \"j\", count(*) from \"beatles\" where false group by rollup(\"j\")") + .returnsRows("[null, 0]"); + } + /** Tests a GROUP BY query that uses * {@link org.apache.calcite.sql.fun.SqlInternalOperators#LITERAL_AGG}. */ @Test void testAggregateLiteralAgg() { diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableAggregateTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableAggregateTest.java new file mode 100644 index 000000000000..0f3ba51b1638 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableAggregateTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.test.enumerable; + +import org.apache.calcite.adapter.java.ReflectiveSchema; +import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.config.Lex; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.test.schemata.hr.HrSchema; + +import org.junit.jupiter.api.Test; + +/** Tests for {@link org.apache.calcite.adapter.enumerable.EnumerableAggregate}. */ +public class EnumerableAggregateTest { + + /** Test case for + * [CALCITE-7828] + * Aggregate whose grouping sets include () returns no rows on empty + * input. + * + *

ROLLUP(deptno) is GROUPING SETS ((deptno), ()), and the () set yields + * one row even when the input is empty, as a query with no GROUP BY does + * (CALCITE-1059). */ + @Test void rollupOnEmptyInput() { + tester() + .query("select deptno, count(*) as c\n" + + "from emps where false group by rollup(deptno)") + .explainContains("EnumerableAggregate(group=[{0}], groups=[[{0}, {}]], c=[COUNT()])\n" + + " EnumerableValues(tuples=[[]])") + .returnsUnordered("deptno=null; c=0"); + } + + /** As {@link #rollupOnEmptyInput()}, but the input is empty at run time + * (not at plan time), so the plan still contains a filter. */ + @Test void rollupOnEmptyRuntimeInput() { + tester() + .query("select deptno, count(*) as c\n" + + "from emps where deptno > 100 group by rollup(deptno)") + .returnsUnordered("deptno=null; c=0"); + } + + /** As {@link #rollupOnEmptyInput()}, with CUBE and more than one grouping + * column. CUBE(deptno, commission) is GROUPING SETS ((deptno, commission), + * (deptno), (commission), ()). */ + @Test void cubeOnEmptyInput() { + tester() + .query("select deptno, commission, count(*) as c\n" + + "from emps where false group by cube(deptno, commission)") + .returnsUnordered("deptno=null; commission=null; c=0"); + } + + /** As {@link #rollupOnEmptyInput()}, with GROUPING SETS. */ + @Test void groupingSetsWithEmptyGroupOnEmptyInput() { + tester() + .query("select deptno, count(*) as c\n" + + "from emps where false group by grouping sets ((deptno), ())") + .explainContains("EnumerableAggregate(group=[{0}], groups=[[{0}, {}]], c=[COUNT()])\n" + + " EnumerableValues(tuples=[[]])") + .returnsUnordered("deptno=null; c=0"); + } + + /** A grouping set other than () yields no row when the input is empty. */ + @Test void groupingSetsWithoutEmptyGroupOnEmptyInput() { + tester() + .query("select deptno, count(*) as c\n" + + "from emps where false group by grouping sets ((deptno), (commission))") + .returnsCount(0); + } + + /** Sanity check: on non-empty input, ROLLUP produces a row per group, + * plus the grand-total row. */ + @Test void rollupOnNonEmptyInput() { + tester() + .query("select deptno, count(*) as c\n" + + "from emps group by rollup(deptno)") + .returnsUnordered( + "deptno=10; c=3", + "deptno=20; c=1", + "deptno=null; c=4"); + } + + /** As {@link #rollupOnEmptyInput()}, with aggregate functions that return + * NULL on empty input (SUM), and GROUPING, which reads the indicator of + * the synthetic empty-group key. */ + @Test void rollupOnEmptyInputSumGrouping() { + tester() + .query("select deptno, grouping(deptno) as g, sum(salary) as s\n" + + "from emps where false group by rollup(deptno)") + .returnsUnordered("deptno=null; g=1; s=null"); + } + + private CalciteAssert.AssertThat tester() { + return CalciteAssert.that() + .with(CalciteConnectionProperty.LEX, Lex.JAVA) + .withSchema("s", new ReflectiveSchema(new HrSchema())); + } +} diff --git a/core/src/test/resources/sql/agg.iq b/core/src/test/resources/sql/agg.iq index 2b035304b872..9770d50a5e87 100644 --- a/core/src/test/resources/sql/agg.iq +++ b/core/src/test/resources/sql/agg.iq @@ -497,6 +497,48 @@ group by rollup(gender); !ok +# [CALCITE-7828] Aggregate whose grouping sets include () returns no rows on empty input +# The empty grouping set, (), yields one row, as a query with no GROUP BY +# These sql programs had been validated in PostgreSQL + +select gender, count(*) as c +from emp +where false +group by rollup(gender); ++--------+---+ +| GENDER | C | ++--------+---+ +| | 0 | ++--------+---+ +(1 row) + +!ok + +select gender, count(*) as c +from emp +where false +group by grouping sets (gender, ()); ++--------+---+ +| GENDER | C | ++--------+---+ +| | 0 | ++--------+---+ +(1 row) + +!ok + +select gender, count(*) as c +from emp +where false +group by grouping sets (gender, deptno); ++--------+---+ +| GENDER | C | ++--------+---+ ++--------+---+ +(0 rows) + +!ok + # ROLLUP plus ORDER BY select gender, count(*) as c from emp diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 2687c1513105..dbfe7d3bd6a9 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -801,13 +801,40 @@ public static Enumerable groupByM Function0 accumulatorInitializer, Function2 accumulatorAdder, final Function2 resultSelector) { + return groupByMultiple(enumerable, keySelectors, accumulatorInitializer, + accumulatorAdder, resultSelector, null); + } + + /** + * Groups the elements of a sequence according to a list of + * specified key selector functions, initializing an accumulator for each + * group and adding to it each time an element with the same key is seen. + * Creates a result value from each accumulator and its key using a + * specified function. + * + *

This method exists to support SQL {@code GROUPING SETS}. + * It does not correspond to any method in {@link Enumerable}. + * + *

If {@code emptyGroupKey} is not null, it is the key of the empty + * grouping set (the one that aggregates over all input rows, like a query + * with no GROUP BY). In that case, a single row is produced for that key + * even when the input is empty, just as a query with no GROUP BY returns + * a row over empty input. + */ + public static Enumerable groupByMultiple( + Enumerable enumerable, List> keySelectors, + Function0 accumulatorInitializer, + Function2 accumulatorAdder, + final Function2 resultSelector, + @Nullable TKey emptyGroupKey) { return groupByMultiple_( new HashMap<>(), enumerable, keySelectors, accumulatorInitializer, accumulatorAdder, - resultSelector); + resultSelector, + emptyGroupKey); } /** @@ -1192,9 +1219,12 @@ private static Enumerable groupBy List> keySelectors, Function0 accumulatorInitializer, Function2 accumulatorAdder, - final Function2 resultSelector) { + final Function2 resultSelector, + @Nullable TKey emptyGroupKey) { + boolean seenRow = false; try (Enumerator os = enumerable.enumerator()) { while (os.moveNext()) { + seenRow = true; for (Function1 keySelector : keySelectors) { TSource o = os.current(); TKey key = keySelector.apply(o); @@ -1214,6 +1244,12 @@ private static Enumerable groupBy } } } + if (!seenRow && emptyGroupKey != null) { + // The empty grouping set (the grand total) produces a row even when the + // input is empty, just as a query with no GROUP BY does + // (see CALCITE-1059). + map.put(emptyGroupKey, accumulatorInitializer.apply()); + } return new LookupResultEnumerable<>(map, resultSelector); }