shiftedProjects =
+ RexUtil.shift(rightProject.getProjects(), newLeft.getRowType().getFieldCount());
+ for (int i = 0; i < shiftedProjects.size(); i++) {
+ newProjectExprs.add(
+ rightHoist.matchIndicator != null && rightHoist.isConstant[i]
+ ? nullGateOuterJoin(shiftedProjects.get(i), rightHoist.matchIndicator, rexBuilder)
+ : shiftedProjects.get(i)
+ );
}
newRight = right.withPartialQuery(PartialDruidQuery.create(rightScan));
- conditionAnalysis = conditionAnalysis.pushThroughRightProject(rightProject);
+ conditionAnalysis = rightHoist.pushedAnalysis;
} else {
// Leave right as-is. Write input refs that do nothing.
for (int i = 0; i < right.getRowType().getFieldCount(); i++) {
@@ -225,19 +230,6 @@ public void onMatch(RelOptRuleCall call)
call.transformTo(relBuilder.build());
}
- private static RexNode makeNullableIfLiteral(final RexNode rexNode, final RexBuilder rexBuilder)
- {
- if (rexNode.isA(SqlKind.LITERAL)) {
- return rexBuilder.makeLiteral(
- RexLiteral.value(rexNode),
- rexBuilder.getTypeFactory().createTypeWithNullability(rexNode.getType(), true),
- true
- );
- } else {
- return rexNode;
- }
- }
-
/**
* Returns whether we can handle the join condition. In case, some conditions in an AND expression are not supported,
* they are extracted into a post-join filter instead.
@@ -582,6 +574,90 @@ private static boolean isRightInputRef(final RexNode rexNode, final int numLeftF
return rexNode.isA(SqlKind.INPUT_REF) && ((RexInputRef) rexNode).getIndex() >= numLeftFields;
}
+ /**
+ * Plan for hoisting a right-side {@link Project} above {@link Join}. Computes the pushed-down condition
+ * analysis, flags each projection expression as constant (no input refs) or not, and (if necessary)
+ * selects an equi-condition's right column to use as a match indicator for {@link #nullGateOuterJoin}.
+ *
+ * Returns null if the right project cannot be hoisted.
+ */
+ @Nullable
+ private static RightHoistPlan planRightHoist(
+ final Join join,
+ final Project rightProject,
+ final ConditionAnalysis analysis
+ )
+ {
+ final List projects = rightProject.getProjects();
+ final boolean[] isConstant = new boolean[projects.size()];
+ boolean hasConstant = false;
+
+ for (int i = 0; i < projects.size(); i++) {
+ isConstant[i] = !RexUtil.containsInputRef(projects.get(i));
+ hasConstant = hasConstant || isConstant[i];
+ }
+
+ final ConditionAnalysis pushed = analysis.pushThroughRightProject(rightProject);
+ if (!hasConstant || !join.getJoinType().generatesNullsOnRight()) {
+ // No null-gating needed: either there are no constants to gate, or the join type does not
+ // null-extend the right side.
+ return new RightHoistPlan(pushed, null, isConstant);
+ }
+
+ if (join.getJoinType() != JoinRelType.LEFT) {
+ // Only LEFT is safely gateable with the matchIndicator scheme. FULL (and any future
+ // null-extending-on-right join types) are skipped. See the Javadoc above for details.
+ return null;
+ }
+
+ RexNode indicator = null;
+ for (RexEquality eq : pushed.equalitySubConditions) {
+ // Exclude IS_NOT_DISTINCT_FROM, because it allows NULL = NULL matches, which would mean
+ // the rhs could potentially be NULL even when the join matches.
+ if (eq.kind == SqlKind.EQUALS) {
+ indicator = eq.right;
+ break;
+ }
+ }
+
+ return indicator == null ? null : new RightHoistPlan(pushed, indicator, isConstant);
+ }
+
+ /**
+ * Wrap a right-side constant expression in a CASE keyed on {@code matchIndicator}, so it evaluates to NULL
+ * for non-matching rows of an outer join. Non-constant expressions don't need this because their input refs
+ * are already NULL-extended by the join.
+ */
+ private static RexNode nullGateOuterJoin(
+ final RexNode rexNode,
+ final RexNode matchIndicator,
+ final RexBuilder rexBuilder
+ )
+ {
+ final RelDataType nullableType =
+ rexBuilder.getTypeFactory().createTypeWithNullability(rexNode.getType(), true);
+ return rexBuilder.makeCall(
+ SqlStdOperatorTable.CASE,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, matchIndicator),
+ rexBuilder.ensureType(nullableType, rexNode, false),
+ rexBuilder.makeNullLiteral(nullableType)
+ );
+ }
+
+ /**
+ * Return value from {@link #planRightHoist(Join, Project, ConditionAnalysis)}.
+ *
+ * @param pushedAnalysis modified condition analysis for the hoisted plan
+ * @param matchIndicator indicator field that can be used to determine if a join happened
+ * @param isConstant whether each field (by index) is a constant
+ */
+ private record RightHoistPlan(
+ ConditionAnalysis pushedAnalysis,
+ @Nullable RexNode matchIndicator,
+ boolean[] isConstant
+ )
+ {
+ }
/**
* Like {@link org.apache.druid.segment.join.Equality} but uses {@link RexNode} instead of
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidStripUnionArmCastRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidStripUnionArmCastRule.java
new file mode 100644
index 000000000000..d8d3b4c5c846
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidStripUnionArmCastRule.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite.rule;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.core.Union;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.rules.SubstitutionRule;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Strips numeric-to-numeric {@code CAST} expressions so {@link DruidUnionDataSourceRule} is able
+ * to generate unions in more plans.
+ *
+ * Example: for {@code SELECT dim1, dim2, m1 FROM foo2 UNION ALL SELECT dim1, dim2, m1 FROM foo}
+ * where {@code foo2.m1} is {@code BIGINT} and {@code foo.m1} is {@code FLOAT}, Calcite produces:
+ *
+ * LogicalUnion(all=[true])
+ * LogicalProject(dim1=[$1], dim2=[$2], m1=[CAST($5):FLOAT])
+ * LogicalTableScan(table=[[druid, foo2]])
+ * LogicalProject(dim1=[$1], dim2=[$2], m1=[$5])
+ * LogicalTableScan(table=[[druid, foo]])
+ *
+ * This rule rewrites it to:
+ *
+ * LogicalUnion(all=[true])
+ * LogicalProject(dim1=[$1], dim2=[$2], m1=[$5]) // CAST stripped
+ * LogicalTableScan(table=[[druid, foo2]])
+ * LogicalProject(dim1=[$1], dim2=[$2], m1=[$5])
+ * LogicalTableScan(table=[[druid, foo]])
+ *
+ *
+ * See {@code union_datasource.iq} for test cases that fail without this rule.
+ */
+public class DruidStripUnionArmCastRule extends RelOptRule implements SubstitutionRule
+{
+ private static final DruidStripUnionArmCastRule INSTANCE = new DruidStripUnionArmCastRule();
+
+ private DruidStripUnionArmCastRule()
+ {
+ super(operand(Union.class, any()));
+ }
+
+ public static DruidStripUnionArmCastRule instance()
+ {
+ return INSTANCE;
+ }
+
+ @Override
+ public void onMatch(final RelOptRuleCall call)
+ {
+ final Union union = call.rel(0);
+ final List newInputs = new ArrayList<>(union.getInputs().size());
+ boolean anyChanged = false;
+ for (final RelNode input : union.getInputs()) {
+ final RelNode arm = input.stripped();
+ if (armHasStrippableCast(arm)) {
+ newInputs.add(stripArmCasts((Project) arm));
+ anyChanged = true;
+ } else {
+ newInputs.add(input);
+ }
+ }
+ if (anyChanged) {
+ final RelNode transformed = union.copy(union.getTraitSet(), newInputs, union.all);
+ if (!transformed.getRowType().equals(union.getRowType())) {
+ return;
+ }
+ call.transformTo(transformed);
+ }
+ }
+
+ /**
+ * Returns true when {@code arm} is a {@link Project} directly atop a {@link TableScan} and at
+ * least one of its projections is a Druid-absorbable numeric {@code CAST}.
+ */
+ private static boolean armHasStrippableCast(final RelNode arm)
+ {
+ if (!(arm instanceof Project project)) {
+ return false;
+ }
+ if (!(project.getInput().stripped() instanceof TableScan)) {
+ return false;
+ }
+ for (final RexNode expr : project.getProjects()) {
+ if (isStrippableCast(expr)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns a new {@link LogicalProject} with strippable CAST replaced by their underlying {@link RexInputRef}.
+ */
+ private static Project stripArmCasts(final Project project)
+ {
+ final List newExprs = new ArrayList<>(project.getProjects().size());
+ for (final RexNode expr : project.getProjects()) {
+ if (isStrippableCast(expr)) {
+ newExprs.add(((RexCall) expr).getOperands().get(0));
+ } else {
+ newExprs.add(expr);
+ }
+ }
+ // Using LogicalProject.create so Calcite re-derives the row type from the new expressions.
+ return LogicalProject.create(
+ project.getInput(),
+ project.getHints(),
+ newExprs,
+ project.getRowType().getFieldNames(),
+ project.getVariablesSet()
+ );
+ }
+
+ /**
+ * Returns true when {@code expr} is a {@code CAST} of a {@link RexInputRef} between numeric types.
+ */
+ private static boolean isStrippableCast(final RexNode expr)
+ {
+ if (!expr.isA(SqlKind.CAST)) {
+ return false;
+ }
+ final RexCall castCall = (RexCall) expr;
+ if (castCall.getOperands().size() != 1 || !(castCall.getOperands().get(0) instanceof RexInputRef)) {
+ return false;
+ }
+ final RelDataType fromType = castCall.getOperands().get(0).getType();
+ final RelDataType toType = castCall.getType();
+ return SqlTypeUtil.isNumeric(fromType) && SqlTypeUtil.isNumeric(toType);
+ }
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterDecomposeConcatRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterDecomposeConcatRule.java
index b4a4b16b8a28..c87d503221ab 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterDecomposeConcatRule.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/FilterDecomposeConcatRule.java
@@ -117,6 +117,17 @@ public RexNode visitCall(final RexCall call)
operand -> rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, operand)
)
);
+ } else if (call.isA(SqlKind.SEARCH)
+ && FlattenConcatRule.isNonTrivialStringConcat(call.getOperands().get(0))) {
+ // Expand SEARCH to OR(EQUALS, ...) so the EQUALS branch above decomposes each CONCAT comparison.
+ final RexNode expanded = RexUtil.expandSearch(rexBuilder, null, call);
+ final RexNode rewritten = expanded.accept(this);
+ //noinspection ObjectEquality
+ if (rewritten != expanded) {
+ return rewritten;
+ }
+ negate = false;
+ newCall = null;
} else {
negate = false;
newCall = null;
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/FixIncorrectInExpansionTypes.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/FixIncorrectInExpansionTypes.java
deleted file mode 100644
index 9c049ac89dab..000000000000
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/FixIncorrectInExpansionTypes.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.sql.calcite.rule;
-
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.rules.SubstitutionRule;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexShuttle;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * Rewrites comparisions to avoid bug FIXME.
- *
- * Rewrites RexCall::VARCHAR = RexLiteral::CHAR to RexCall::VARCHAR =
- * RexLiteral::VARCHAR
- *
- * needed until CALCITE-6435 is fixed & released.
- */
-public class FixIncorrectInExpansionTypes extends RelOptRule implements SubstitutionRule
-{
- public FixIncorrectInExpansionTypes()
- {
- super(operand(RelNode.class, any()));
- }
-
- @Override
- public void onMatch(RelOptRuleCall call)
- {
- final RelNode oldNode = call.rel(0);
- final RewriteShuttle shuttle = new RewriteShuttle(oldNode.getCluster().getRexBuilder());
- final RelNode newNode = oldNode.accept(shuttle);
-
- // noinspection ObjectEquality
- if (newNode != oldNode) {
- call.transformTo(newNode);
- call.getPlanner().prune(oldNode);
- }
- }
-
- private static class RewriteShuttle extends RexShuttle
- {
- private final RexBuilder rexBuilder;
-
- public RewriteShuttle(RexBuilder rexBuilder)
- {
- this.rexBuilder = rexBuilder;
- }
-
- @Override
- public RexNode visitCall(RexCall call)
- {
- RexNode newNode = super.visitCall(call);
- if (newNode.getKind() == SqlKind.EQUALS || newNode.getKind() == SqlKind.NOT_EQUALS) {
- RexCall newCall = (RexCall) newNode;
- RexNode op0 = newCall.getOperands().get(0);
- RexNode op1 = newCall.getOperands().get(1);
- if (RexUtil.isLiteral(op1, false)) {
-
- if (op1.getType().getSqlTypeName() == SqlTypeName.CHAR
- && op0.getType().getSqlTypeName() == SqlTypeName.VARCHAR) {
-
- RexNode newLiteral = rexBuilder.ensureType(op0.getType(), op1, true);
- return rexBuilder.makeCall(
- newCall.getOperator(),
- op0,
- newLiteral
- );
- }
- }
- }
- return newNode;
- }
- }
-}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/InlineValuesSubQueryRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/InlineValuesSubQueryRule.java
new file mode 100644
index 000000000000..290e979667c7
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/InlineValuesSubQueryRule.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.calcite.rule;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableRangeSet;
+import com.google.common.collect.Range;
+import com.google.common.collect.TreeRangeSet;
+import org.apache.calcite.plan.RelOptLattice;
+import org.apache.calcite.plan.RelOptMaterialization;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelHomogeneousShuttle;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.Values;
+import org.apache.calcite.rel.rules.SubQueryRemoveRule;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUnknownAs;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.tools.Program;
+import org.apache.calcite.util.Sarg;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.sql.calcite.planner.DruidRexExecutor;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.utils.CollectionUtils;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Rewrites {@code [NOT] IN (SELECT col FROM (VALUES ...))} into {@code SEARCH(, Sarg[...])}
+ * before Calcite's {@link SubQueryRemoveRule#rewriteIn} runs. When that rule applies along with an UNNEST,
+ * the subquery ends up being too complex for us to plan. See {@code qaUnnest/mv_sql_subquery_with_where.09.all.iq}
+ * for a test case that would fail without this rule.
+ */
+public class InlineValuesSubQueryRule implements Program
+{
+ private final PlannerContext plannerContext;
+
+ public InlineValuesSubQueryRule(final PlannerContext plannerContext)
+ {
+ this.plannerContext = plannerContext;
+ }
+
+ @Override
+ public RelNode run(
+ final RelOptPlanner planner,
+ final RelNode rel,
+ final RelTraitSet requiredOutputTraits,
+ final List materializations,
+ final List lattices
+ )
+ {
+ return rel.accept(new InlineValuesRelShuttle(plannerContext));
+ }
+
+ private static class InlineValuesRelShuttle extends RelHomogeneousShuttle
+ {
+ private final PlannerContext plannerContext;
+
+ InlineValuesRelShuttle(final PlannerContext plannerContext)
+ {
+ this.plannerContext = plannerContext;
+ }
+
+ @Override
+ public RelNode visit(RelNode other)
+ {
+ final RelNode visited = super.visit(other);
+ if (visited instanceof Filter filter && RexUtil.SubQueryFinder.containsSubQuery(filter)) {
+ final RexNode condition = filter.getCondition();
+ final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
+ final RexNode newCondition = condition.accept(new InlineValuesRexShuttle(rexBuilder, plannerContext));
+ if (newCondition != condition) {
+ return filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
+ }
+ }
+ return visited;
+ }
+ }
+
+ private static class InlineValuesRexShuttle extends RexShuttle
+ {
+ private final RexBuilder rexBuilder;
+ private final int inSubQueryThreshold;
+ private final DruidRexExecutor executor;
+
+ InlineValuesRexShuttle(final RexBuilder rexBuilder, final PlannerContext plannerContext)
+ {
+ this.rexBuilder = rexBuilder;
+ this.inSubQueryThreshold = plannerContext.queryContext().getInSubQueryThreshold();
+ this.executor = new DruidRexExecutor(plannerContext);
+ }
+
+ @Override
+ public RexNode visitSubQuery(final RexSubQuery subQuery)
+ {
+ if (subQuery.getKind() == SqlKind.IN) {
+ final RexNode inlined = tryInlineIn(subQuery);
+ if (inlined != null) {
+ return inlined;
+ }
+ }
+ return super.visitSubQuery(subQuery);
+ }
+
+ @Override
+ public RexNode visitCall(final RexCall call)
+ {
+ // NOT IN appears as NOT(IN(...))
+ if (call.getKind() == SqlKind.NOT && call.getOperands().size() == 1) {
+ final RexNode operand = call.getOperands().get(0);
+ if (operand instanceof RexSubQuery && operand.getKind() == SqlKind.IN) {
+ final RexNode inlined = tryInlineIn((RexSubQuery) operand);
+ if (inlined != null) {
+ return rexBuilder.makeCall(SqlStdOperatorTable.NOT, inlined);
+ }
+ }
+ }
+ return super.visitCall(call);
+ }
+
+ @Nullable
+ private RexNode tryInlineIn(final RexSubQuery subQuery)
+ {
+ if (subQuery.getOperands().size() != 1) {
+ return null;
+ }
+
+ // Accept a bare Values, or a Project-over-Values. For Project, we evaluate the projection
+ // expression on each Values tuple via the executor.
+ final Values valuesRel;
+ final RexNode projExpr;
+ if (subQuery.rel instanceof Values) {
+ valuesRel = (Values) subQuery.rel;
+ projExpr = null;
+ } else if (subQuery.rel instanceof Project
+ && ((Project) subQuery.rel).getProjects().size() == 1
+ && ((Project) subQuery.rel).getInput() instanceof Values) {
+ valuesRel = (Values) ((Project) subQuery.rel).getInput();
+ projExpr = CollectionUtils.getOnlyElement(
+ ((Project) subQuery.rel).getProjects(),
+ xs -> DruidException.defensive("Expected 1 project, got[%s]", xs)
+ );
+ } else {
+ return null;
+ }
+
+ if (valuesRel.getRowType().getFieldCount() != 1) {
+ return null;
+ }
+
+ final ImmutableList> tuples = valuesRel.getTuples();
+ if (tuples.isEmpty() || tuples.size() >= inSubQueryThreshold) {
+ return null;
+ }
+
+ final RelDataType targetType =
+ projExpr != null ? projExpr.getType() : valuesRel.getRowType().getFieldList().getFirst().getType();
+
+ final List literals = new ArrayList<>(tuples.size());
+ for (final ImmutableList tuple : tuples) {
+ if (tuple.size() != 1) {
+ return null;
+ }
+ final RexLiteral raw = tuple.getFirst();
+ final RexLiteral literal;
+ if (projExpr == null) {
+ literal = raw;
+ } else {
+ // Substitute the row's literal for $0 in the projection, then evaluate.
+ final RexNode substituted = projExpr.accept(new RexShuttle()
+ {
+ @Override
+ public RexNode visitInputRef(final RexInputRef ref)
+ {
+ return ref.getIndex() == 0 ? raw : ref;
+ }
+ });
+ final List reducedValues = new ArrayList<>(1);
+ executor.reduce(rexBuilder, ImmutableList.of(substituted), reducedValues);
+ final RexNode reduced = CollectionUtils.getOnlyElement(
+ reducedValues,
+ xs -> DruidException.defensive("Expected 1 value, got[%s]", xs)
+ );
+ if (!(reduced instanceof RexLiteral)) {
+ return null;
+ }
+ literal = (RexLiteral) reduced;
+ }
+ if (literal.isNull()) {
+ // Bounce out if we have an IN/NOT IN with NULL in the list. Let other rules try to handle it.
+ return null;
+ }
+ literals.add(literal);
+ }
+
+ final RexNode lhs = CollectionUtils.getOnlyElement(
+ subQuery.getOperands(),
+ xs -> DruidException.defensive("Expected 1 operator, got[%s]", xs)
+ );
+
+ return buildSearch(lhs, literals, targetType);
+ }
+
+ /**
+ * Build a SEARCH(lhs, Sarg[v1, v2, ...]) expression from the extracted literals. Literals
+ * must be non-null; the caller is responsible for bailing out if any NULL is present in
+ * the VALUES list.
+ */
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private RexNode buildSearch(
+ final RexNode lhs,
+ final List literals,
+ final RelDataType targetType
+ )
+ {
+ final TreeRangeSet rangeSet = TreeRangeSet.create();
+ for (final RexLiteral literal : literals) {
+ rangeSet.add(Range.singleton(literal.getValueAs(Comparable.class)));
+ }
+
+ final Sarg sarg = Sarg.of(RexUnknownAs.UNKNOWN, ImmutableRangeSet.copyOf(rangeSet));
+ return RexUtil.sargRef(rexBuilder, lhs, sarg, targetType, RexUnknownAs.UNKNOWN);
+ }
+ }
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java b/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java
index 9c345dafad0c..444f4d2b9c31 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java
@@ -31,6 +31,7 @@
import org.apache.calcite.sql.type.SqlOperandCountRanges;
import org.apache.calcite.sql.type.SqlSingleOperandTypeChecker;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
@@ -58,6 +59,12 @@ private RowSignatures()
// No instantiation.
}
+ /**
+ * Creates a {@link RowSignature} from a Calcite row type with a list of field names. The provided list
+ * of field names is used instead of the field names from the Calcite row type. Care is required when
+ * selecting the field names, because field names in Druid are the primary way of identifying a field,
+ * whereas field names in Calcite are just labels (and may not be unique).
+ */
public static RowSignature fromRelDataType(final List rowOrder, final RelDataType rowType)
{
if (rowOrder.size() != rowType.getFieldCount()) {
@@ -75,6 +82,15 @@ public static RowSignature fromRelDataType(final List rowOrder, final Re
return rowSignatureBuilder.build();
}
+ /**
+ * Creates a {@link RowSignature} from a Calcite row type, adjusting field names if needed
+ * such that they are unique.
+ */
+ public static RowSignature fromRelDataTypeWithUniqueFields(final RelDataType rowType)
+ {
+ return fromRelDataType(SqlValidatorUtil.uniquify(rowType.getFieldNames()), rowType);
+ }
+
/**
* Return the "natural" {@link StringComparator} for an extraction from a row signature. This will be a
* lexicographic comparator for String types and a numeric comparator for Number types.
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
index 2ca95310e1cc..936d37c32c9a 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
@@ -1515,7 +1515,7 @@ public void testArrayContainsFilterWithDynamicParameter()
Druids.ScanQueryBuilder builder = newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
- .filters(expressionFilter("array_contains(array(1,null),array((\"dbl1\" > 1)))"))
+ .filters(expressionFilter("array_contains(array(1,null),array((\"dbl1\" > 1.0)))"))
.columns("dim3")
.columnTypes(ColumnType.STRING)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
@@ -5141,7 +5141,7 @@ public void testUnnestVirtualWithColumns1()
.intervals(querySegmentSpec(Filtration.eternity()))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.filters(or(
- range("m1", ColumnType.LONG, null, "10", false, false),
+ range("m1", ColumnType.FLOAT, null, "10.0", false, false),
equality("j0.unnest", "b", ColumnType.STRING)
))
.context(QUERY_CONTEXT_UNNEST)
@@ -5722,7 +5722,7 @@ public void testUnnestWithINFiltersWithLeftRewrite()
.dataSource(UnnestDataSource.create(
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE3),
- range("m1", ColumnType.LONG, null, 10L, false, true)
+ range("m1", ColumnType.FLOAT, null, 10.0, false, true)
),
nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING),
in("j0.unnest", ImmutableSet.of("a", "b"))
@@ -5924,8 +5924,8 @@ public void testUnnestWithMultipleAndFiltersOnSelectedColumns()
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE3),
and(
- range("m1", ColumnType.LONG, null, 10L, false, true),
- range("m2", ColumnType.LONG, null, 10L, false, true)
+ range("m1", ColumnType.FLOAT, null, 10.0, false, true),
+ range("m2", ColumnType.DOUBLE, null, 10.0, false, true)
)
),
nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING),
@@ -5964,7 +5964,7 @@ public void testUnnestWithMultipleOrFiltersOnSelectedColumns()
.filters(
or(
equality("j0.unnest", "b", ColumnType.STRING),
- range("m1", ColumnType.LONG, null, 2L, false, true)
+ range("m1", ColumnType.FLOAT, null, 2.0, false, true)
)
)
.columns("j0.unnest")
@@ -6082,8 +6082,8 @@ public void testUnnestWithMultipleOrFiltersOnSelectedNonUnnestedColumns()
FilteredDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE3),
or(
- range("m1", ColumnType.LONG, null, 2L, false, true),
- range("m2", ColumnType.LONG, null, 2L, false, true)
+ range("m1", ColumnType.FLOAT, null, 2.0, false, true),
+ range("m2", ColumnType.DOUBLE, null, 2.0, false, true)
)
),
nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING),
@@ -6122,7 +6122,7 @@ public void testUnnestWithMultipleOrFiltersOnSelectedVirtualColumns()
.filters(
or(
in("j0.unnest", ImmutableSet.of("a", "aa")),
- range("m1", ColumnType.LONG, null, 2L, false, true)
+ range("m1", ColumnType.FLOAT, null, 2.0, false, true)
)
)
.columns("j0.unnest")
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
index d89d618bda13..4ff935a6fbdf 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
@@ -880,9 +880,9 @@ public void testGroupByInsertAddNonDefinedColumnIntoNonSealedCatalogTable()
dimensions(
new DefaultDimensionSpec("v0", "d0", ColumnType.LONG),
new DefaultDimensionSpec("b", "d1", ColumnType.STRING),
- new DefaultDimensionSpec("c", "d3", ColumnType.LONG),
- new DefaultDimensionSpec("d", "d4", ColumnType.LONG),
- new DefaultDimensionSpec("e", "d5", ColumnType.STRING)
+ new DefaultDimensionSpec("c", "d2", ColumnType.LONG),
+ new DefaultDimensionSpec("d", "d3", ColumnType.LONG),
+ new DefaultDimensionSpec("e", "d4", ColumnType.STRING)
)
)
.setAggregatorSpecs(
@@ -902,7 +902,7 @@ public void testGroupByInsertAddNonDefinedColumnIntoNonSealedCatalogTable()
)
.setPostAggregatorSpecs(
expressionPostAgg("p0", "1", ColumnType.LONG),
- expressionPostAgg("p1", "CAST(\"d3\", 'DOUBLE')", ColumnType.DOUBLE)
+ expressionPostAgg("p1", "CAST(\"d2\", 'DOUBLE')", ColumnType.DOUBLE)
)
.setContext(CalciteIngestionDmlTest.PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT)
.build()
@@ -1123,9 +1123,9 @@ public void testGroupByInsertWithSourceIntoCatalogTable()
dimensions(
new DefaultDimensionSpec("v0", "d0", ColumnType.LONG),
new DefaultDimensionSpec("b", "d1", ColumnType.STRING),
- new DefaultDimensionSpec("c", "d3", ColumnType.LONG),
- new DefaultDimensionSpec("d", "d4", ColumnType.LONG),
- new DefaultDimensionSpec("e", "d5", ColumnType.STRING)
+ new DefaultDimensionSpec("c", "d2", ColumnType.LONG),
+ new DefaultDimensionSpec("d", "d3", ColumnType.LONG),
+ new DefaultDimensionSpec("e", "d4", ColumnType.STRING)
)
)
.setAggregatorSpecs(
@@ -1145,7 +1145,7 @@ public void testGroupByInsertWithSourceIntoCatalogTable()
)
.setPostAggregatorSpecs(
expressionPostAgg("p0", "1", ColumnType.LONG),
- expressionPostAgg("p1", "CAST(\"d3\", 'DOUBLE')", ColumnType.DOUBLE)
+ expressionPostAgg("p1", "CAST(\"d2\", 'DOUBLE')", ColumnType.DOUBLE)
)
.setContext(CalciteIngestionDmlTest.PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT)
.build()
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
index f5ace8e84ebb..d78aa5d24de3 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
@@ -3569,8 +3569,8 @@ public void testLeftJoinRightTableCanBeEmpty()
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.filters(range(
"m1",
- ColumnType.LONG,
- 2L,
+ ColumnType.FLOAT,
+ 2.0,
null,
true,
false
@@ -4754,10 +4754,10 @@ public void testJoinWithEquiAndNonEquiCondition(Map queryContext
JoinType.INNER
)
)
- .virtualColumns(expressionVirtualColumn("v0", "(\"m1\" + \"j0.m1\")", ColumnType.DOUBLE))
+ .virtualColumns(expressionVirtualColumn("v0", "(\"m1\" + \"j0.m1\")", ColumnType.FLOAT))
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(
- equality("v0", 6.0, ColumnType.DOUBLE)
+ equality("v0", 6.0, ColumnType.FLOAT)
)
.columns("m1", "j0.m1")
.columnTypes(ColumnType.FLOAT, ColumnType.FLOAT)
@@ -4970,8 +4970,8 @@ public void testNestedGroupByOnInlineDataSourceWithFilter(Map qu
.dataSource(CalciteTests.DATASOURCE1)
.intervals(querySegmentSpec(Intervals.of(
"2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z")))
- .columns("dim1", "m2")
- .columnTypes(ColumnType.STRING, ColumnType.DOUBLE)
+ .columns("dim1")
+ .columnTypes(ColumnType.STRING)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(queryContext)
.build()
@@ -5665,11 +5665,11 @@ public void testPlanWithInFilterMoreThanInSubQueryThreshold()
new Object[]{29064L}
),
RowSignature.builder()
- .add("ROW_VALUE", ColumnType.LONG)
+ .add("EXPR$0", ColumnType.LONG)
.build()
),
"j0.",
- "(\"l1\" == \"j0.ROW_VALUE\")",
+ "(\"l1\" == \"j0.EXPR$0\")",
JoinType.INNER,
null,
ExprMacroTable.nil(),
@@ -5690,7 +5690,7 @@ public void testPlanWithInFilterMoreThanInSubQueryThreshold()
);
}
- @NotYetSupported({Modes.SORT_REMOVE_TROUBLE, Modes.DD_SORT_REMOVE_TROUBLE})
+ @NotYetSupported(Modes.SORT_REMOVE_TROUBLE)
@MethodSource("provideQueryContexts")
@ParameterizedTest(name = "{0}")
public void testRegressionFilteredAggregatorsSubqueryJoins(Map queryContext)
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
index f4b65da38850..e4055f84513f 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java
@@ -518,9 +518,8 @@ public void testFilterNotInAndIsNotNull()
buildFilterTestExpectedQuery(
expressionVirtualColumn("v0", LOOKUP_EXPRESSION, ColumnType.STRING),
and(
- not(equality("v0", "x6", ColumnType.STRING)),
- not(equality("v0", "nonexistent", ColumnType.STRING)),
- notNull("v0")
+ notNull("v0"),
+ not(in("v0", ImmutableList.of("nonexistent", "x6")))
)
),
ImmutableList.of(new Object[]{"xabc", 1L})
@@ -574,11 +573,8 @@ public void testFilterNotInOrIsNull()
buildFilterTestExpectedQuery(
expressionVirtualColumn("v0", LOOKUP_EXPRESSION, ColumnType.STRING),
or(
- and(
- not(equality("v0", "x6", ColumnType.STRING)),
- not(equality("v0", "nonexistent", ColumnType.STRING))
- ),
- isNull("v0")
+ isNull("v0"),
+ not(in("v0", ImmutableList.of("nonexistent", "x6")))
)
),
ImmutableList.of(
@@ -617,7 +613,7 @@ public void testFilterNotIn()
QUERY_CONTEXT,
buildFilterTestExpectedQuery(
expressionVirtualColumn("v0", LOOKUP_EXPRESSION, ColumnType.STRING),
- and(not(equality("v0", "x6", ColumnType.STRING)), not(equality("v0", "nonexistent", ColumnType.STRING)))
+ not(in("v0", ImmutableList.of("nonexistent", "x6")))
),
ImmutableList.of(new Object[]{"xabc", 1L})
);
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
index 6e2bb2cddeee..f46408da189e 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
@@ -1072,9 +1072,6 @@ public void testStringToMVOfConstant()
@Test
public void testStringToMVOfConstantGroupedBy()
{
- // Cannot vectorize due to usage of expressions.
- cannotVectorize();
-
testBuilder()
.sql("SELECT m1, STRING_TO_MV('a,b', ',') AS mv FROM druid.numfoo GROUP BY 1, 2")
.expectedQuery(
@@ -1082,30 +1079,23 @@ public void testStringToMVOfConstantGroupedBy()
.setDataSource(CalciteTests.DATASOURCE3)
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
- .setVirtualColumns(
- expressionVirtualColumn("v0", "string_to_array('a,b',',')", ColumnType.STRING)
- )
.setDimensions(dimensions(
- new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT),
- new DefaultDimensionSpec("v0", "d1", ColumnType.STRING)
+ new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)
))
+ .setPostAggregatorSpecs(
+ expressionPostAgg("p0", "string_to_array('a,b',',')", ColumnType.STRING)
+ )
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
)
.expectedResults(
ImmutableList.of(
- new Object[]{1.0f, "a"},
- new Object[]{1.0f, "b"},
- new Object[]{2.0f, "a"},
- new Object[]{2.0f, "b"},
- new Object[]{3.0f, "a"},
- new Object[]{3.0f, "b"},
- new Object[]{4.0f, "a"},
- new Object[]{4.0f, "b"},
- new Object[]{5.0f, "a"},
- new Object[]{5.0f, "b"},
- new Object[]{6.0f, "a"},
- new Object[]{6.0f, "b"}
+ new Object[]{1.0f, "[\"a\",\"b\"]"},
+ new Object[]{2.0f, "[\"a\",\"b\"]"},
+ new Object[]{3.0f, "[\"a\",\"b\"]"},
+ new Object[]{4.0f, "[\"a\",\"b\"]"},
+ new Object[]{5.0f, "[\"a\",\"b\"]"},
+ new Object[]{6.0f, "[\"a\",\"b\"]"}
)
)
.run();
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
index 842dc283fc6e..0665304fa0f8 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
@@ -7249,7 +7249,7 @@ public void testNvlJsonValueDoubleMissingColumn()
),
expressionVirtualColumn("v2", "notnull(nvl(\"v1\",1.0))", ColumnType.LONG)
)
- .filters(range("v0", ColumnType.LONG, 0.0, null, true, false))
+ .filters(range("v0", ColumnType.DOUBLE, 0.0, null, true, false))
.limit(1)
.columns("v1", "v0", "v2")
.columnTypes(ColumnType.DOUBLE, ColumnType.DOUBLE, ColumnType.LONG)
@@ -7281,7 +7281,7 @@ public void testNvlJsonValueDoubleSometimesMissing()
.virtualColumns(
new NestedFieldVirtualColumn("nest", "$.y", "v0", ColumnType.DOUBLE),
expressionVirtualColumn("v1", "nvl(\"v0\",1.0)", ColumnType.DOUBLE),
- expressionVirtualColumn("v2", "(nvl(\"v0\",1.0) > 0)", ColumnType.LONG),
+ expressionVirtualColumn("v2", "(nvl(\"v0\",1.0) > 0.0)", ColumnType.LONG),
expressionVirtualColumn("v3", "(nvl(\"v0\",1.0) == 1.0)", ColumnType.LONG)
)
.columns("v0", "v1", "v2", "v3")
@@ -7325,7 +7325,7 @@ public void testNvlJsonValueDoubleSometimesMissingRangeFilter()
new NestedFieldVirtualColumn("nest", "$.y", "v1", ColumnType.DOUBLE),
expressionVirtualColumn("v2", "notnull(nvl(\"v1\",1.0))", ColumnType.LONG)
)
- .filters(range("v0", ColumnType.LONG, 0.0, null, true, false))
+ .filters(range("v0", ColumnType.DOUBLE, 0.0, null, true, false))
.columns("v1", "v0", "v2")
.columnTypes(ColumnType.DOUBLE, ColumnType.DOUBLE, ColumnType.LONG)
.build()
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index fcb944852453..848b54604b71 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -182,9 +182,14 @@ public void testInformationSchemaTables()
testQuery(
"SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, IS_JOINABLE, IS_BROADCAST\n"
+ "FROM INFORMATION_SCHEMA.TABLES\n"
- + "WHERE TABLE_TYPE IN ('SYSTEM_TABLE', 'TABLE', 'VIEW')",
+ + "WHERE TABLE_TYPE IN ('SYSTEM_TABLE', 'TABLE', 'VIEW')\n"
+ + "ORDER BY TABLE_SCHEMA, TABLE_NAME",
ImmutableList.of(),
ImmutableList.builder()
+ .add(new Object[]{"INFORMATION_SCHEMA", "COLUMNS", "SYSTEM_TABLE", "NO", "NO"})
+ .add(new Object[]{"INFORMATION_SCHEMA", "ROUTINES", "SYSTEM_TABLE", "NO", "NO"})
+ .add(new Object[]{"INFORMATION_SCHEMA", "SCHEMATA", "SYSTEM_TABLE", "NO", "NO"})
+ .add(new Object[]{"INFORMATION_SCHEMA", "TABLES", "SYSTEM_TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.ARRAYS_DATASOURCE, "TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.BROADCAST_DATASOURCE, "TABLE", "YES", "YES"})
.add(new Object[]{"druid", CalciteTests.DATASOURCE1, "TABLE", "NO", "NO"})
@@ -200,10 +205,6 @@ public void testInformationSchemaTables()
.add(new Object[]{"druid", CalciteTests.USERVISITDATASOURCE, "TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.WIKIPEDIA, "TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.WIKIPEDIA_FIRST_LAST, "TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "COLUMNS", "SYSTEM_TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "ROUTINES", "SYSTEM_TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "SCHEMATA", "SYSTEM_TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "TABLES", "SYSTEM_TABLE", "NO", "NO"})
.add(new Object[]{"lookup", "lookyloo", "TABLE", "YES", "YES"})
.add(new Object[]{"lookup", "lookyloo-chain", "TABLE", "YES", "YES"})
.add(new Object[]{"lookup", "lookyloo121", "TABLE", "YES", "YES"})
@@ -226,10 +227,15 @@ public void testInformationSchemaTables()
PLANNER_CONFIG_DEFAULT,
"SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, IS_JOINABLE, IS_BROADCAST\n"
+ "FROM INFORMATION_SCHEMA.TABLES\n"
- + "WHERE TABLE_TYPE IN ('SYSTEM_TABLE', 'TABLE', 'VIEW')",
+ + "WHERE TABLE_TYPE IN ('SYSTEM_TABLE', 'TABLE', 'VIEW')\n"
+ + "ORDER BY TABLE_SCHEMA, TABLE_NAME",
CalciteTests.SUPER_USER_AUTH_RESULT,
ImmutableList.of(),
ImmutableList.builder()
+ .add(new Object[]{"INFORMATION_SCHEMA", "COLUMNS", "SYSTEM_TABLE", "NO", "NO"})
+ .add(new Object[]{"INFORMATION_SCHEMA", "ROUTINES", "SYSTEM_TABLE", "NO", "NO"})
+ .add(new Object[]{"INFORMATION_SCHEMA", "SCHEMATA", "SYSTEM_TABLE", "NO", "NO"})
+ .add(new Object[]{"INFORMATION_SCHEMA", "TABLES", "SYSTEM_TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.ARRAYS_DATASOURCE, "TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.BROADCAST_DATASOURCE, "TABLE", "YES", "YES"})
.add(new Object[]{"druid", CalciteTests.DATASOURCE1, "TABLE", "NO", "NO"})
@@ -246,10 +252,6 @@ public void testInformationSchemaTables()
.add(new Object[]{"druid", CalciteTests.USERVISITDATASOURCE, "TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.WIKIPEDIA, "TABLE", "NO", "NO"})
.add(new Object[]{"druid", CalciteTests.WIKIPEDIA_FIRST_LAST, "TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "COLUMNS", "SYSTEM_TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "ROUTINES", "SYSTEM_TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "SCHEMATA", "SYSTEM_TABLE", "NO", "NO"})
- .add(new Object[]{"INFORMATION_SCHEMA", "TABLES", "SYSTEM_TABLE", "NO", "NO"})
.add(new Object[]{"lookup", "lookyloo", "TABLE", "YES", "YES"})
.add(new Object[]{"lookup", "lookyloo-chain", "TABLE", "YES", "YES"})
.add(new Object[]{"lookup", "lookyloo121", "TABLE", "YES", "YES"})
@@ -2333,7 +2335,7 @@ public void testFilterOnFloat()
.intervals(querySegmentSpec(Filtration.eternity()))
.granularity(Granularities.ALL)
.aggregators(aggregators(new CountAggregatorFactory("a0")))
- .filters(equality("m1", 1.0, ColumnType.DOUBLE))
+ .filters(equality("m1", 1.0, ColumnType.FLOAT))
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
@@ -2434,8 +2436,8 @@ public void testHavingOnDoubleSum()
having(
range(
"a0",
- ColumnType.LONG,
- 1L,
+ ColumnType.DOUBLE,
+ 1.0,
null,
true,
false
@@ -2677,8 +2679,8 @@ public void testHavingOnFloatSum()
having(
range(
"a0",
- ColumnType.LONG,
- 1L,
+ ColumnType.FLOAT,
+ 1.0,
null,
true,
false
@@ -3204,7 +3206,7 @@ public void testGroupByCaseWhenOfTripleAnd()
.setVirtualColumns(
expressionVirtualColumn(
"v0",
- "case_searched((((\"m1\" > 1) && (\"m1\" < 5)) && (\"cnt\" == 1)),'x',null)",
+ "case_searched((((\"m1\" > 1.0) && (\"m1\" < 5.0)) && (\"cnt\" == 1)),'x',null)",
ColumnType.STRING
)
)
@@ -3586,7 +3588,7 @@ public void testDoublePredicateFilterNulls()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.granularity(Granularities.ALL)
- .filters(range("dbl1", ColumnType.LONG, 0L, null, true, false))
+ .filters(range("dbl1", ColumnType.DOUBLE, 0.0, null, true, false))
.aggregators(aggregators(new CountAggregatorFactory("a0")))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -3607,7 +3609,7 @@ public void testFloatPredicateFilterNulls()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.granularity(Granularities.ALL)
- .filters(range("f1", ColumnType.LONG, 0L, null, true, false))
+ .filters(range("f1", ColumnType.FLOAT, 0.0, null, true, false))
.aggregators(aggregators(new CountAggregatorFactory("a0")))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -3716,7 +3718,7 @@ public void testCoalesceColumnsFilter()
)
)
.setDimFilter(or(
- in("dim2", ImmutableSet.of("a", "abc")),
+ and(notNull("dim2"), in("dim2", ImmutableSet.of("a", "abc"))),
and(isNull("dim2"), in("dim1", ImmutableList.of("a", "abc")))
))
.setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
@@ -3753,7 +3755,7 @@ public void testCoalesceColumnsFilterWithEquality()
)
)
.setDimFilter(or(
- in("dim2", ImmutableSet.of("a", "abc")),
+ and(notNull("dim2"), in("dim2", ImmutableSet.of("a", "abc"))),
and(NullFilter.forColumn("dim2"), in("dim1", ImmutableList.of("a", "abc")))
))
.setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
@@ -4483,15 +4485,15 @@ public void testFilterOnStringAsNumber()
expressionVirtualColumn(
"v0",
"floor(CAST(\"dim1\", 'DOUBLE'))",
- ColumnType.DOUBLE
+ ColumnType.FLOAT
)
)
.setDimFilter(
or(
equality("dim1", 10L, ColumnType.LONG),
and(
- equality("v0", 10.0, ColumnType.DOUBLE),
- range("dim1", ColumnType.DOUBLE, 9.0, 10.5, true, false)
+ equality("v0", 10.0, ColumnType.FLOAT),
+ range("dim1", ColumnType.FLOAT, 9.0, 10.5, true, false)
)
)
)
@@ -4516,7 +4518,7 @@ public void testFilterOnStringAsNumber2()
.setGranularity(Granularities.ALL)
.setDimensions(dimensions(new DefaultDimensionSpec("dim1", "d0")))
.setDimFilter(
- range("dim1", ColumnType.DOUBLE, 9.0, 10.5, true, false)
+ range("dim1", ColumnType.FLOAT, 9.0, 10.5, true, false)
)
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
@@ -5080,7 +5082,7 @@ public void testExpressionFilteringAndGrouping()
.setVirtualColumns(
expressionVirtualColumn("v0", "(floor((\"m1\" / 2)) * 2)", ColumnType.FLOAT)
)
- .setDimFilter(range("v0", ColumnType.LONG, -1L, null, true, false))
+ .setDimFilter(range("v0", ColumnType.FLOAT, -1.0, null, true, false))
.setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.FLOAT)))
.setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0")))
.setLimitSpec(
@@ -5179,7 +5181,7 @@ public void testExpressionFilteringAndGroupingOnStringCastToNumber()
)
)
.setDimFilter(
- range("v0", ColumnType.LONG, -1L, null, true, false)
+ range("v0", ColumnType.FLOAT, -1.0, null, true, false)
)
.setDimensions(dimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.FLOAT)))
.setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0")))
@@ -5936,7 +5938,7 @@ public void testCountStarWithBoundFilterSimplifyOnMetric()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(querySegmentSpec(Filtration.eternity()))
.granularity(Granularities.ALL)
- .filters(range("m1", ColumnType.DOUBLE, 2.5, 3.5, true, true))
+ .filters(range("m1", ColumnType.FLOAT, 2.5, 3.5, true, true))
.aggregators(aggregators(new CountAggregatorFactory("a0")))
.context(QUERY_CONTEXT_DEFAULT)
.build()
@@ -7073,7 +7075,7 @@ public void testCountDistinctOfCaseWhen()
false,
true
),
- range("m1", ColumnType.LONG, 4L, null, false, false)
+ range("m1", ColumnType.FLOAT, 4.0, null, false, false)
),
new FilteredAggregatorFactory(
new CardinalityAggregatorFactory(
@@ -7083,11 +7085,11 @@ public void testCountDistinctOfCaseWhen()
false,
true
),
- range("m1", ColumnType.LONG, 4L, null, false, false)
+ range("m1", ColumnType.FLOAT, 4.0, null, false, false)
),
new FilteredAggregatorFactory(
new HyperUniquesAggregatorFactory("a2", "unique_dim1", false, true),
- range("m1", ColumnType.LONG, 4L, null, false, false)
+ range("m1", ColumnType.FLOAT, 4.0, null, false, false)
)
)
)
@@ -8752,13 +8754,11 @@ public void testGroupByLimitPushdownExtraction()
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(
- expressionVirtualColumn("v0", "'a'", ColumnType.STRING),
- expressionVirtualColumn("v1", "substring(\"dim5\", 0, 1)", ColumnType.STRING)
+ expressionVirtualColumn("v0", "substring(\"dim5\", 0, 1)", ColumnType.STRING)
)
.setDimensions(
dimensions(
- new DefaultDimensionSpec("v0", "d0"),
- new DefaultDimensionSpec("v1", "d1")
+ new DefaultDimensionSpec("v0", "d0")
)
)
.setDimFilter(equality("dim4", "a", ColumnType.STRING))
@@ -8767,6 +8767,7 @@ public void testGroupByLimitPushdownExtraction()
new CountAggregatorFactory("a0")
)
)
+ .setPostAggregatorSpecs(expressionPostAgg("p0", "'a'", ColumnType.STRING))
.setLimitSpec(
new DefaultLimitSpec(
ImmutableList.of(),
@@ -11207,8 +11208,8 @@ public void testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim()
having(
range(
"a0",
- ColumnType.LONG,
- 1L,
+ ColumnType.DOUBLE,
+ 1.0,
null,
true,
false
@@ -12955,7 +12956,7 @@ public void testFilterFloatDimension()
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("dim1")
.columnTypes(ColumnType.STRING)
- .filters(equality("f1", 0.1, ColumnType.DOUBLE))
+ .filters(equality("f1", 0.1, ColumnType.FLOAT))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(1)
.context(QUERY_CONTEXT_DEFAULT)
@@ -13239,9 +13240,8 @@ public void testGroupByWithLiteralInSubqueryGrouping()
GroupByQuery.builder()
.setDataSource(CalciteTests.DATASOURCE3)
.setVirtualColumns(
- expressionVirtualColumn("v0", "'dummy'", ColumnType.STRING),
expressionVirtualColumn(
- "v1",
+ "v0",
"case_searched((\"dim4\" == 'b'),\"dim4\",null)",
ColumnType.STRING
)
@@ -13249,10 +13249,10 @@ public void testGroupByWithLiteralInSubqueryGrouping()
.setInterval(querySegmentSpec(Filtration.eternity()))
.setDimensions(
dimensions(
- new DefaultDimensionSpec("v0", "d0", ColumnType.STRING),
- new DefaultDimensionSpec("v1", "d1", ColumnType.STRING)
+ new DefaultDimensionSpec("v0", "d0", ColumnType.STRING)
)
)
+ .setPostAggregatorSpecs(expressionPostAgg("p0", "'dummy'", ColumnType.STRING))
.setGranularity(Granularities.ALL)
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
@@ -13395,6 +13395,7 @@ public void testRepeatedIdenticalVirtualExpressionGrouping()
new DefaultDimensionSpec("v0", "d0", ColumnType.LONG)
)
)
+ .setPostAggregatorSpecs(expressionPostAgg("p0", "1", ColumnType.LONG))
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
@@ -14779,7 +14780,7 @@ public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithMultipleCons
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(
and(
- equality("m1", 2.0, ColumnType.DOUBLE),
+ equality("m1", 2.0, ColumnType.FLOAT),
equality("dim1", "10.1", ColumnType.STRING)
)
)
@@ -15256,7 +15257,7 @@ public void testFilterWithNVLAndNotIn()
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
.columns("__time", "dim1")
.columnTypes(ColumnType.LONG, ColumnType.STRING)
- .filters(not(in("dim1", Arrays.asList("", "a"))))
+ .filters(and(notNull("dim1"), not(in("dim1", Arrays.asList("", "a")))))
.build()
),
ImmutableList.of(
@@ -15283,7 +15284,7 @@ public void testFilterWithNVLAndInIsNotTrue()
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
.columns("__time", "dim1")
.columnTypes(ColumnType.LONG, ColumnType.STRING)
- .filters(not(in("dim1", Arrays.asList("", "a"))))
+ .filters(and(notNull("dim1"), not(in("dim1", Arrays.asList("", "a")))))
.build()
),
ImmutableList.of(
@@ -15310,7 +15311,7 @@ public void testFilterWithNvlishCaseAndNotIn()
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
.columns("__time", "dim1")
.columnTypes(ColumnType.LONG, ColumnType.STRING)
- .filters(not(in("dim1", Arrays.asList("", "a"))))
+ .filters(and(notNull("dim1"), not(in("dim1", Arrays.asList("", "a")))))
.build()
),
ImmutableList.of(
@@ -15337,7 +15338,7 @@ public void testFilterWithNvlishCase2AndNotIn()
"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z")))
.columns("__time", "dim1")
.columnTypes(ColumnType.LONG, ColumnType.STRING)
- .filters(not(in("dim1", Arrays.asList("", "a"))))
+ .filters(and(notNull("dim1"), not(in("dim1", Arrays.asList("", "a")))))
.build()
),
ImmutableList.of(
@@ -15490,22 +15491,11 @@ public void testDistinctNotSupportedWithWindow()
assertThat(e, invalidSqlContains("DISTINCT is not supported for window functions"));
}
- @Test
- public void testLogicalCorrelateTrimFieldsStillNeeded()
- {
- assertEquals(
- "1.37.0",
- RelNode.class.getPackage().getImplementationVersion(),
- "Calcite version changed; check if DruidRelFieldTrimmer#trimFields(LogicalCorrelate correlate,...) is still needed or not!"
- );
- }
-
-
@Test
public void testUnSupportedAggInSelectWindow()
{
assertEquals(
- "1.37.0",
+ "1.41.0",
RelNode.class.getPackage().getImplementationVersion(),
"Calcite version changed; check if CALCITE-6500 is fixed and update:\n * method DruidSqlValidator#validateWindowClause"
);
@@ -15568,8 +15558,8 @@ public void testInGroupByLimitOutGroupByOrderBy()
.setFilter(
range(
"d0",
- ColumnType.LONG,
- 2L,
+ ColumnType.DOUBLE,
+ 2.0,
null,
true,
false
@@ -15648,8 +15638,8 @@ public void testInGroupByOrderByLimitOutGroupByOrderByLimit()
.setFilter(
range(
"d0",
- ColumnType.LONG,
- 2L,
+ ColumnType.DOUBLE,
+ 2.0,
null,
true,
false
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
index 1550a7a8066c..59fef5a20a7a 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
@@ -21,7 +21,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import org.apache.calcite.rel.RelNode;
import org.apache.druid.error.DruidException;
import org.apache.druid.error.DruidExceptionMatcher;
import org.apache.druid.java.util.common.DateTimes;
@@ -64,8 +63,6 @@
import java.util.List;
import java.util.Map;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
public class CalciteSelectQueryTest extends BaseCalciteQueryTest
{
@Test
@@ -2171,7 +2168,7 @@ public void testCountDistinctNonApproximateEmptySet()
dimensions(
new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)))
.setDimFilter(
- range("m1", ColumnType.DOUBLE, null, -1.0, false, true))
+ range("m1", ColumnType.FLOAT, null, -1.0, false, true))
.build())
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
@@ -2210,7 +2207,7 @@ public void testCountDistinctNonApproximateBasic()
dimensions(
new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)))
.setDimFilter(
- range("m1", ColumnType.DOUBLE, null, 111.0, false, true))
+ range("m1", ColumnType.FLOAT, null, 111.0, false, true))
.build())
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
@@ -2387,12 +2384,6 @@ public void testTimseriesResultCachePullConsistency()
@Test
public void testSqlToRelInConversion()
{
- assertEquals(
- "1.37.0",
- RelNode.class.getPackage().getImplementationVersion(),
- "Calcite version changed; check if CALCITE-6435 is fixed and remove:\n * method CalciteRulesManager#sqlToRelWorkaroundProgram\n * FixIncorrectInExpansionTypes class\n* this assertion"
- );
-
testBuilder()
.sql(
"SELECT channel FROM wikipedia\n"
@@ -2411,16 +2402,10 @@ public void testSqlToRelInConversion()
@Test
public void testRejectHavingWithWindowExpression()
{
- assertEquals(
- "1.37.0",
- RelNode.class.getPackage().getImplementationVersion(),
- "Calcite version changed; check if CALCITE-6473 is fixed and remove:\n * this assertion\n * DruidSqlValidator#validateHavingClause"
- );
-
testQueryThrows(
"SELECT cityName,sum(1) OVER () as w FROM wikipedia group by cityName HAVING w > 10",
DruidException.class,
- invalidSqlContains("Window functions are not allowed in HAVING")
+ invalidSqlContains("not permitted in the HAVING clause")
);
}
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
index 6e314058433d..22a79a1fbd9b 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java
@@ -568,8 +568,7 @@ public void testUsingSubqueryAsFilterOnTwoColumns(String testName, Map"),
+ SORT_REMOVE_TROUBLE(Scope.DECOUPLED, AssertionError.class, ""),
UNNEST_INLINED(Scope.DECOUPLED, Exception.class, "Missing conversion is Uncollect"),
UNNEST_RESULT_MISMATCH(Scope.DECOUPLED, AssertionError.class, "(Result count mismatch|column content mismatch)"),
@@ -115,7 +115,6 @@ enum Modes
DD_WINDOW(Scope.DECOUPLED_DART, DruidException.class, "DruidWindow.DRUID_LOGICAL"),
DD_UNNEST_RESULT_MISMATCH(Scope.DECOUPLED_DART, AssertionError.class, "(Result count mismatch|column content mismatch)"),
DD_UNNEST_INLINED(Scope.DECOUPLED_DART, Exception.class, "Missing conversion is Uncollect"),
- DD_SORT_REMOVE_TROUBLE(Scope.DECOUPLED_DART, DruidException.class, "Calcite assertion violated.*Sort\\."),
DD_JOIN_CONDITION_NORMALIZATION(Scope.DECOUPLED_DART, DruidException.class, "Cannot handle equality"),
DD_RESULT_MISMATCH_FLOAT_DOUBLE(Scope.DECOUPLED_DART, AssertionError.class, "column content mismatch");
// @formatter:on
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
index b8f66a8439ce..fe413defa223 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
@@ -2813,7 +2813,7 @@ public void testCalciteLiteralToDruidLiteral()
plannerContext,
rexBuilder.makeExactLiteral(
BigDecimal.valueOf(123L),
- rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DECIMAL)
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, 19, 0)
)
)
);
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
index 519d4410ad0b..f58340d2b894 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/DruidRexExecutorTest.java
@@ -238,7 +238,7 @@ public void testArrayOfDoublesReduction()
{
DruidRexExecutor rexy = new DruidRexExecutor(PLANNER_CONTEXT);
List reduced = new ArrayList<>();
- BasicSqlType basicSqlType = new BasicSqlType(DruidTypeSystem.INSTANCE, SqlTypeName.DECIMAL);
+ BasicSqlType basicSqlType = new BasicSqlType(DruidTypeSystem.INSTANCE, SqlTypeName.DECIMAL, 19, 10);
ArraySqlType arraySqlType = new ArraySqlType(basicSqlType, false);
List elements = ImmutableList.of(BigDecimal.valueOf(50.12), BigDecimal.valueOf(12.1));
RexNode literal = rexBuilder.makeLiteral(elements, arraySqlType, true);
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidRelTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidRelTest.java
index 292ba32cff88..61dcd59ea1d4 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidRelTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/rel/DruidRelTest.java
@@ -24,8 +24,10 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.Convention;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.core.Correlate;
import org.apache.calcite.rel.core.CorrelationId;
@@ -76,6 +78,7 @@
import static org.apache.druid.sql.calcite.util.CalciteTests.POLICY_RESTRICTION;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -109,8 +112,7 @@ public class DruidRelTest
private RelOptTable mockRelOptTable;
@Mock
private RelOptCluster mockRelOptCluster;
- @Mock
- private RelTraitSet mockRelTraitSet;
+ private final RelTraitSet mockRelTraitSet = RelTraitSet.createEmpty().plus(Convention.NONE);
@Mock
private PlannerContext mockPlannerContext;
@@ -121,6 +123,7 @@ public void setup() throws Exception
{
MockitoAnnotations.openMocks(this);
when(mockRelOptCluster.getTypeFactory()).thenReturn(DEFAULT_TYPE_FACTORY);
+ when(mockRelOptCluster.traitSetOf(any(RelTrait.class))).thenReturn(mockRelTraitSet);
when(mockRelOptTable.getRowType()).thenReturn(REC_TYPE);
PlannerToolbox mockPlannerToolbox = mock(PlannerToolbox.class);
@@ -210,8 +213,7 @@ public void testDruidOuterQueryRel()
{
// Arrange
PartialDruidQuery partialDruidQuerySpy = spy(druidQueryRelNode.getPartialDruidQuery());
- when(partialDruidQuerySpy.getTraitSet(any(), any())).thenReturn(mockRelTraitSet);
- when(mockRelTraitSet.containsIfApplicable(any())).thenReturn(true);
+ doReturn(mockRelTraitSet).when(partialDruidQuerySpy).getTraitSet(any(), any());
PartialDruidQuery inputQuery = PartialDruidQuery.createOuterQuery(
partialDruidQuerySpy,
druidQueryRelNode.getPlannerContext()
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java
index 6771cb4e2a3b..54619b5b10bd 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java
@@ -20,6 +20,7 @@
package org.apache.druid.sql.calcite.rule;
import com.google.common.collect.ImmutableList;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexLiteral;
@@ -93,9 +94,13 @@ public void testGetValueFromLiteral()
private static RexLiteral makeLiteral(Comparable val, SqlTypeName typeName, Class javaType)
{
+ final RelDataType type =
+ typeName == SqlTypeName.DECIMAL
+ ? DruidTypeSystem.TYPE_FACTORY.createSqlType(typeName, 10, 2)
+ : DruidTypeSystem.TYPE_FACTORY.createSqlType(typeName);
return (RexLiteral) new RexBuilder(DruidTypeSystem.TYPE_FACTORY).makeLiteral(
typeName == SqlTypeName.DECIMAL && val != null ? new BigDecimal(String.valueOf(val)) : val,
- DruidTypeSystem.TYPE_FACTORY.createSqlType(typeName),
+ type,
false
);
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
index bb381b18718f..f6b91e408dbe 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
@@ -376,7 +376,7 @@ public void setUp(@TempDir File tmpDir) throws Exception
private final long totalMemory = JvmUtils.getTotalMemory();
private final String version = GuavaUtils.firstNonNull(
- SystemSchemaTest.class.getPackage().getImplementationVersion(),
+ DruidNode.class.getPackage().getImplementationVersion(),
DruidNode.UNKNOWN_VERSION
);
diff --git a/sql/src/test/quidem/druidjoinrule-fullouter-dataset/left-dims-index.json b/sql/src/test/quidem/druidjoinrule-fullouter-dataset/left-dims-index.json
new file mode 100644
index 000000000000..1c897d8f239a
--- /dev/null
+++ b/sql/src/test/quidem/druidjoinrule-fullouter-dataset/left-dims-index.json
@@ -0,0 +1,49 @@
+{
+ "type": "index_parallel",
+ "spec": {
+ "ioConfig": {
+ "type": "index_parallel",
+ "inputSource": {
+ "type": "inline",
+ "data": "{\"timestamp\":\"2000-01-01T00:00:00\",\"a\":\"abc\"}\n{\"timestamp\":\"2000-01-01T00:00:00\",\"a\":\"xyz\"}\n"
+ },
+ "inputFormat": {
+ "type": "json"
+ },
+ "appendToExisting": false
+ },
+ "tuningConfig": {
+ "type": "index_parallel",
+ "partitionsSpec": {
+ "type": "dynamic",
+ "maxRowsPerSegment": 10000000,
+ "maxTotalRows": 500000000
+ },
+ "maxNumConcurrentSubTasks": 1
+ },
+ "dataSchema": {
+ "dataSource": "left_dims",
+ "granularitySpec": {
+ "type": "uniform",
+ "queryGranularity": "NONE",
+ "rollup": false,
+ "segmentGranularity": "YEAR"
+ },
+ "timestampSpec": {
+ "column": "timestamp",
+ "format": "auto"
+ },
+ "dimensionsSpec": {
+ "dimensions": [
+ {
+ "type": "string",
+ "name": "a"
+ }
+ ]
+ },
+ "metricsSpec": null,
+ "transformSpec": null
+ }
+ },
+ "context": null
+}
diff --git a/sql/src/test/quidem/druidjoinrule-fullouter-dataset/right-nulls-index.json b/sql/src/test/quidem/druidjoinrule-fullouter-dataset/right-nulls-index.json
new file mode 100644
index 000000000000..6915f13c53d3
--- /dev/null
+++ b/sql/src/test/quidem/druidjoinrule-fullouter-dataset/right-nulls-index.json
@@ -0,0 +1,49 @@
+{
+ "type": "index_parallel",
+ "spec": {
+ "ioConfig": {
+ "type": "index_parallel",
+ "inputSource": {
+ "type": "inline",
+ "data": "{\"timestamp\":\"2000-01-01T00:00:00\",\"k\":\"abc\"}\n{\"timestamp\":\"2000-01-01T00:00:00\",\"k\":null}\n"
+ },
+ "inputFormat": {
+ "type": "json"
+ },
+ "appendToExisting": false
+ },
+ "tuningConfig": {
+ "type": "index_parallel",
+ "partitionsSpec": {
+ "type": "dynamic",
+ "maxRowsPerSegment": 10000000,
+ "maxTotalRows": 500000000
+ },
+ "maxNumConcurrentSubTasks": 1
+ },
+ "dataSchema": {
+ "dataSource": "right_nulls",
+ "granularitySpec": {
+ "type": "uniform",
+ "queryGranularity": "NONE",
+ "rollup": false,
+ "segmentGranularity": "YEAR"
+ },
+ "timestampSpec": {
+ "column": "timestamp",
+ "format": "auto"
+ },
+ "dimensionsSpec": {
+ "dimensions": [
+ {
+ "type": "string",
+ "name": "k"
+ }
+ ]
+ },
+ "metricsSpec": null,
+ "transformSpec": null
+ }
+ },
+ "context": null
+}
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/filtered_sum.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/filtered_sum.iq
index 7da2f4ffd9d6..23f91b897ead 100644
--- a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/filtered_sum.iq
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/filtered_sum.iq
@@ -52,7 +52,7 @@ SELECT COUNT(1)FILTER(WHERE l1=7),COUNT(1)FILTER(WHERE l1!=7),MIN(l2) is null,
LogicalProject(EXPR$0=[$0], EXPR$1=[$1], EXPR$2=[IS NULL($2)], EXPR$3=[$3], EXPR$4=[$4])
LogicalAggregate(group=[{}], EXPR$0=[COUNT() FILTER $0], EXPR$1=[COUNT() FILTER $1], agg#2=[MIN($2)], EXPR$3=[SUM($3)], EXPR$4=[SUM($2) FILTER $0])
LogicalProject($f1=[IS TRUE(=($0, 7))], $f2=[IS TRUE(<>($0, 7))], l2=[$1], $f4=[CASE(=($0, 7), $1, 0:BIGINT)])
- LogicalFilter(condition=[SEARCH($0, Sarg[(0..10)])])
+ LogicalFilter(condition=[SEARCH($0, Sarg[(0L:BIGINT..10L:BIGINT)]:BIGINT)])
LogicalProject(l1=[$11], l2=[$12])
LogicalTableScan(table=[[druid, numfoo]])
@@ -113,7 +113,7 @@ SELECT COUNT(1)FILTER(WHERE l1=7),COUNT(1)FILTER(WHERE l1!=7),MIN(l2) is null,
LogicalProject(EXPR$0=[$0], EXPR$1=[$1], EXPR$2=[IS NULL($2)], EXPR$3=[$3], EXPR$4=[$4])
LogicalAggregate(group=[{}], EXPR$0=[COUNT() FILTER $0], EXPR$1=[COUNT() FILTER $1], agg#2=[MIN($2)], EXPR$3=[SUM($3)], EXPR$4=[SUM($2) FILTER $0])
LogicalProject($f1=[IS TRUE(=($0, 7))], $f2=[IS TRUE(<>($0, 7))], l2=[$1], $f4=[CASE(=($0, 7), $1, 0:BIGINT)])
- LogicalFilter(condition=[SEARCH($0, Sarg[(0..10)])])
+ LogicalFilter(condition=[SEARCH($0, Sarg[(0L:BIGINT..10L:BIGINT)]:BIGINT)])
LogicalProject(l1=[$11], l2=[$12])
LogicalTableScan(table=[[druid, numfoo]])
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/full_outer_constant_project.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/full_outer_constant_project.iq
new file mode 100644
index 000000000000..7e2dc61ff26d
--- /dev/null
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/full_outer_constant_project.iq
@@ -0,0 +1,108 @@
+!use druidtest://?componentSupplier=StandardComponentSupplier&datasets=sql/src/test/quidem/druidjoinrule-fullouter-dataset
+!set outputformat mysql
+
+# Ensures that FULL OUTER JOIN constant projects are handled properly by the hoisting
+# logic in DruidJoinRule.
+#
+# Table left_dims.a: 'abc', 'xyz'
+# Table right_nulls.k: 'abc', NULL
+
+SELECT l.a, r.tag
+FROM left_dims l
+FULL JOIN (SELECT k, 42 AS tag FROM right_nulls) r ON l.a = r.k
+ORDER BY 1, 2;
++-----+-----+
+| a | tag |
++-----+-----+
+| | 42 |
+| abc | 42 |
+| xyz | |
++-----+-----+
+(3 rows)
+
+!ok
+{
+ "queryType" : "windowOperator",
+ "dataSource" : {
+ "type" : "join",
+ "left" : {
+ "type" : "table",
+ "name" : "left_dims"
+ },
+ "right" : {
+ "type" : "query",
+ "query" : {
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "right_nulls"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "virtualColumns" : [ {
+ "type" : "expression",
+ "name" : "v0",
+ "expression" : "42",
+ "outputType" : "LONG"
+ } ],
+ "resultFormat" : "compactedList",
+ "columns" : [ "k", "v0" ],
+ "columnTypes" : [ "STRING", "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+ }
+ },
+ "rightPrefix" : "j0.",
+ "condition" : "(\"a\" == \"j0.k\")",
+ "joinType" : "FULL"
+ },
+ "intervals" : {
+ "type" : "LegacySegmentSpec",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "outputSignature" : [ {
+ "name" : "a",
+ "type" : "STRING"
+ }, {
+ "name" : "j0.v0",
+ "type" : "LONG"
+ } ],
+ "operatorDefinition" : [ {
+ "type" : "naiveSort",
+ "columns" : [ {
+ "column" : "a",
+ "direction" : "ASC"
+ }, {
+ "column" : "j0.v0",
+ "direction" : "ASC"
+ } ]
+ }, {
+ "type" : "scan",
+ "timeRange" : null,
+ "filter" : null,
+ "offsetLimit" : null,
+ "projectedColumns" : [ "a", "j0.v0" ],
+ "virtualColumns" : null,
+ "ordering" : null
+ } ],
+ "leafOperators" : [ {
+ "type" : "scan",
+ "timeRange" : null,
+ "filter" : null,
+ "offsetLimit" : {
+ "offset" : 0,
+ "limit" : 9223372036854775807
+ },
+ "projectedColumns" : [ "a", "j0.v0" ],
+ "virtualColumns" : null,
+ "ordering" : null
+ } ],
+ "granularity" : {
+ "type" : "all"
+ }
+}
+!nativePlan
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/interval_arithmetic.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/interval_arithmetic.iq
new file mode 100644
index 000000000000..2a1d79a0cead
--- /dev/null
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/interval_arithmetic.iq
@@ -0,0 +1,180 @@
+!set sqlCurrentTimestamp 2000-01-01T00:00:00Z
+!set outputformat mysql
+!use druidtest:///
+
+SELECT __time FROM foo
+WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '2' DAY
+ AND __time < CURRENT_TIMESTAMP + INTERVAL '2' DAY
+ORDER BY __time;
++-------------------------+
+| __time |
++-------------------------+
+| 2000-01-01 00:00:00.000 |
+| 2000-01-02 00:00:00.000 |
++-------------------------+
+(2 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "1999-12-30T00:00:00.000Z/2000-01-03T00:00:00.000Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "order" : "ascending",
+ "columns" : [ "__time" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+SELECT __time FROM foo
+WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '2' WEEK
+ AND __time < CURRENT_TIMESTAMP + INTERVAL '2' WEEK
+ORDER BY __time;
++-------------------------+
+| __time |
++-------------------------+
+| 2000-01-01 00:00:00.000 |
+| 2000-01-02 00:00:00.000 |
+| 2000-01-03 00:00:00.000 |
++-------------------------+
+(3 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "1999-12-18T00:00:00.000Z/2000-01-15T00:00:00.000Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "order" : "ascending",
+ "columns" : [ "__time" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+SELECT __time FROM foo
+WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '2' MONTH
+ AND __time < CURRENT_TIMESTAMP + INTERVAL '2' MONTH
+ORDER BY __time;
++-------------------------+
+| __time |
++-------------------------+
+| 2000-01-01 00:00:00.000 |
+| 2000-01-02 00:00:00.000 |
+| 2000-01-03 00:00:00.000 |
++-------------------------+
+(3 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "1999-11-01T00:00:00.000Z/2000-03-01T00:00:00.000Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "order" : "ascending",
+ "columns" : [ "__time" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+SELECT __time FROM foo
+WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '2' QUARTER
+ AND __time < CURRENT_TIMESTAMP + INTERVAL '2' QUARTER
+ORDER BY __time;
++-------------------------+
+| __time |
++-------------------------+
+| 2000-01-01 00:00:00.000 |
+| 2000-01-02 00:00:00.000 |
+| 2000-01-03 00:00:00.000 |
++-------------------------+
+(3 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "1999-07-01T00:00:00.000Z/2000-07-01T00:00:00.000Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "order" : "ascending",
+ "columns" : [ "__time" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+SELECT __time FROM foo
+WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '2' YEAR
+ AND __time < CURRENT_TIMESTAMP + INTERVAL '2' YEAR
+ORDER BY __time;
++-------------------------+
+| __time |
++-------------------------+
+| 2000-01-01 00:00:00.000 |
+| 2000-01-02 00:00:00.000 |
+| 2000-01-03 00:00:00.000 |
+| 2001-01-01 00:00:00.000 |
+| 2001-01-02 00:00:00.000 |
+| 2001-01-03 00:00:00.000 |
++-------------------------+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "1998-01-01T00:00:00.000Z/2002-01-01T00:00:00.000Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "order" : "ascending",
+ "columns" : [ "__time" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/notinvalues.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/notinvalues.iq
new file mode 100644
index 000000000000..51f0b99e7aba
--- /dev/null
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/notinvalues.iq
@@ -0,0 +1,497 @@
+!use druidtest:///
+!set outputformat mysql
+
+SELECT cnt
+FROM foo
+WHERE cnt NOT IN (SELECT col FROM (VALUES(0), (999)) AS t(col));
++-----+
+| cnt |
++-----+
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
++-----+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "not",
+ "field" : {
+ "type" : "inType",
+ "column" : "cnt",
+ "matchValueType" : "LONG",
+ "sortedValues" : [ 0, 999 ]
+ }
+ },
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# Different types: lhs (m2) is DOUBLE, VALUES contains BIGINT literals.
+SELECT m2
+FROM foo
+WHERE m2 NOT IN (SELECT col FROM (VALUES(0), (999)) AS t(col));
++-----+
+| m2 |
++-----+
+| 1.0 |
+| 2.0 |
+| 3.0 |
+| 4.0 |
+| 5.0 |
+| 6.0 |
++-----+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "not",
+ "field" : {
+ "type" : "inType",
+ "column" : "m2",
+ "matchValueType" : "DOUBLE",
+ "sortedValues" : [ 0.0, 999.0 ]
+ }
+ },
+ "columns" : [ "m2" ],
+ "columnTypes" : [ "DOUBLE" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+SELECT cnt
+FROM foo
+WHERE cnt NOT IN (SELECT CAST(col AS BIGINT) FROM (VALUES(0.1), (999.1)) AS t(col));
++-----+
+| cnt |
++-----+
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
++-----+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "not",
+ "field" : {
+ "type" : "inType",
+ "column" : "cnt",
+ "matchValueType" : "LONG",
+ "sortedValues" : [ 0, 999 ]
+ }
+ },
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+SELECT cnt
+FROM foo
+WHERE cnt NOT IN (SELECT col + 1 FROM (VALUES(1), (999)) AS t(col));
++-----+
+| cnt |
++-----+
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
++-----+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "not",
+ "field" : {
+ "type" : "inType",
+ "column" : "cnt",
+ "matchValueType" : "LONG",
+ "sortedValues" : [ 2, 1000 ]
+ }
+ },
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# Verify that inSubQueryThreshold is respected: lowering it to 0 forces the rule to be skipped.
+# Need !set + !use to reconnect so the property takes effect.
+!set inSubQueryThreshold 0
+!use druidtest:///
+!set outputformat mysql
+SELECT cnt
+FROM foo
+WHERE cnt NOT IN (SELECT col FROM (VALUES(0), (999)) AS t(col));
++-----+
+| cnt |
++-----+
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
++-----+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "join",
+ "left" : {
+ "type" : "join",
+ "left" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "right" : {
+ "type" : "query",
+ "query" : {
+ "queryType" : "timeseries",
+ "dataSource" : {
+ "type" : "inline",
+ "columnNames" : [ "col" ],
+ "columnTypes" : [ "LONG" ],
+ "rows" : [ [ 0 ], [ 999 ] ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "aggregations" : [ {
+ "type" : "count",
+ "name" : "a0"
+ } ]
+ }
+ },
+ "rightPrefix" : "j0.",
+ "condition" : "1",
+ "joinType" : "INNER"
+ },
+ "right" : {
+ "type" : "inline",
+ "columnNames" : [ "col" ],
+ "columnTypes" : [ "LONG" ],
+ "rows" : [ [ 0 ], [ 999 ] ]
+ },
+ "rightPrefix" : "_j0.",
+ "condition" : "(\"cnt\" == \"_j0.col\")",
+ "joinType" : "LEFT"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "virtualColumns" : [ {
+ "type" : "expression",
+ "name" : "v0",
+ "expression" : "(notnull(\"_j0.col\") || null)",
+ "outputType" : "LONG"
+ } ],
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "or",
+ "fields" : [ {
+ "type" : "equals",
+ "column" : "j0.a0",
+ "matchValueType" : "LONG",
+ "matchValue" : 0
+ }, {
+ "type" : "and",
+ "fields" : [ {
+ "type" : "null",
+ "column" : "v0"
+ }, {
+ "type" : "expression",
+ "expression" : "(\"j0.a0\" >= \"j0.a0\")"
+ }, {
+ "type" : "not",
+ "field" : {
+ "type" : "null",
+ "column" : "cnt"
+ }
+ } ]
+ } ]
+ },
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# Verify that planning works properly when NULL appears in the VALUES list.
+SELECT cnt
+FROM foo
+WHERE cnt NOT IN (SELECT col FROM (VALUES (0), (999), (CAST(NULL AS BIGINT))) AS t(col));
++-----+
+| cnt |
++-----+
++-----+
+(0 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "join",
+ "left" : {
+ "type" : "join",
+ "left" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "right" : {
+ "type" : "query",
+ "query" : {
+ "queryType" : "timeseries",
+ "dataSource" : {
+ "type" : "inline",
+ "columnNames" : [ "col" ],
+ "columnTypes" : [ "LONG" ],
+ "rows" : [ [ 0 ], [ 999 ], [ null ] ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "aggregations" : [ {
+ "type" : "count",
+ "name" : "a0"
+ }, {
+ "type" : "filtered",
+ "aggregator" : {
+ "type" : "count",
+ "name" : "a1"
+ },
+ "filter" : {
+ "type" : "not",
+ "field" : {
+ "type" : "null",
+ "column" : "col"
+ }
+ },
+ "name" : "a1"
+ } ]
+ }
+ },
+ "rightPrefix" : "j0.",
+ "condition" : "1",
+ "joinType" : "INNER"
+ },
+ "right" : {
+ "type" : "inline",
+ "columnNames" : [ "col", "i" ],
+ "columnTypes" : [ "LONG", "LONG" ],
+ "rows" : [ [ 0, 1 ], [ 999, 1 ], [ null, 1 ] ]
+ },
+ "rightPrefix" : "_j0.",
+ "condition" : "(\"cnt\" == \"_j0.col\")",
+ "joinType" : "LEFT"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "or",
+ "fields" : [ {
+ "type" : "equals",
+ "column" : "j0.a0",
+ "matchValueType" : "LONG",
+ "matchValue" : 0
+ }, {
+ "type" : "and",
+ "fields" : [ {
+ "type" : "null",
+ "column" : "_j0.i"
+ }, {
+ "type" : "expression",
+ "expression" : "(\"j0.a1\" >= \"j0.a0\")"
+ }, {
+ "type" : "not",
+ "field" : {
+ "type" : "null",
+ "column" : "cnt"
+ }
+ } ]
+ } ]
+ },
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# NULL in IN list with a matching lhs: the match still produces TRUE.
+SELECT cnt
+FROM foo
+WHERE cnt IN (SELECT col FROM (VALUES (1), (999), (CAST(NULL AS BIGINT))) AS t(col));
++-----+
+| cnt |
++-----+
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
+| 1 |
++-----+
+(6 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "join",
+ "left" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "right" : {
+ "type" : "inline",
+ "columnNames" : [ "col" ],
+ "columnTypes" : [ "LONG" ],
+ "rows" : [ [ 1 ], [ 999 ], [ null ] ]
+ },
+ "rightPrefix" : "j0.",
+ "condition" : "(\"cnt\" == \"j0.col\")",
+ "joinType" : "INNER"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# NULL in IN list with a non-matching lhs: UNKNOWN (NULL in list), filter drops.
+SELECT cnt
+FROM foo
+WHERE cnt IN (SELECT col FROM (VALUES (0), (999), (CAST(NULL AS BIGINT))) AS t(col));
++-----+
+| cnt |
++-----+
++-----+
+(0 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "join",
+ "left" : {
+ "type" : "table",
+ "name" : "foo"
+ },
+ "right" : {
+ "type" : "inline",
+ "columnNames" : [ "col" ],
+ "columnTypes" : [ "LONG" ],
+ "rows" : [ [ 0 ], [ 999 ], [ null ] ]
+ },
+ "rightPrefix" : "j0.",
+ "condition" : "(\"cnt\" == \"j0.col\")",
+ "joinType" : "INNER"
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "columns" : [ "cnt" ],
+ "columnTypes" : [ "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_datasource.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_datasource.iq
new file mode 100644
index 000000000000..ffeab90a6488
--- /dev/null
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_datasource.iq
@@ -0,0 +1,422 @@
+!use druidtest:///
+!set outputformat mysql
+
+# Case 1: BIGINT (foo2.m1) vs FLOAT (foo.m1).
+SELECT dim1, dim2, SUM(m1), COUNT(*)
+FROM (SELECT dim1, dim2, m1 FROM foo2 UNION ALL SELECT dim1, dim2, m1 FROM foo)
+WHERE dim2 = 'a' OR dim2 = 'en'
+GROUP BY 1, 2;
++-------+------+--------+--------+
+| dim1 | dim2 | EXPR$2 | EXPR$3 |
++-------+------+--------+--------+
+| | a | 1.0 | 1 |
+| 1 | a | 4.0 | 1 |
+| druid | en | 1.0 | 1 |
++-------+------+--------+--------+
+(3 rows)
+
+!ok
+{
+ "queryType" : "groupBy",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo2"
+ }, {
+ "type" : "table",
+ "name" : "foo"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "filter" : {
+ "type" : "inType",
+ "column" : "dim2",
+ "matchValueType" : "STRING",
+ "sortedValues" : [ "a", "en" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "dimensions" : [ {
+ "type" : "default",
+ "dimension" : "dim1",
+ "outputName" : "d0",
+ "outputType" : "STRING"
+ }, {
+ "type" : "default",
+ "dimension" : "dim2",
+ "outputName" : "d1",
+ "outputType" : "STRING"
+ } ],
+ "aggregations" : [ {
+ "type" : "doubleSum",
+ "name" : "a0",
+ "fieldName" : "m1"
+ }, {
+ "type" : "count",
+ "name" : "a1"
+ } ],
+ "limitSpec" : {
+ "type" : "NoopLimitSpec"
+ }
+}
+!nativePlan
+
+# Case 2: Same numeric types. foo.m1 FLOAT and numfoo.m1 FLOAT.
+SELECT dim1, SUM(m1), COUNT(*)
+FROM (SELECT dim1, m1 FROM foo UNION ALL SELECT dim1, m1 FROM numfoo)
+WHERE dim1 = 'a' OR dim1 = '1'
+GROUP BY 1;
++------+--------+--------+
+| dim1 | EXPR$1 | EXPR$2 |
++------+--------+--------+
+| 1 | 8.0 | 2 |
++------+--------+--------+
+(1 row)
+
+!ok
+{
+ "queryType" : "groupBy",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo"
+ }, {
+ "type" : "table",
+ "name" : "numfoo"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "filter" : {
+ "type" : "inType",
+ "column" : "dim1",
+ "matchValueType" : "STRING",
+ "sortedValues" : [ "1", "a" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "dimensions" : [ {
+ "type" : "default",
+ "dimension" : "dim1",
+ "outputName" : "d0",
+ "outputType" : "STRING"
+ } ],
+ "aggregations" : [ {
+ "type" : "doubleSum",
+ "name" : "a0",
+ "fieldName" : "m1"
+ }, {
+ "type" : "count",
+ "name" : "a1"
+ } ],
+ "limitSpec" : {
+ "type" : "NoopLimitSpec"
+ }
+}
+!nativePlan
+
+# Case 3: STRING (foo.dim3) vs LONG (foo2.dim3). Non-numeric type mismatch.
+SELECT dim2, COUNT(*)
+FROM (SELECT dim2, dim3 FROM foo UNION ALL SELECT dim2, dim3 FROM foo2)
+WHERE dim2 = 'a' OR dim2 = 'en'
+GROUP BY 1;
++------+--------+
+| dim2 | EXPR$1 |
++------+--------+
+| a | 2 |
+| en | 1 |
++------+--------+
+(2 rows)
+
+!ok
+{
+ "queryType" : "groupBy",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo"
+ }, {
+ "type" : "table",
+ "name" : "foo2"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "filter" : {
+ "type" : "inType",
+ "column" : "dim2",
+ "matchValueType" : "STRING",
+ "sortedValues" : [ "a", "en" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "dimensions" : [ {
+ "type" : "default",
+ "dimension" : "dim2",
+ "outputName" : "d0",
+ "outputType" : "STRING"
+ } ],
+ "aggregations" : [ {
+ "type" : "count",
+ "name" : "a0"
+ } ],
+ "limitSpec" : {
+ "type" : "NoopLimitSpec"
+ }
+}
+!nativePlan
+
+# Case 4: Three-way table-level union: foo (FLOAT), foo2 (BIGINT), numfoo (FLOAT).
+SELECT dim1, SUM(m1), COUNT(*)
+FROM (SELECT dim1, m1 FROM foo UNION ALL SELECT dim1, m1 FROM foo2 UNION ALL SELECT dim1, m1 FROM numfoo)
+WHERE dim1 = 'a' OR dim1 = '1'
+GROUP BY 1;
++------+--------+--------+
+| dim1 | EXPR$1 | EXPR$2 |
++------+--------+--------+
+| 1 | 8.0 | 2 |
++------+--------+--------+
+(1 row)
+
+!ok
+{
+ "queryType" : "groupBy",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo"
+ }, {
+ "type" : "table",
+ "name" : "foo2"
+ }, {
+ "type" : "table",
+ "name" : "numfoo"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "filter" : {
+ "type" : "inType",
+ "column" : "dim1",
+ "matchValueType" : "STRING",
+ "sortedValues" : [ "1", "a" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "dimensions" : [ {
+ "type" : "default",
+ "dimension" : "dim1",
+ "outputName" : "d0",
+ "outputType" : "STRING"
+ } ],
+ "aggregations" : [ {
+ "type" : "doubleSum",
+ "name" : "a0",
+ "fieldName" : "m1"
+ }, {
+ "type" : "count",
+ "name" : "a1"
+ } ],
+ "limitSpec" : {
+ "type" : "NoopLimitSpec"
+ }
+}
+!nativePlan
+
+# Case 5: Filter in one arm. Not a bare TableScan, DruidUnionDataSourceRule cannot match.
+SELECT dim1, SUM(m1), COUNT(*)
+FROM (SELECT dim1, m1 FROM foo2 WHERE dim1 = '1' UNION ALL SELECT dim1, m1 FROM foo)
+GROUP BY 1;
+SQL requires union between inputs that are not simple table scans and involve a filter or aliasing
+!error
+
+# Case 6: Column aliasing. Names differ between arms (mapping required).
+SELECT c, COUNT(*)
+FROM (SELECT dim1 AS c, m1 FROM foo UNION ALL SELECT dim2 AS c, m1 FROM foo2)
+WHERE c = 'a' OR c = '1'
+GROUP BY 1;
+SQL requires union between two tables and column names queried for each table are different
+!error
+
+# Case 7: Outer WHERE only, no GROUP BY.
+SELECT dim1, m1
+FROM (SELECT dim1, m1 FROM foo2 UNION ALL SELECT dim1, m1 FROM foo)
+WHERE dim1 = 'a' OR dim1 = '1';
++------+-----+
+| dim1 | m1 |
++------+-----+
+| 1 | 4.0 |
++------+-----+
+(1 row)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo2"
+ }, {
+ "type" : "table",
+ "name" : "foo"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "resultFormat" : "compactedList",
+ "filter" : {
+ "type" : "inType",
+ "column" : "dim1",
+ "matchValueType" : "STRING",
+ "sortedValues" : [ "1", "a" ]
+ },
+ "columns" : [ "dim1", "m1" ],
+ "columnTypes" : [ "STRING", "LONG" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# Case 8: Outer projection with concat on dim1.
+SELECT dim1 || '!' AS tag, m1
+FROM (SELECT dim1, m1 FROM foo2 UNION ALL SELECT dim1, m1 FROM foo);
++---------+-----+
+| tag | m1 |
++---------+-----+
+| ! | 1.0 |
+| 1! | 4.0 |
+| 10.1! | 2.0 |
+| 2! | 3.0 |
+| abc! | 6.0 |
+| def! | 5.0 |
+| druid! | 1.0 |
+| друид! | 1.0 |
+| דרואיד! | 1.0 |
++---------+-----+
+(9 rows)
+
+!ok
+{
+ "queryType" : "scan",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo2"
+ }, {
+ "type" : "table",
+ "name" : "foo"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "virtualColumns" : [ {
+ "type" : "expression",
+ "name" : "v0",
+ "expression" : "concat(\"dim1\",'!')",
+ "outputType" : "STRING"
+ }, {
+ "type" : "expression",
+ "name" : "v1",
+ "expression" : "\"m1\"",
+ "outputType" : "FLOAT"
+ } ],
+ "resultFormat" : "compactedList",
+ "columns" : [ "v0", "v1" ],
+ "columnTypes" : [ "STRING", "FLOAT" ],
+ "granularity" : {
+ "type" : "all"
+ },
+ "legacy" : false
+}
+!nativePlan
+
+# Case 9: Multiple mismatching columns at once (m1 + dim3).
+SELECT dim1, SUM(m1), MAX(dim3), COUNT(*)
+FROM (SELECT dim1, m1, dim3 FROM foo UNION ALL SELECT dim1, m1, dim3 FROM foo2)
+GROUP BY 1;
+SQL requires union between inputs that are not simple table scans and involve a filter or aliasing
+!error
+
+# Case 10: SELECT * union. full row, m1 FLOAT vs BIGINT, dim3 STRING vs LONG.
+SELECT dim2, SUM(m1), COUNT(*)
+FROM (SELECT * FROM foo UNION ALL SELECT * FROM foo2)
+WHERE dim2 = 'a' OR dim2 = 'en'
+GROUP BY 1;
++------+--------+--------+
+| dim2 | EXPR$1 | EXPR$2 |
++------+--------+--------+
+| a | 5.0 | 2 |
+| en | 1.0 | 1 |
++------+--------+--------+
+(2 rows)
+
+!ok
+{
+ "queryType" : "groupBy",
+ "dataSource" : {
+ "type" : "union",
+ "dataSources" : [ {
+ "type" : "table",
+ "name" : "foo"
+ }, {
+ "type" : "table",
+ "name" : "foo2"
+ } ]
+ },
+ "intervals" : {
+ "type" : "intervals",
+ "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+ },
+ "filter" : {
+ "type" : "inType",
+ "column" : "dim2",
+ "matchValueType" : "STRING",
+ "sortedValues" : [ "a", "en" ]
+ },
+ "granularity" : {
+ "type" : "all"
+ },
+ "dimensions" : [ {
+ "type" : "default",
+ "dimension" : "dim2",
+ "outputName" : "d0",
+ "outputType" : "STRING"
+ } ],
+ "aggregations" : [ {
+ "type" : "doubleSum",
+ "name" : "a0",
+ "fieldName" : "m1"
+ }, {
+ "type" : "count",
+ "name" : "a1"
+ } ],
+ "limitSpec" : {
+ "type" : "NoopLimitSpec"
+ }
+}
+!nativePlan
diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq
index dc714560fed6..caad1d27c04b 100644
--- a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq
+++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/union_removed_branch_union_nulls.iq
@@ -28,8 +28,7 @@ LogicalUnion(all=[true])
LogicalValues(tuples=[[{ null, null }]])
!convertedPlan
-DruidProject(EXPR$0=[CAST($0):BIGINT], channel=[CAST($1):VARCHAR], druid=[logical])
- DruidValues(tuples=[[{ null, null }]])
+DruidValues(tuples=[[{ null, null }]])
!druidPlan
{
@@ -37,21 +36,15 @@ DruidProject(EXPR$0=[CAST($0):BIGINT], channel=[CAST($1):VARCHAR], druid=[logica
"dataSource" : {
"type" : "inline",
"columnNames" : [ "EXPR$0", "EXPR$1" ],
- "columnTypes" : [ "STRING", "STRING" ],
+ "columnTypes" : [ "LONG", "STRING" ],
"rows" : [ [ null, null ] ]
},
"intervals" : {
"type" : "intervals",
"intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
},
- "virtualColumns" : [ {
- "type" : "expression",
- "name" : "v0",
- "expression" : "CAST(\"EXPR$0\", 'LONG')",
- "outputType" : "LONG"
- } ],
"resultFormat" : "compactedList",
- "columns" : [ "v0", "EXPR$1" ],
+ "columns" : [ "EXPR$0", "EXPR$1" ],
"columnTypes" : [ "LONG", "STRING" ],
"granularity" : {
"type" : "all"
diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumns.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumns.iq
index df2fe0dd3f45..7187ba4a9f75 100644
--- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumns.iq
+++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumns.iq
@@ -23,22 +23,22 @@
LogicalProject(arrayString=[CAST(ARRAY('a':VARCHAR, 'b':VARCHAR)):VARCHAR ARRAY], uln=[CAST(1:BIGINT):BIGINT], udn=[$1], usn=[$2])
LogicalUnnest(unnestExpr=[$0], filter=[=($0, 'a')])
LogicalProject(arrayStringNulls=[$0], udn=[$2])
- LogicalUnnest(unnestExpr=[$1], filter=[=($0, 2.2)])
+ LogicalUnnest(unnestExpr=[$1], filter=[=($0, 2.2E0)])
LogicalProject(arrayStringNulls=[$0], arrayDoubleNulls=[$2])
LogicalUnnest(unnestExpr=[$1], filter=[=($0, 1)])
LogicalProject(arrayStringNulls=[$2], arrayLongNulls=[$4], arrayDoubleNulls=[$6])
- LogicalFilter(condition=[=($1, CAST(ARRAY('a', 'b')):VARCHAR ARRAY NOT NULL)])
+ LogicalFilter(condition=[=($1, ARRAY('a':VARCHAR, 'b':VARCHAR))])
LogicalTableScan(table=[[druid, arrays]])
!logicalPlan
DruidProject(arrayString=[CAST(ARRAY('a':VARCHAR, 'b':VARCHAR)):VARCHAR ARRAY], uln=[CAST(1:BIGINT):BIGINT], udn=[$1], usn=[$2], druid=[logical])
DruidUnnest(unnestExpr=[$0], filter=[=($0, 'a')])
DruidProject(arrayStringNulls=[$0], udn=[$2], druid=[logical])
- DruidUnnest(unnestExpr=[$1], filter=[=($0, 2.2)])
+ DruidUnnest(unnestExpr=[$1], filter=[=($0, 2.2E0)])
DruidProject(arrayStringNulls=[$0], arrayDoubleNulls=[$2], druid=[logical])
DruidUnnest(unnestExpr=[$1], filter=[=($0, 1)])
DruidProject(arrayStringNulls=[$2], arrayLongNulls=[$4], arrayDoubleNulls=[$6], druid=[logical])
- DruidFilter(condition=[=($1, CAST(ARRAY('a', 'b')):VARCHAR ARRAY NOT NULL)])
+ DruidFilter(condition=[=($1, ARRAY('a':VARCHAR, 'b':VARCHAR))])
DruidTableScan(table=[[druid, arrays]], druid=[logical])
!druidPlan
diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumnsOrFilters.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumnsOrFilters.iq
index 03d6e16878f9..17ebe651458f 100644
--- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumnsOrFilters.iq
+++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumnsOrFilters.iq
@@ -28,24 +28,24 @@
LogicalProject(arrayString=[CAST(ARRAY('a':VARCHAR, 'b':VARCHAR)):VARCHAR ARRAY], uln=[$1], udn=[$2], usn=[$3])
LogicalUnnest(unnestExpr=[$0], filter=[=($0, 'a')])
LogicalProject(arrayStringNulls=[$0], uln=[$2], udn=[$3])
- LogicalFilter(condition=[OR(=($2, 1), =($3, 2.2))])
+ LogicalFilter(condition=[OR(=($2, 1), =($3, 2.2E0))])
LogicalUnnest(unnestExpr=[$1])
LogicalProject(arrayStringNulls=[$0], arrayDoubleNulls=[$2], uln=[$3])
LogicalUnnest(unnestExpr=[$1])
LogicalProject(arrayStringNulls=[$2], arrayLongNulls=[$4], arrayDoubleNulls=[$6])
- LogicalFilter(condition=[=($1, CAST(ARRAY('a', 'b')):VARCHAR ARRAY NOT NULL)])
+ LogicalFilter(condition=[=($1, ARRAY('a':VARCHAR, 'b':VARCHAR))])
LogicalTableScan(table=[[druid, arrays]])
!logicalPlan
DruidProject(arrayString=[CAST(ARRAY('a':VARCHAR, 'b':VARCHAR)):VARCHAR ARRAY], uln=[$1], udn=[$2], usn=[$3], druid=[logical])
DruidUnnest(unnestExpr=[$0], filter=[=($0, 'a')])
DruidProject(arrayStringNulls=[$0], uln=[$2], udn=[$3], druid=[logical])
- DruidFilter(condition=[OR(=($2, 1), =($3, 2.2))])
+ DruidFilter(condition=[OR(=($2, 1), =($3, 2.2E0))])
DruidUnnest(unnestExpr=[$1])
DruidProject(arrayStringNulls=[$0], arrayDoubleNulls=[$2], uln=[$3], druid=[logical])
DruidUnnest(unnestExpr=[$1])
DruidProject(arrayStringNulls=[$2], arrayLongNulls=[$4], arrayDoubleNulls=[$6], druid=[logical])
- DruidFilter(condition=[=($1, CAST(ARRAY('a', 'b')):VARCHAR ARRAY NOT NULL)])
+ DruidFilter(condition=[=($1, ARRAY('a':VARCHAR, 'b':VARCHAR))])
DruidTableScan(table=[[druid, arrays]], druid=[logical])
!druidPlan
diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim.iq
index d3e26335e788..de2c214f90d8 100644
--- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim.iq
+++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim.iq
@@ -27,7 +27,7 @@ ORDER BY dim2, gran desc;
LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC])
LogicalAggregate(group=[{0, 1}], EXPR$2=[SUM($2)])
LogicalProject(dim2=[$1], gran=[TIME_FLOOR($0, 'P1M')], s=[$2])
- LogicalFilter(condition=[>($2, 1)])
+ LogicalFilter(condition=[>($2, 1.0E0)])
LogicalAggregate(group=[{0, 1}], s=[SUM($2)])
LogicalProject(gran=[TIME_FLOOR($0, 'P1D')], dim2=[$2], m1=[$5])
LogicalTableScan(table=[[druid, foo]])
@@ -36,7 +36,7 @@ LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC])
DruidSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC], druid=[logical])
DruidAggregate(group=[{0, 1}], EXPR$2=[SUM($2)], druid=[logical])
DruidProject(dim2=[$1], gran=[TIME_FLOOR($0, 'P1M')], s=[$2], druid=[logical])
- DruidFilter(condition=[>($2, 1)])
+ DruidFilter(condition=[>($2, 1.0E0)])
DruidAggregate(group=[{0, 1}], s=[SUM($2)], druid=[logical])
DruidProject(gran=[TIME_FLOOR($0, 'P1D')], dim2=[$2], m1=[$5], druid=[logical])
DruidTableScan(table=[[druid, foo]], druid=[logical])
@@ -86,8 +86,8 @@ DruidSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC], druid=[logical])
"filter" : {
"type" : "range",
"column" : "a0",
- "matchValueType" : "LONG",
- "lower" : 1,
+ "matchValueType" : "DOUBLE",
+ "lower" : 1.0,
"lowerOpen" : true
},
"finalize" : true
diff --git a/sql/src/test/resources/calcite/tests/window/virtualColumns.sqlTest b/sql/src/test/resources/calcite/tests/window/virtualColumns.sqlTest
index 9b5aa6e1365e..b399647d018f 100644
--- a/sql/src/test/resources/calcite/tests/window/virtualColumns.sqlTest
+++ b/sql/src/test/resources/calcite/tests/window/virtualColumns.sqlTest
@@ -22,7 +22,7 @@ expectedOperators:
frame: { type: "rows" }
aggregations:
- { type: "doubleMin", name: "w0", fieldName: "_v0" }
- - { type: "longMin", name: "w1", fieldName: "v1" }
+ - { type: "longMin", name: "w1", fieldName: "v2" }
expectedResults:
- ["_",42.0,11]
diff --git a/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartitionInitialSort.sqlTest b/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartitionInitialSort.sqlTest
index 4939057621e5..991a4d2c421d 100644
--- a/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartitionInitialSort.sqlTest
+++ b/sql/src/test/resources/calcite/tests/window/wikipediaSimplePartitionInitialSort.sqlTest
@@ -24,19 +24,15 @@ expectedOperators:
processor:
type: "composing"
processors:
- - { "type": "first", "inputColumn": "a0", "outputColumn": "w1" }
- - { "type": "last", "inputColumn": "a0", "outputColumn": "w2" }
- - { "type": "percentile", "outputColumn": "w3", "numBuckets": 3 }
+ - { "type": "offset", "inputColumn": "d0", "outputColumn": "w1", "offset": -2 }
+ - { "type": "offset", "inputColumn": "d0", "outputColumn": "w2", "offset": 1 }
+ - { "type": "first", "inputColumn": "a0", "outputColumn": "w3" }
+ - { "type": "last", "inputColumn": "a0", "outputColumn": "w4" }
+ - { "type": "percentile", "outputColumn": "w5", "numBuckets": 3 }
- type: "framedAgg"
frame: { type: "rows" }
aggregations:
- { "type": "longSum", "name": "w0", "fieldName": "a0" }
- - type: "window"
- processor:
- type: "composing"
- processors:
- - { "type": "offset", "inputColumn": "d0", "outputColumn": "w4", "offset": -2 }
- - { "type": "offset", "inputColumn": "d0", "outputColumn": "w5", "offset": 1 }
expectedResults:
- [1442016000000,null,29873,8414700,null,1442019600000,29873,276159,1]
diff --git a/sql/src/test/resources/drill/window/queries/druid_queries/empty_and_non_empty_over/wikipedia_query_3.e b/sql/src/test/resources/drill/window/queries/druid_queries/empty_and_non_empty_over/wikipedia_query_3.e
index 8cd625edad22..e33f40c8949e 100644
--- a/sql/src/test/resources/drill/window/queries/druid_queries/empty_and_non_empty_over/wikipedia_query_3.e
+++ b/sql/src/test/resources/drill/window/queries/druid_queries/empty_and_non_empty_over/wikipedia_query_3.e
@@ -1,16 +1,16 @@
-Republic of Korea Seoul 0 0 16 4 2 null 1 2 0 0 1 2 13 null 12 3 1 2 13
-Republic of Korea Seoul 2 4 16 5 3 0 2 3 1 1 2 3 13 Austria 11 3 2 2 12
-Republic of Korea Seoul 4 4 16 6 4 2 3 4 2 2 2 3 12 Republic of Korea 10 3 3 2 11
-Republic of Korea Seoul 5 5 16 7 5 4 4 5 3 3 2 3 11 Republic of Korea 9 3 4 2 10
-Republic of Korea Seoul 6 6 16 8 6 5 5 6 4 3 2 3 10 Republic of Korea 8 3 5 2 9
-Republic of Korea Seoul 12 12 16 9 7 6 6 7 5 3 2 3 9 Republic of Korea 7 3 6 2 8
-Republic of Korea Seoul 15 15 16 10 8 12 7 8 6 3 2 3 8 Republic of Korea 6 3 7 2 7
-Republic of Korea Seoul 19 19 16 11 9 15 8 9 7 3 2 3 7 Republic of Korea 5 3 8 2 6
-Republic of Korea Seoul 22 22 16 12 10 19 9 10 8 3 2 3 6 Republic of Korea 4 3 9 2 5
-Republic of Korea Seoul 24 24 16 13 11 22 10 11 9 3 2 3 5 Republic of Korea 3 3 10 2 4
-Republic of Korea Seoul 243 243 16 14 14 24 11 12 10 3 2 3 4 Austria 2 2 11 2 3
-Republic of Korea Seoul 663 663 16 15 15 243 12 13 11 3 2 3 3 Republic of Korea 1 1 12 2 2
-Republic of Korea Seoul 1036 1036 16 16 16 663 13 13 12 3 2 2 2 Republic of Korea 0 0 13 1 1
-Austria Vienna 0 0 16 1 1 1036 1 2 0 0 1 2 3 Republic of Korea 2 2 1 2 3
-Austria Vienna 72 72 16 2 12 0 2 3 1 1 2 3 3 Republic of Korea 1 1 2 2 2
-Austria Vienna 93 93 16 3 13 72 3 3 2 2 2 2 2 Austria 0 0 3 1 1
+Republic of Korea Seoul 0 0 16 1 1 null 1 2 0 0 1 2 13 null 12 3 1 2 13
+Republic of Korea Seoul 2 4 16 2 3 0 2 3 1 1 2 3 13 Austria 11 3 2 2 12
+Republic of Korea Seoul 4 4 16 3 4 2 3 4 2 2 2 3 12 Republic of Korea 10 3 3 2 11
+Republic of Korea Seoul 5 5 16 4 5 4 4 5 3 3 2 3 11 Republic of Korea 9 3 4 2 10
+Republic of Korea Seoul 6 6 16 5 6 5 5 6 4 3 2 3 10 Republic of Korea 8 3 5 2 9
+Republic of Korea Seoul 12 12 16 6 7 6 6 7 5 3 2 3 9 Republic of Korea 7 3 6 2 8
+Republic of Korea Seoul 15 15 16 7 8 12 7 8 6 3 2 3 8 Republic of Korea 6 3 7 2 7
+Republic of Korea Seoul 19 19 16 8 9 15 8 9 7 3 2 3 7 Republic of Korea 5 3 8 2 6
+Republic of Korea Seoul 22 22 16 9 10 19 9 10 8 3 2 3 6 Republic of Korea 4 3 9 2 5
+Republic of Korea Seoul 24 24 16 10 11 22 10 11 9 3 2 3 5 Republic of Korea 3 3 10 2 4
+Republic of Korea Seoul 243 243 16 11 14 24 11 12 10 3 2 3 4 Austria 2 2 11 2 3
+Republic of Korea Seoul 663 663 16 12 15 243 12 13 11 3 2 3 3 Republic of Korea 1 1 12 2 2
+Republic of Korea Seoul 1036 1036 16 13 16 663 13 13 12 3 2 2 2 Republic of Korea 0 0 13 1 1
+Austria Vienna 0 0 16 14 2 1036 1 2 0 0 1 2 3 Republic of Korea 2 2 1 2 3
+Austria Vienna 72 72 16 15 12 0 2 3 1 1 2 3 3 Republic of Korea 1 1 2 2 2
+Austria Vienna 93 93 16 16 13 72 3 3 2 2 2 2 2 Austria 0 0 3 1 1