You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2024/10/21 18:03:18 UTC
(impala) branch master updated (64e43ad46 -> 3cf05fe21)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 64e43ad46 IMPALA-13410: Document reading Puffin files
new 7b109ca45 IMPALA-13457: Calcite planner: fix datetime/interval issues for tpcds queries
new 3cf05fe21 IMPALA-13461: Added rules to make tpcds queries work.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
java/calcite-planner/pom.xml | 17 ++
.../calcite/coercenodes/CoerceOperandShuttle.java | 31 ++-
.../impala/calcite/functions/IntervalExpr.java | 119 ++++++++++++
.../impala/calcite/functions/RexCallConverter.java | 62 ++++++
.../calcite/operators/ImpalaOperatorTable.java | 35 +++-
.../impala/calcite/rules/ExtractLiteralAgg.java | 86 +++++++++
.../calcite/rules/ImpalaMinusToDistinctRule.java | 214 +++++++++++++++++++++
.../impala/calcite/service/CalciteOptimizer.java | 44 ++++-
.../calcite/service/CalciteRelNodeConverter.java | 47 ++++-
9 files changed, 638 insertions(+), 17 deletions(-)
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/IntervalExpr.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ExtractLiteralAgg.java
create mode 100644 java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaMinusToDistinctRule.java
(impala) 01/02: IMPALA-13457: Calcite planner: fix datetime/interval issues for tpcds queries
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7b109ca45fc6a177dfe4796f0808400c32149715
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Wed Oct 16 13:49:00 2024 -0700
IMPALA-13457: Calcite planner: fix datetime/interval issues for tpcds queries
Added support for datetime interval operators. A dummy IntervalExpr
was added to support logical to physical representation. The
IntervalExpr is removed once the actual "+" operation for datetime
is created, but the translation framework is simplified by working
on the Calcite interval type in a separate temporary Expr.
Some logic was inserted into CoerceOperandShuttle. This object is
used to add casts when there is limitation in Calcite for number types.
When we have an interval, we always want to represent it as a special
type so we do not want to introduce a cast.
Also, in ImpalaOperatorTable, the ability to use Impala functions over
Calcite functions are needed because Calcite translates the year function
into "extract" which causes some issues. Just using the Impala signature
works for us here.
Change-Id: I2b4afc3ab1d17ba1f168904a6ded052e1d62b3fe
Reviewed-on: http://gerrit.cloudera.org:8080/21946
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../calcite/coercenodes/CoerceOperandShuttle.java | 31 +++++-
.../impala/calcite/functions/IntervalExpr.java | 119 +++++++++++++++++++++
.../impala/calcite/functions/RexCallConverter.java | 62 +++++++++++
.../calcite/operators/ImpalaOperatorTable.java | 35 ++++--
4 files changed, 238 insertions(+), 9 deletions(-)
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java
index 2de3550de..40e583ba8 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java
@@ -30,7 +30,10 @@ import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlDatetimePlusOperator;
+import org.apache.calcite.sql.fun.SqlDatetimeSubtractionOperator;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -70,7 +73,7 @@ public class CoerceOperandShuttle extends RexShuttle {
private final RexBuilder rexBuilder;
private final List<RelNode> inputs;
- public static Set<SqlKind> NO_CASTING_NEEDED =
+ public static Set<SqlKind> NO_CAST_OPERATORS =
ImmutableSet.<SqlKind> builder()
// Cast doesn't need any operand casting because it is already a cast.
.add(SqlKind.CAST)
@@ -102,7 +105,7 @@ public class CoerceOperandShuttle extends RexShuttle {
RexCall castedOperandsCall = (RexCall) super.visitCall(call);
// Certain operators will never need casting for their operands.
- if (NO_CASTING_NEEDED.contains(castedOperandsCall.getOperator().getKind())) {
+ if (!isCastingNeeded(castedOperandsCall)) {
return castedOperandsCall;
}
@@ -217,6 +220,30 @@ public class CoerceOperandShuttle extends RexShuttle {
return retType;
}
+ private boolean isCastingNeeded(RexCall rexCall) {
+ if (NO_CAST_OPERATORS.contains(rexCall.getOperator().getKind())) {
+ return false;
+ }
+
+ // Time stamp operators don't require casting. When they are converted
+ // into Expr objects, they undergo special processing, so we leave them
+ // as/is.
+ if (isTimestampArithExpr(rexCall)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ private static boolean isTimestampArithExpr(RexCall rexCall) {
+ return rexCall.getOperator() instanceof SqlDatetimePlusOperator
+ || rexCall.getOperator() instanceof SqlDatetimeSubtractionOperator
+ || SqlTypeName.INTERVAL_TYPES.contains(rexCall.getType().getSqlTypeName())
+ || ((rexCall.getOperator().equals("+") || rexCall.getOperator().equals("-")) &&
+ (SqlTypeUtil.isDatetime(rexCall.getOperands().get(0).getType()) ||
+ SqlTypeUtil.isDatetime(rexCall.getOperands().get(1).getType())));
+ }
+
/**
* Handle getting the type of index. If there is only one input, then we
* just use the index value to get the type. If there are two inputs,
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/IntervalExpr.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/IntervalExpr.java
new file mode 100644
index 000000000..e2e8ebe5b
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/IntervalExpr.java
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.functions;
+
+import com.google.common.base.Preconditions;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.Expr;
+import org.apache.impala.analysis.ToSqlOptions;
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.thrift.TExprNode;
+
+/**
+ * IntervalExpr is a temporary expression class. It is needed because of
+ * the way Calcite handles Interval expressions and how the Calcite RexNode to
+ * Impala translation works.
+ * A calcite timestamp add expression might look like this:
+ * TIMESTAMP +( TIMESTAMP, INTERVAL YEAR *( INT, INTERVAL_YEAR)
+ * The main "plus" operation returns a timestamp.
+ * The two parameters are a timestamp and the interval to add.
+ * The inner "multiply" is an interval expression. The first parameter
+ * of the interval expression in this case is the number of years to
+ * multiply. The second parameter is a default "1" of INTERVAL_YEAR.
+ *
+ * To be concrete about this, if we were adding 12 years to our timestamp
+ * expression, the inner interval expression looks like this:
+ * mult(12, 1: INTERVAL_YEAR)
+ *
+ * This inner interval expression doesn't have any Impala equivalent, so that's
+ * why this class is necessary. This object will only exist as a placeholder.
+ * When the + expression is created as a ArithTimestampExpr, the information of
+ * the IntervalExpr will be used and then the object will be discarded.
+ */
+public class IntervalExpr extends Expr {
+
+ private final RexCall intervalExpr_;
+
+ private final Expr literal_;
+
+ private final String timeUnit_;
+
+ public IntervalExpr(RexCall intervalExpr, Expr literalExpr) {
+ Preconditions.checkState(intervalExpr.getOperator().getKind().equals(SqlKind.TIMES));
+ Preconditions.checkState(SqlTypeName.INTERVAL_TYPES.contains(
+ intervalExpr.getOperands().get(1).getType().getSqlTypeName()));
+ intervalExpr_ = intervalExpr;
+ literal_ = literalExpr;
+ timeUnit_ = calcTimeIntervalUnit(intervalExpr);
+
+ }
+
+ public Expr getLiteral() {
+ return literal_;
+ }
+
+ public String getTimeUnit() {
+ return timeUnit_;
+ }
+
+ private String calcTimeIntervalUnit(RexCall call) {
+ // hack. Not sure the best way to get week out. Also need to figure out
+ // fracitional seconds
+ if (call.getOperands().get(1).getType().toString().contains("WEEK")) {
+ return "WEEK";
+ }
+ if (call.getOperands().get(1).getType().toString().contains("MILLISECOND")) {
+ return "MILLISECOND";
+ }
+ if (call.getOperands().get(1).getType().toString().contains("MICROSECOND")) {
+ return "MICROSECOND";
+ }
+ if (call.getOperands().get(1).getType().toString().contains("NANOSECOND")) {
+ return "NANOSECOND";
+ }
+ return call.getOperands().get(1).getType().getSqlTypeName().getStartUnit().toString();
+ }
+
+ @Override
+ protected void analyzeImpl(Analyzer analyzer) throws AnalysisException {
+ throw new RuntimeException("not implemented");
+ }
+
+ @Override
+ protected float computeEvalCost() {
+ throw new RuntimeException("not implemented");
+ }
+
+ @Override
+ protected String toSqlImpl(ToSqlOptions options) {
+ throw new RuntimeException("not implemented");
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ throw new RuntimeException("not implemented");
+ }
+
+ @Override
+ public Expr clone() {
+ throw new RuntimeException("not implemented");
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
index 7bbf01a02..5ef0859d6 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
@@ -25,11 +25,15 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.ArithmeticExpr;
import org.apache.impala.analysis.BinaryPredicate;
import org.apache.impala.analysis.CaseWhenClause;
import org.apache.impala.analysis.CompoundPredicate;
import org.apache.impala.analysis.Expr;
+import org.apache.impala.analysis.TimestampArithmeticExpr;
import org.apache.impala.calcite.type.ImpalaTypeConverter;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.Type;
@@ -77,6 +81,12 @@ public class RexCallConverter {
String funcName = rexCall.getOperator().getName().toLowerCase();
+ // Date addition expressions have special handling.
+ Expr dateExpr = handleDateExprs(funcName, rexCall, params, rexBuilder);
+ if (dateExpr != null) {
+ return dateExpr;
+ }
+
Function fn = getFunction(rexCall);
if (fn == null) {
@@ -104,6 +114,19 @@ public class RexCallConverter {
}
}
+ public static Expr handleDateExprs(String funcName, RexCall rexCall,
+ List<Expr> params, RexBuilder rexBuilder) {
+
+ if (SqlTypeName.INTERVAL_TYPES.contains(rexCall.getType().getSqlTypeName())) {
+ return new IntervalExpr(rexCall, params.get(0));
+ }
+
+ if (isTimestampArithExpr(params)) {
+ return createTimestampExpr(rexCall, params);
+ }
+ return null;
+ }
+
private static Function getFunction(RexCall call) {
List<RelDataType> argTypes = Lists.transform(call.getOperands(), RexNode::getType);
String name = call.getOperator().getName();
@@ -167,4 +190,43 @@ public class RexCallConverter {
Preconditions.checkNotNull(op, "Unknown Calcite op: " + sqlKind);
return new AnalyzedBinaryCompExpr(fn, op, params.get(0), params.get(1));
}
+
+ private static Expr createTimestampExpr(RexCall rexCall, List<Expr> params) {
+ // one of the timestamp expressions should be a date time. The other should
+ // be some number to add or subtract. timestampParamIndex contains the
+ // parameter that is the timestamp.
+ int timestampParamIndex =
+ SqlTypeUtil.isDatetime(rexCall.getOperands().get(0).getType())
+ ? 0
+ : 1;
+ // and intervalIndex contains the non-timestampParamIndex parameter
+ int intervalIndex = timestampParamIndex == 0 ? 1 : 0;
+
+ IntervalExpr intervalExpr = (IntervalExpr) params.get(intervalIndex);
+ ArithmeticExpr.Operator op = getImpalaOp(rexCall);
+ return new TimestampArithmeticExpr(op, params.get(timestampParamIndex),
+ intervalExpr.getLiteral(), intervalExpr.getTimeUnit(), intervalIndex == 0
+ );
+ }
+
+ private static ArithmeticExpr.Operator getImpalaOp(RexCall rexCall) {
+ if (rexCall.getOperator().getName().equals("DATE_ADD") ||
+ rexCall.getOperator().getKind().equals(SqlKind.PLUS)) {
+ return ArithmeticExpr.Operator.ADD;
+ }
+ if (rexCall.getOperator().getName().equals("DATE_SUB") ||
+ rexCall.getOperator().getKind().equals(SqlKind.MINUS)) {
+ return ArithmeticExpr.Operator.SUBTRACT;
+ }
+ throw new RuntimeException("Unknown Operator found in arith expr: " +
+ rexCall.getOperator().getName());
+ }
+
+ public static boolean isTimestampArithExpr(List<Expr> params) {
+ if (params.size() != 2) {
+ return false;
+ }
+ return params.get(0) instanceof IntervalExpr ||
+ params.get(1) instanceof IntervalExpr;
+ }
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
index 7aee0ab37..323198440 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
@@ -18,6 +18,7 @@
package org.apache.impala.calcite.operators;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.validate.SqlNameMatcher;
@@ -29,6 +30,7 @@ import org.apache.impala.catalog.BuiltinsDb;
import org.apache.impala.catalog.Db;
import java.util.List;
+import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,12 +49,28 @@ import org.slf4j.LoggerFactory;
* the function name is known by Impala. If so, an ImpalaOperator class is generated
* on the fly.
*
+ * There are some special functions where we would prefer to use the Impala operator
+ * over the Calcite operator due to some incompatibility. These are specified in
+ * USE_IMPALA_OPERATOR.
+ *
* TODO: IMPALA-13095: Handle UDFs
*/
public class ImpalaOperatorTable extends ReflectiveSqlOperatorTable {
protected static final Logger LOG =
LoggerFactory.getLogger(ImpalaOperatorTable.class.getName());
+ public static Set<String> USE_IMPALA_OPERATOR =
+ ImmutableSet.<String> builder()
+ .add("year")
+ .add("month")
+ .add("week")
+ .add("day")
+ .add("hour")
+ .add("minute")
+ .add("second")
+ .add("millisecond")
+ .build();
+
private static ImpalaOperatorTable INSTANCE;
/**
@@ -70,12 +88,15 @@ public class ImpalaOperatorTable extends ReflectiveSqlOperatorTable {
return;
}
- // Check Calcite operator table for existence.
- SqlStdOperatorTable.instance().lookupOperatorOverloads(opName, category, syntax,
- operatorList, nameMatcher);
- Preconditions.checkState(operatorList.size() <= 1);
- if (operatorList.size() == 1) {
- return;
+ String lowercaseOpName = opName.getSimple().toLowerCase();
+ if (!USE_IMPALA_OPERATOR.contains(lowercaseOpName)) {
+ // Check Calcite operator table for existence.
+ SqlStdOperatorTable.instance().lookupOperatorOverloads(opName, category, syntax,
+ operatorList, nameMatcher);
+ Preconditions.checkState(operatorList.size() <= 1);
+ if (operatorList.size() == 1) {
+ return;
+ }
}
// There shouldn't be more than one opName with our usage, so throw an exception
@@ -86,7 +107,7 @@ public class ImpalaOperatorTable extends ReflectiveSqlOperatorTable {
}
// Check Impala Builtins for existence: TODO: IMPALA-13095: handle UDFs
- if (!BuiltinsDb.getInstance().containsFunction(opName.getSimple())) {
+ if (!BuiltinsDb.getInstance().containsFunction(lowercaseOpName)) {
return;
}
(impala) 02/02: IMPALA-13461: Added rules to make tpcds queries work.
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3cf05fe21ae3a79ec682a41c1cb4341c980a4aa4
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Wed Oct 16 10:46:03 2024 -0700
IMPALA-13461: Added rules to make tpcds queries work.
Among the rules that were added:
The "Minus" RelNode is not handled directly by the physical node
translator and is changed into other nodes that are handled. This
was added in the ImpalaMinusToDistinctRule
The ExtractLiteralAgg rule compensates for the fact that a literal
value cannot be used directly in an agg.
The CalciteRelNodeConverter handles breaking down a SubQuery RelNode
into simpler RelNodes that can be optimized.
The pom.xml file was also changed. There is a java bug in java 8
that causes incremental compiles to fail. So we do a full compile
for the Calcite planner now.
Change-Id: I03a38aaa5c413b9b4d2f4c179de07935b672a031
Reviewed-on: http://gerrit.cloudera.org:8080/21941
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
---
java/calcite-planner/pom.xml | 17 ++
.../impala/calcite/rules/ExtractLiteralAgg.java | 86 +++++++++
.../calcite/rules/ImpalaMinusToDistinctRule.java | 214 +++++++++++++++++++++
.../impala/calcite/service/CalciteOptimizer.java | 44 ++++-
.../calcite/service/CalciteRelNodeConverter.java | 47 ++++-
5 files changed, 400 insertions(+), 8 deletions(-)
diff --git a/java/calcite-planner/pom.xml b/java/calcite-planner/pom.xml
index 2ccd519f7..e00925046 100644
--- a/java/calcite-planner/pom.xml
+++ b/java/calcite-planner/pom.xml
@@ -48,6 +48,11 @@ under the License.
<artifactId>avatica-core</artifactId>
<version>1.23.0</version>
</dependency>
+ <dependency>
+ <groupId>org.immutables</groupId>
+ <artifactId>value</artifactId>
+ <version>2.10.1</version>
+ </dependency>
</dependencies>
<build>
@@ -60,6 +65,18 @@ under the License.
</plugins>
</pluginManagement>
<plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>auto-clean</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ExtractLiteralAgg.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ExtractLiteralAgg.java
new file mode 100644
index 000000000..ed1c9cfe9
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ExtractLiteralAgg.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.rules;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * ExtractLiteralAgg rule gets rid of the LITERAL_AGG into
+ * something Impala can handle.
+ *
+ * Calcite can create an Aggregate RelNode with a single LITERAL_AGG AggregateCall
+ * to check for existence in a subquery. In this case, the Aggregate will do a group
+ * by, and just spit out a literal value for each row.
+ * The LITERAL_AGG function does not exist in Impala. In order to create a RelNode
+ * structure that Impala supports, this Aggregate is turned into an Aggregate that
+ * still does the "group by" on the relevant groups, but places a Project RelNode
+ * on top that returns the literal value.
+ */
+public class ExtractLiteralAgg extends RelOptRule {
+
+ public ExtractLiteralAgg() {
+ super(operand(LogicalAggregate.class, none()));
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final LogicalAggregate aggregate = call.rel(0);
+
+ if (hasSingleLiteralAgg(aggregate.getAggCallList())) {
+ LogicalAggregate newAggregate = LogicalAggregate.create(aggregate.getInput(),
+ aggregate.getHints(), aggregate.getGroupSet(), aggregate.getGroupSets(),
+ new ArrayList<>());
+ RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
+ List<RexNode> projects = new ArrayList<>();
+ // Because we know there is only one agg call and its a literal agg, the
+ // first n - 1 columns are not changed in the newly created project.
+ for (int i = 0; i < aggregate.getRowType().getFieldCount() - 1; ++i) {
+ projects.add(rexBuilder.makeInputRef(aggregate, i));
+ }
+ AggregateCall aggCall = aggregate.getAggCallList().get(0);
+ Preconditions.checkState(aggCall.rexList.get(0) instanceof RexLiteral);
+ // Add the nth column as the LITERAL_AGG value
+ projects.add(aggCall.rexList.get(0));
+ RelNode projectRelNode = LogicalProject.create(newAggregate, new ArrayList<>(),
+ projects, aggregate.getRowType().getFieldNames(), new HashSet<>());
+ call.transformTo(projectRelNode);
+ }
+ }
+
+ private boolean hasSingleLiteralAgg(List<AggregateCall> aggCallList) {
+ if (aggCallList.size() != 1) {
+ return false;
+ }
+ AggregateCall aggCall = aggCallList.get(0);
+ return (aggCall.getAggregation().getName().equals("LITERAL_AGG"));
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaMinusToDistinctRule.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaMinusToDistinctRule.java
new file mode 100644
index 000000000..71702b73f
--- /dev/null
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaMinusToDistinctRule.java
@@ -0,0 +1,214 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.rules;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelRule;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Minus;
+import org.apache.calcite.rel.rules.TransformationRule;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Util;
+import org.apache.impala.calcite.operators.ImpalaOperator;
+
+import com.google.common.collect.ImmutableList;
+
+import org.immutables.value.Value;
+
+import java.math.BigDecimal;
+
+/**
+ * ImpalaMinusToDistinctRule is the same as Calcite's MinustToDistinctRule with some
+ * changes. The Calcite code was grabbed from here:
+ * https://github.com/apache/calcite/blob/calcite-1.36.0/core/src/main/java/org/apache/...
+ * /calcite/rel/rules/MinusToDistinctRule.java
+ *
+ * The Calcite code generates a "count() filter" operation which is not supported by
+ * Impala. A couple of small changes were made in the creation of the RelNodes which
+ * can be found in this file by searching for 'IMPALA CHANGE' in multiple places.
+ */
+
+/**
+ * Planner rule that translates a distinct
+ * {@link org.apache.calcite.rel.core.Minus}
+ * (<code>all</code> = <code>false</code>)
+ * into a group of operators composed of
+ * {@link org.apache.calcite.rel.core.Union},
+ * {@link org.apache.calcite.rel.core.Aggregate},
+ * {@link org.apache.calcite.rel.core.Filter},etc.
+ *
+ * <p>For example, the query plan
+
+ * <blockquote><pre>{@code
+ * LogicalMinus(all=[false])
+ * LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2])
+ * LogicalFilter(condition=[=($7, 10)])
+ * LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ * LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2])
+ * LogicalFilter(condition=[=($7, 20)])
+ * LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ * }</pre></blockquote>
+ *
+ * <p> will convert to
+ *
+ * <blockquote><pre>{@code
+ * LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2])
+ * LogicalFilter(condition=[AND(>($3, 0), =($4, 0))])
+ * LogicalAggregate(group=[{0, 1, 2}], agg#0=[COUNT() FILTER $3],
+ * agg#1=[COUNT() FILTER $4])
+ * LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], $f3=[=($3, 0)], $f4=[=($3, 1)])
+ * LogicalUnion(all=[true])
+ * LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], $f3=[0])
+ * LogicalFilter(condition=[=($7, 10)])
+ * LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ * LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], $f3=[1])
+ * LogicalFilter(condition=[=($7, 20)])
+ * LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ * }</pre></blockquote>
+ *
+ * @see CoreRules#MINUS_TO_DISTINCT
+ */
+@Value.Enclosing
+public class ImpalaMinusToDistinctRule
+ extends RelRule<ImpalaMinusToDistinctRule.Config>
+ implements TransformationRule {
+
+ protected ImpalaMinusToDistinctRule(Config config) {
+ super(config);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Minus minus = call.rel(0);
+
+ if (minus.all) {
+ // Nothing we can do
+ return;
+ }
+
+ final RelOptCluster cluster = minus.getCluster();
+ final RelBuilder relBuilder = call.builder();
+ final RexBuilder rexBuilder = cluster.getRexBuilder();
+ final int branchCount = minus.getInputs().size();
+
+ // For each child branch in minus, add a column which indicates the branch index
+ //
+ // e.g., select EMPNO from emp -> select EMPNO, 0 from emp
+ // 0 indicates that it comes from the first child branch (operand) of the minus
+ // operator
+ for (int i = 0; i < branchCount; i++) {
+ relBuilder.push(minus.getInput(i));
+ relBuilder.projectPlus(relBuilder.literal(new BigDecimal(i)));
+ }
+
+ // create a union above all the branches
+ relBuilder.union(true, branchCount);
+
+ final RelNode union = relBuilder.peek();
+ final int originalFieldCnt = union.getRowType().getFieldCount() - 1;
+
+ ImmutableList.Builder<RexNode> projects = ImmutableList.builder();
+ // skip the branch index column
+ projects.addAll(Util.first(relBuilder.fields(), originalFieldCnt));
+
+ // IMPALA CHANGE 1, commented out below code because Impala does not support
+ // COUNT FILTER operation
+ // On top of the Union, add a Project and add one boolean column per branch counter,
+ // where the i-th boolean column is true iff the tuple comes from the i-th branch
+ //
+ // e.g., LogicalProject(EMPNO=[$0], $f1=[=($1, 0)], $f2=[=($1, 1)], $f3=[=($1, 2)])
+ // $f1,$f2,$f3 are the boolean indicate whether it comes from the corresponding branch
+ //for (int i = 0; i < branchCount; i++) {
+ // projects.add(
+ // relBuilder.equals(relBuilder.field(originalFieldCnt),
+ // relBuilder.literal(new BigDecimal(i))));
+ //}
+ // IMPALA CHANGE 2, instead, added this code:
+ // The count filter operation is not supported. Using the above example, the code
+ // we create for Impala is:
+ // e.g., LogicalProject(EMPNO=[$0], $f1=[if(=($1, 0),1,null)],
+ // $f2=[if(=($1, 1),1,null)],
+ // $f3=[if(=($1, 2),1,null)])
+ // A '1' is created for counting purposes only if it comes from the branch, and a null
+ // otherwise. Since the count() operation will not count null values for the field,
+ // we get the count for each branch for each new column
+ for (int i = 0; i < branchCount; i++) {
+ RexNode equalsNode = relBuilder.equals(
+ relBuilder.field(originalFieldCnt), relBuilder.literal(new BigDecimal(i)));
+ RexNode one = relBuilder.literal(new BigDecimal(1));
+ RexNode ifOp = rexBuilder.makeCall(new ImpalaOperator("if"),
+ ImmutableList.of(equalsNode, one, rexBuilder.makeNullLiteral(one.getType())));
+ projects.add(ifOp);
+ }
+
+ relBuilder.project(projects.build());
+
+ // IMPALA CHANGE 3: commented out below code because Impala does not support
+ // COUNT FILTER operation
+ // Add the count(*) filter $f1(..) for each branch
+ //ImmutableList.Builder<RelBuilder.AggCall> aggCalls = ImmutableList.builder();
+ //for (int i = 0; i < branchCount; i++) {
+ // aggCalls.add(relBuilder.countStar(null).filter(
+ // relBuilder.field(originalFieldCnt + i)));
+ //}
+ // IMPALA CHANGE 4: Instead, added just a plain count. This combined with the above
+ // logic should provide the same count as the count filter operation.
+ ImmutableList.Builder<RelBuilder.AggCall> aggCalls = ImmutableList.builder();
+ for (int i = 0; i < branchCount; i++) {
+ aggCalls.add(relBuilder.count(relBuilder.field(originalFieldCnt + i)));
+ }
+
+ final ImmutableBitSet groupSet = ImmutableBitSet.range(originalFieldCnt);
+ relBuilder.aggregate(relBuilder.groupKey(groupSet), aggCalls.build());
+
+ ImmutableList.Builder<RexNode> filters = ImmutableList.builder();
+ for (int i = 0; i < branchCount; i++) {
+ SqlOperator operator =
+ i == 0 ? SqlStdOperatorTable.GREATER_THAN
+ : SqlStdOperatorTable.EQUALS;
+ filters.add(
+ rexBuilder.makeCall(operator, relBuilder.field(originalFieldCnt + i),
+ relBuilder.literal(new BigDecimal(0))));
+ }
+
+ relBuilder.filter(filters.build());
+ relBuilder.project(Util.first(relBuilder.fields(), originalFieldCnt));
+ call.transformTo(relBuilder.build());
+ }
+
+ /**
+ * Rule configuration.
+ */
+ @Value.Immutable
+ public interface Config extends RelRule.Config {
+ Config DEFAULT = ImmutableImpalaMinusToDistinctRule.Config.builder()
+ .operandSupplier(b -> b.operand(LogicalMinus.class).anyInputs()).build();
+
+ @Override default ImpalaMinusToDistinctRule toRule() {
+ return new ImpalaMinusToDistinctRule(this);
+ }
+ }
+}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java
index fe0f74407..0b3aaef6c 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java
@@ -34,6 +34,8 @@ import org.apache.impala.calcite.coercenodes.CoerceNodes;
import org.apache.impala.calcite.rel.node.ConvertToImpalaRelRules;
import org.apache.impala.calcite.rel.node.ImpalaPlanRel;
import org.apache.impala.calcite.rules.ConvertToCNFRules;
+import org.apache.impala.calcite.rules.ExtractLiteralAgg;
+import org.apache.impala.calcite.rules.ImpalaMinusToDistinctRule;
import org.apache.impala.common.ImpalaException;
import org.slf4j.Logger;
@@ -65,6 +67,14 @@ public class CalciteOptimizer implements CompilerStep {
// Run essential Join Node rules
RelNode optimizedPlan = runJoinProgram(relBuilder, expandedNodesPlan);
+ logDebug(optimizedPlan);
+
+ // Run some essential rules needed to create working RelNodes after
+ // optimization
+ optimizedPlan = runPreImpalaConvertProgram(relBuilder, optimizedPlan);
+
+ logDebug(optimizedPlan);
+
ImpalaPlanRel finalOptimizedPlan =
runImpalaConvertProgram(relBuilder, optimizedPlan);
@@ -78,9 +88,12 @@ public class CalciteOptimizer implements CompilerStep {
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
builder.addRuleCollection(ImmutableList.of(
+ CoreRules.INTERSECT_TO_DISTINCT,
+ CoreRules.UNION_TO_DISTINCT,
new ConvertToCNFRules.FilterConvertToCNFRule(),
new ConvertToCNFRules.JoinConvertToCNFRule(),
- new ConvertToCNFRules.ProjectConvertToCNFRule()
+ new ConvertToCNFRules.ProjectConvertToCNFRule(),
+ ImpalaMinusToDistinctRule.Config.DEFAULT.toRule()
));
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
@@ -116,21 +129,40 @@ public class CalciteOptimizer implements CompilerStep {
// MULTI_JOIN_OPTIMIZE rule is used to determine the join
// ordering.
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
+ builder.addRuleInstance(CoreRules.JOIN_CONDITION_PUSH);
builder.addRuleInstance(CoreRules.JOIN_TO_MULTI_JOIN);
builder.addRuleInstance(CoreRules.MULTI_JOIN_OPTIMIZE);
+ builder.addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE);
return runProgram(plan, builder.build());
}
- public ImpalaPlanRel runImpalaConvertProgram(RelBuilder relBuilder,
+ public RelNode runPreImpalaConvertProgram(RelBuilder relBuilder,
RelNode plan) throws ImpalaException {
- RelNode preConversionNode =
- CoerceNodes.coerceNodes(plan, plan.getCluster().getRexBuilder());
-
HepProgramBuilder builder = new HepProgramBuilder();
+ // Impala cannot handle the LITERAL_AGG method so we need to create
+ // an equivalent plan
+ RelNode retRelNode = plan;
builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
+ builder.addRuleCollection(ImmutableList.of(
+ new ExtractLiteralAgg()
+ ));
+
+
+ retRelNode = runProgram(retRelNode, builder.build());
+
+ // Fix up the operands for the nodes which may also change some return types that
+ // propagate upwards in the plan.
+ return CoerceNodes.coerceNodes(retRelNode, plan.getCluster().getRexBuilder());
+
+ }
+
+ public ImpalaPlanRel runImpalaConvertProgram(RelBuilder relBuilder,
+ RelNode plan) throws ImpalaException {
+ HepProgramBuilder builder = new HepProgramBuilder();
+
builder.addRuleCollection(ImmutableList.of(
new ConvertToImpalaRelRules.ImpalaScanRule(),
new ConvertToImpalaRelRules.ImpalaSortRule(),
@@ -142,7 +174,7 @@ public class CalciteOptimizer implements CompilerStep {
new ConvertToImpalaRelRules.ImpalaValuesRule()
));
- return (ImpalaPlanRel) runProgram(preConversionNode, builder.build());
+ return (ImpalaPlanRel) runProgram(plan, builder.build());
}
private RelNode runProgram(RelNode currentNode, HepProgram program) {
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java
index 30ce7e444..a230205d2 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java
@@ -17,20 +17,33 @@
package org.apache.impala.calcite.service;
+import com.google.common.collect.ImmutableList;
import org.apache.calcite.plan.ConventionTraitDef;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCostImpl;
import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.hep.HepMatchOrder;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.rules.CoreRules;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.sql.SqlExplainFormat;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql2rel.RelDecorrelator;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.impala.calcite.operators.ImpalaConvertletTable;
+import org.apache.calcite.sql2rel.StandardConvertletTable;
+import org.apache.calcite.tools.RelBuilder;
+
+import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,13 +80,30 @@ public class CalciteRelNodeConverter implements CompilerStep {
validator_.getCatalogReader(),
cluster_,
ImpalaConvertletTable.INSTANCE,
- SqlToRelConverter.config());
+ SqlToRelConverter.config().withCreateValuesRel(false));
// Convert the valid AST into a logical plan
RelRoot root = relConverter.convertQuery(validatedNode, false, true);
RelNode relNode = root.project();
logDebug(relNode);
- return relNode;
+
+ RelNode subQueryRemovedPlan =
+ runProgram(
+ ImmutableList.of(
+ CoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
+ CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE,
+ CoreRules.FILTER_SUB_QUERY_TO_CORRELATE
+ ),
+ relNode);
+ logDebug(subQueryRemovedPlan);
+
+ RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster_,
+ validator_.getCatalogReader());
+ RelNode decorrelatedPlan =
+ RelDecorrelator.decorrelateQuery(subQueryRemovedPlan, relBuilder);
+
+ logDebug(decorrelatedPlan);
+ return decorrelatedPlan;
}
public RelOptCluster getCluster() {
@@ -93,4 +123,17 @@ public class CalciteRelNodeConverter implements CompilerStep {
LOG.info(RelOptUtil.dumpPlan("[Logical plan]", (RelNode) resultObject,
SqlExplainFormat.TEXT, SqlExplainLevel.NON_COST_ATTRIBUTES));
}
+
+ private RelNode runProgram(List<RelOptRule> rules, RelNode currentNode) {
+ HepProgramBuilder builder = new HepProgramBuilder();
+ // rules to convert Calcite nodes into ImpalaPlanRel nodes
+ builder.addRuleCollection(rules);
+ builder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
+
+ HepPlanner planner = new HepPlanner(builder.build(),
+ currentNode.getCluster().getPlanner().getContext(),
+ false, null, RelOptCostImpl.FACTORY);
+ planner.setRoot(currentNode);
+ return planner.findBestExp();
+ }
}