You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/22 03:14:17 UTC
kylin git commit: KYLIN-1770 upgrade Calcite to 1.10
Repository: kylin
Updated Branches:
refs/heads/master f0fbffd58 -> c3bb878fa
KYLIN-1770 upgrade Calcite to 1.10
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3bb878f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3bb878f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3bb878f
Branch: refs/heads/master
Commit: c3bb878fa87f17d4002080e2f2cb5849677f9c8b
Parents: f0fbffd
Author: Billy Liu <bi...@apache.org>
Authored: Mon Dec 19 18:53:52 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 22 11:14:03 2016 +0800
----------------------------------------------------------------------
.../calcite/sql2rel/SqlToRelConverter.java | 1178 ++++++++++--------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
pom.xml | 5 +-
.../kylin/query/relnode/OLAPTableScan.java | 2 +
4 files changed, 653 insertions(+), 532 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3bb878f/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index cf36f61..7da3040 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -16,6 +16,24 @@
*/
package org.apache.calcite.sql2rel;
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
import org.apache.calcite.avatica.util.Spaces;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Convention;
@@ -159,6 +177,7 @@ import org.apache.calcite.util.NumberUtil;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.trace.CalciteTrace;
+import org.slf4j.Logger;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
@@ -170,27 +189,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
/*
+ * The code has synced with calcite. Hope one day, we could remove the hardcode override point.
* OVERRIDE POINT:
* - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
* - isTrimUnusedFields(), override to false
@@ -209,57 +209,42 @@ import static org.apache.calcite.util.Static.RESOURCE;
public class SqlToRelConverter {
//~ Static fields/initializers ---------------------------------------------
- protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
-
- private static final BigDecimal TWO = BigDecimal.valueOf(2L);
-
/** Size of the smallest IN list that will be converted to a semijoin to a
* static table. */
- public static final int IN_SUBQUERY_THRESHOLD = 20;
+ public static final int DEFAULT_IN_SUBQUERY_THRESHOLD = 20;
+ protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer();
+ private static final BigDecimal TWO = BigDecimal.valueOf(2L);
//~ Instance fields --------------------------------------------------------
-
+ public final SqlToRelConverter.Config config;
+ public final RelOptTable.ViewExpander viewExpander;
protected final SqlValidator validator;
protected final RexBuilder rexBuilder;
protected final Prepare.CatalogReader catalogReader;
protected final RelOptCluster cluster;
- private DefaultValueFactory defaultValueFactory;
- private SubqueryConverter subqueryConverter;
protected final List<RelNode> leaves = new ArrayList<>();
+ protected final RelDataTypeFactory typeFactory;
private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
private final SqlOperatorTable opTab;
- private boolean shouldConvertTableAccess;
- protected final RelDataTypeFactory typeFactory;
private final SqlNodeToRexConverter exprConverter;
- private boolean decorrelationEnabled;
- private boolean trimUnusedFields;
- private boolean shouldCreateValuesRel;
- private boolean isExplain;
- private int nDynamicParamsInExplain;
-
/**
* Fields used in name resolution for correlated subqueries.
*/
private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred = new HashMap<>();
-
/**
* Stack of names of datasets requested by the <code>
* TABLE(SAMPLE(<datasetName>, <query>))</code> construct.
*/
private final Deque<String> datasetStack = new ArrayDeque<>();
-
/**
* Mapping of non-correlated subqueries that have been converted to their
* equivalent constants. Used to avoid re-evaluating the subquery if it's
* already been evaluated.
*/
private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs = new HashMap<>();
-
- public final RelOptTable.ViewExpander viewExpander;
-
- /** Whether to expand sub-queries. If false, each sub-query becomes a
- * {@link org.apache.calcite.rex.RexSubQuery}. */
- private boolean expand = true;
+ private DefaultValueFactory defaultValueFactory;
+ private SubqueryConverter subqueryConverter;
+ private int explainParamCount;
//~ Constructors -----------------------------------------------------------
/**
@@ -272,13 +257,18 @@ public class SqlToRelConverter {
* @param rexBuilder Rex builder
* @param convertletTable Expression converter
*/
- @Deprecated // will be removed before 2.0
+ @Deprecated // to be removed before 2.0
public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable) {
- this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable);
+ this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable, Config.DEFAULT);
}
- /* Creates a converter. */
+ @Deprecated // to be removed before 2.0
public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable) {
+ this(viewExpander, validator, catalogReader, cluster, convertletTable, Config.DEFAULT);
+ }
+
+ /* Creates a converter. */
+ public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable, Config config) {
this.viewExpander = viewExpander;
this.opTab = (validator == null) ? SqlStdOperatorTable.instance() : validator.getOperatorTable();
this.validator = validator;
@@ -288,17 +278,164 @@ public class SqlToRelConverter {
this.rexBuilder = cluster.getRexBuilder();
this.typeFactory = rexBuilder.getTypeFactory();
this.cluster = Preconditions.checkNotNull(cluster);
- this.shouldConvertTableAccess = true;
this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable);
- decorrelationEnabled = true;
- trimUnusedFields = false;
- shouldCreateValuesRel = true;
- isExplain = false;
- nDynamicParamsInExplain = 0;
+ this.explainParamCount = 0;
+ this.config = new ConfigBuilder().withConfig(config).build();
}
//~ Methods ----------------------------------------------------------------
+ private static boolean isStream(SqlNode query) {
+ return query instanceof SqlSelect && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ }
+
+ public static boolean isOrdered(SqlNode query) {
+ switch (query.getKind()) {
+ case SELECT:
+ return ((SqlSelect) query).getOrderList() != null && ((SqlSelect) query).getOrderList().size() > 0;
+ case WITH:
+ return isOrdered(((SqlWith) query).body);
+ case ORDER_BY:
+ return ((SqlOrderBy) query).orderList.size() > 0;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Returns whether a given node contains a {@link SqlInOperator}.
+ *
+ * @param node a RexNode tree
+ */
+ private static boolean containsInOperator(SqlNode node) {
+ try {
+ SqlVisitor<Void> visitor = new SqlBasicVisitor<Void>() {
+ public Void visit(SqlCall call) {
+ if (call.getOperator() instanceof SqlInOperator) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visit(call);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
+ }
+ }
+
+ /**
+ * Push down all the NOT logical operators into any IN/NOT IN operators.
+ *
+ * @param sqlNode the root node from which to look for NOT operators
+ * @return the transformed SqlNode representation with NOT pushed down.
+ */
+ private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
+ if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
+ SqlCall sqlCall = (SqlCall) sqlNode;
+ if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+ SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
+ for (int i = 0; i < sqlOperands.length; i++) {
+ sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
+ }
+ return sqlNode;
+ } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode childNode = sqlCall.operand(0);
+ assert childNode instanceof SqlCall;
+ SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
+ if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
+ SqlNode[] andOperands = childSqlCall.getOperands();
+ SqlNode[] orOperands = new SqlNode[andOperands.length];
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]);
+ }
+ for (int i = 0; i < orOperands.length; i++) {
+ orOperands[i] = pushDownNotForIn(orOperands[i]);
+ }
+ return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
+ SqlNode[] orOperands = childSqlCall.getOperands();
+ SqlNode[] andOperands = new SqlNode[orOperands.length];
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]);
+ }
+ for (int i = 0; i < andOperands.length; i++) {
+ andOperands[i] = pushDownNotForIn(andOperands[i]);
+ }
+ return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]);
+ } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
+ SqlNode[] notOperands = childSqlCall.getOperands();
+ assert notOperands.length == 1;
+ return pushDownNotForIn(notOperands[0]);
+ } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
+ SqlNode[] inOperands = childSqlCall.getOperands();
+ SqlInOperator inOp = (SqlInOperator) childSqlCall.getOperator();
+ if (inOp.isNotIn()) {
+ return SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
+ } else {
+ return SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
+ }
+ } else {
+ // childSqlCall is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // sqlNode is "leaf" node in a logical expression tree
+ // (only considering AND, OR, NOT)
+ return sqlNode;
+ }
+ } else {
+ // tree rooted at sqlNode does not contain inOperator
+ return sqlNode;
+ }
+ }
+
+ private static boolean containsNullLiteral(SqlNodeList valueList) {
+ for (SqlNode node : valueList.getList()) {
+ if (node instanceof SqlLiteral) {
+ SqlLiteral lit = (SqlLiteral) node;
+ if (lit.getValue() == null) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static JoinRelType convertJoinType(JoinType joinType) {
+ switch (joinType) {
+ case COMMA:
+ case INNER:
+ case CROSS:
+ return JoinRelType.INNER;
+ case FULL:
+ return JoinRelType.FULL;
+ case LEFT:
+ return JoinRelType.LEFT;
+ case RIGHT:
+ return JoinRelType.RIGHT;
+ default:
+ throw Util.unexpected(joinType);
+ }
+ }
+
+ private static boolean desc(RelFieldCollation.Direction direction) {
+ switch (direction) {
+ case DESCENDING:
+ case STRICTLY_DESCENDING:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /** Creates a builder for a {@link Config}. */
+ public static ConfigBuilder configBuilder() {
+ return new ConfigBuilder();
+ }
+
/**
* @return the RelOptCluster in use.
*/
@@ -345,9 +482,9 @@ public class SqlToRelConverter {
* @return the current count before the optional increment
*/
public int getDynamicParamCountInExplain(boolean increment) {
- int retVal = nDynamicParamsInExplain;
+ int retVal = explainParamCount;
if (increment) {
- ++nDynamicParamsInExplain;
+ ++explainParamCount;
}
return retVal;
}
@@ -392,41 +529,14 @@ public class SqlToRelConverter {
}
/**
- * Indicates that the current statement is part of an EXPLAIN PLAN statement
- *
- * @param nDynamicParams number of dynamic parameters in the statement
- */
- public void setIsExplain(int nDynamicParams) {
- isExplain = true;
- nDynamicParamsInExplain = nDynamicParams;
- }
-
- /**
- * Controls whether table access references are converted to physical rels
- * immediately. The optimizer doesn't like leaf rels to have
- * {@link Convention#NONE}. However, if we are doing further conversion
- * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to
- * defer conversion. To have any effect, this must be called before any
- * convert method.
+ * Sets the number of dynamic parameters in the current EXPLAIN PLAN
+ * statement.
*
- * @param enabled true for immediate conversion (the default); false to
- * generate logical LogicalTableScan instances
+ * @param explainParamCount number of dynamic parameters in the statement
*/
- public void enableTableAccessConversion(boolean enabled) {
- shouldConvertTableAccess = enabled;
- }
-
- /**
- * Controls whether instances of
- * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These
- * may not be supported by all physical implementations. To have any effect,
- * this must be called before any convert method.
- *
- * @param enabled true to allow LogicalValues to be generated (the default);
- * false to force substitution of Project+OneRow instead
- */
- public void enableValuesRelCreation(boolean enabled) {
- shouldCreateValuesRel = enabled;
+ public void setDynamicParamCountInExplain(int explainParamCount) {
+ assert config.isExplain();
+ this.explainParamCount = explainParamCount;
}
private void checkConvertedType(SqlNode query, RelNode result) {
@@ -439,7 +549,7 @@ public class SqlToRelConverter {
// validator type information associated with its result,
// hence the namespace check above.)
final List<RelDataTypeField> validatedFields = validator.getValidatedNodeType(query).getFieldList();
- final RelDataType validatedRowType = validator.getTypeFactory().createStructType(Pair.right(validatedFields), SqlValidatorUtil.uniquify(Pair.left(validatedFields)));
+ final RelDataType validatedRowType = validator.getTypeFactory().createStructType(Pair.right(validatedFields), SqlValidatorUtil.uniquify(Pair.left(validatedFields), catalogReader.isCaseSensitive()));
final List<RelDataTypeField> convertedFields = result.getRowType().getFieldList().subList(0, validatedFields.size());
final RelDataType convertedRowType = validator.getTypeFactory().createStructType(convertedFields);
@@ -616,23 +726,6 @@ public class SqlToRelConverter {
return root;
}
- private static boolean isStream(SqlNode query) {
- return query instanceof SqlSelect && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
- }
-
- public static boolean isOrdered(SqlNode query) {
- switch (query.getKind()) {
- case SELECT:
- return ((SqlSelect) query).getOrderList() != null && ((SqlSelect) query).getOrderList().size() > 0;
- case WITH:
- return isOrdered(((SqlWith) query).body);
- case ORDER_BY:
- return ((SqlOrderBy) query).orderList.size() > 0;
- default:
- return false;
- }
- }
-
private RelCollation requiredCollation(RelNode r) {
if (r instanceof Sort) {
return ((Sort) r).collation;
@@ -813,100 +906,10 @@ public class SqlToRelConverter {
}
/**
- * Returns whether a given node contains a {@link SqlInOperator}.
+ * Converts a WHERE clause.
*
- * @param node a RexNode tree
- */
- private static boolean containsInOperator(SqlNode node) {
- try {
- SqlVisitor<Void> visitor = new SqlBasicVisitor<Void>() {
- public Void visit(SqlCall call) {
- if (call.getOperator() instanceof SqlInOperator) {
- throw new Util.FoundOne(call);
- }
- return super.visit(call);
- }
- };
- node.accept(visitor);
- return false;
- } catch (Util.FoundOne e) {
- Util.swallow(e, null);
- return true;
- }
- }
-
- /**
- * Push down all the NOT logical operators into any IN/NOT IN operators.
- *
- * @param sqlNode the root node from which to look for NOT operators
- * @return the transformed SqlNode representation with NOT pushed down.
- */
- private static SqlNode pushDownNotForIn(SqlNode sqlNode) {
- if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
- SqlCall sqlCall = (SqlCall) sqlNode;
- if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
- SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
- for (int i = 0; i < sqlOperands.length; i++) {
- sqlOperands[i] = pushDownNotForIn(sqlOperands[i]);
- }
- return sqlNode;
- } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode childNode = sqlCall.operand(0);
- assert childNode instanceof SqlCall;
- SqlBasicCall childSqlCall = (SqlBasicCall) childNode;
- if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) {
- SqlNode[] andOperands = childSqlCall.getOperands();
- SqlNode[] orOperands = new SqlNode[andOperands.length];
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]);
- }
- for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = pushDownNotForIn(orOperands[i]);
- }
- return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
- SqlNode[] orOperands = childSqlCall.getOperands();
- SqlNode[] andOperands = new SqlNode[orOperands.length];
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]);
- }
- for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = pushDownNotForIn(andOperands[i]);
- }
- return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]);
- } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
- SqlNode[] notOperands = childSqlCall.getOperands();
- assert notOperands.length == 1;
- return pushDownNotForIn(notOperands[0]);
- } else if (childSqlCall.getOperator() instanceof SqlInOperator) {
- SqlNode[] inOperands = childSqlCall.getOperands();
- SqlInOperator inOp = (SqlInOperator) childSqlCall.getOperator();
- if (inOp.isNotIn()) {
- return SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
- } else {
- return SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]);
- }
- } else {
- // childSqlCall is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // sqlNode is "leaf" node in a logical expression tree
- // (only considering AND, OR, NOT)
- return sqlNode;
- }
- } else {
- // tree rooted at sqlNode does not contain inOperator
- return sqlNode;
- }
- }
-
- /**
- * Converts a WHERE clause.
- *
- * @param bb Blackboard
- * @param where WHERE clause, may be null
+ * @param bb Blackboard
+ * @param where WHERE clause, may be null
*/
private void convertWhere(final Blackboard bb, final SqlNode where) {
if (where == null) {
@@ -968,7 +971,7 @@ public class SqlToRelConverter {
case IN:
call = (SqlBasicCall) subQuery.node;
query = call.operand(1);
- if (!expand && !(query instanceof SqlNodeList)) {
+ if (!config.isExpand() && !(query instanceof SqlNodeList)) {
return;
}
final SqlNode leftKeyNode = call.operand(0);
@@ -1056,7 +1059,7 @@ public class SqlToRelConverter {
// boolean indicating whether the subquery returned 0 or >= 1 row.
call = (SqlBasicCall) subQuery.node;
query = call.operand(0);
- if (!expand) {
+ if (!config.isExpand()) {
return;
}
converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, subQuery.logic, true, null);
@@ -1070,7 +1073,7 @@ public class SqlToRelConverter {
case SCALAR_QUERY:
// Convert the subquery. If it's non-correlated, convert it
// to a constant expression.
- if (!expand) {
+ if (!config.isExpand()) {
return;
}
call = (SqlBasicCall) subQuery.node;
@@ -1169,18 +1172,6 @@ public class SqlToRelConverter {
}
}
- private static boolean containsNullLiteral(SqlNodeList valueList) {
- for (SqlNode node : valueList.getList()) {
- if (node instanceof SqlLiteral) {
- SqlLiteral lit = (SqlLiteral) node;
- if (lit.getValue() == null) {
- return true;
- }
- }
- }
- return false;
- }
-
/**
* Determines if a subquery is non-correlated and if so, converts it to a
* constant.
@@ -1199,7 +1190,7 @@ public class SqlToRelConverter {
// it again.
RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
if (constExpr == null) {
- constExpr = subqueryConverter.convertSubquery(call, this, isExists, isExplain);
+ constExpr = subqueryConverter.convertSubquery(call, this, isExists, config.isExplain());
}
if (constExpr != null) {
subQuery.expr = constExpr;
@@ -1270,14 +1261,14 @@ public class SqlToRelConverter {
for (SqlNode rightVals : valuesList) {
RexNode rexComparison;
if (leftKeys.size() == 1) {
- rexComparison = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, leftKeys.get(0), rexBuilder.ensureType(leftKeys.get(0).getType(), bb.convertExpression(rightVals), true));
+ rexComparison = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, leftKeys.get(0), ensureSqlType(leftKeys.get(0).getType(), bb.convertExpression(rightVals)));
} else {
assert rightVals instanceof SqlCall;
final SqlBasicCall call = (SqlBasicCall) rightVals;
assert (call.getOperator() instanceof SqlRowOperator) && call.operandCount() == leftKeys.size();
rexComparison = RexUtil.composeConjunction(rexBuilder, Iterables.transform(Pair.zip(leftKeys, call.getOperandList()), new Function<Pair<RexNode, SqlNode>, RexNode>() {
public RexNode apply(Pair<RexNode, SqlNode> pair) {
- return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, pair.left, rexBuilder.ensureType(pair.left.getType(), bb.convertExpression(pair.right), true));
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, pair.left, ensureSqlType(pair.left.getType(), bb.convertExpression(pair.right)));
}
}), false);
}
@@ -1294,6 +1285,16 @@ public class SqlToRelConverter {
return result;
}
+ /** Ensures that an expression has a given {@link SqlTypeName}, applying a
+ * cast if necessary. If the expression already has the right type family,
+ * returns the expression unchanged. */
+ private RexNode ensureSqlType(RelDataType type, RexNode node) {
+ if (type.getSqlTypeName() == node.getType().getSqlTypeName() || (type.getSqlTypeName() == SqlTypeName.VARCHAR && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
+ return node;
+ }
+ return rexBuilder.ensureType(type, node, true);
+ }
+
/**
* Gets the list size threshold under which {@link #convertInToOr} is used.
* Lists of this size or greater will instead be converted to use a join
@@ -1302,9 +1303,11 @@ public class SqlToRelConverter {
* predicate. A threshold of 0 forces usage of an inline table in all cases; a
* threshold of Integer.MAX_VALUE forces usage of OR in all cases
*
- * @return threshold, default {@link #IN_SUBQUERY_THRESHOLD}
+ * @return threshold, default {@link #DEFAULT_IN_SUBQUERY_THRESHOLD}
*/
+ @Deprecated // to be removed before 2.0
protected int getInSubqueryThreshold() {
+ //return config.getInSubqueryThreshold();
/* OVERRIDE POINT */
return Integer.MAX_VALUE;
}
@@ -1374,7 +1377,7 @@ public class SqlToRelConverter {
if ((rexLiteral == null) && allowLiteralsOnly) {
return null;
}
- if ((rexLiteral == null) || !shouldCreateValuesRel) {
+ if ((rexLiteral == null) || !config.isCreateValuesRel()) {
// fallback to convertRowConstructor
tuple = null;
break;
@@ -1387,7 +1390,7 @@ public class SqlToRelConverter {
}
} else {
RexLiteral rexLiteral = convertLiteralInValuesList(node, bb, rowType, 0);
- if ((rexLiteral != null) && shouldCreateValuesRel) {
+ if ((rexLiteral != null) && config.isCreateValuesRel()) {
tupleList.add(ImmutableList.of(rexLiteral));
continue;
} else {
@@ -1705,7 +1708,7 @@ public class SqlToRelConverter {
boolean[] usedDataset = { false };
RelOptTable table = SqlValidatorUtil.getRelOptTable(fromNamespace, catalogReader, datasetName, usedDataset);
final RelNode tableRel;
- if (shouldConvertTableAccess) {
+ if (config.isConvertTableAccess()) {
tableRel = toRel(table);
} else {
tableRel = LogicalTableScan.create(cluster, table);
@@ -1815,7 +1818,7 @@ public class SqlToRelConverter {
final SqlUserDefinedTableMacro udf = (SqlUserDefinedTableMacro) operator;
final TranslatableTable table = udf.getTable(typeFactory, callBinding.operands());
final RelDataType rowType = table.getRowType(typeFactory);
- RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table);
+ RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table, udf.getNameAsId().names);
RelNode converted = toRel(relOptTable);
bb.setRoot(converted, true);
return;
@@ -1890,16 +1893,13 @@ public class SqlToRelConverter {
String originalRelName = lookup.getOriginalRelName();
String originalFieldName = fieldAccess.getField().getName();
- int[] nsIndexes = { -1 };
- final SqlValidatorScope[] ancestorScopes = { null };
- SqlValidatorNamespace foundNs = lookup.bb.scope.resolve(ImmutableList.of(originalRelName), ancestorScopes, nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
-
- int childNamespaceIndex = nsIndexes[0];
-
- SqlValidatorScope ancestorScope = ancestorScopes[0];
+ SqlValidatorScope.ResolvedImpl resolved = new SqlValidatorScope.ResolvedImpl();
+ lookup.bb.scope.resolve(ImmutableList.of(originalRelName), false, resolved);
+ assert resolved.count() == 1;
+ final SqlValidatorScope.Resolve resolve = resolved.only();
+ final SqlValidatorNamespace foundNs = resolve.namespace;
+ final int childNamespaceIndex = resolve.path.steps().get(0).i;
+ final SqlValidatorScope ancestorScope = resolve.scope;
boolean correlInCurrentScope = ancestorScope == bb.scope;
if (!correlInCurrentScope) {
@@ -1982,14 +1982,10 @@ public class SqlToRelConverter {
DeferredLookup lookup = mapCorrelToDeferred.get(correlName);
String originalRelName = lookup.getOriginalRelName();
- int[] nsIndexes = { -1 };
- final SqlValidatorScope[] ancestorScopes = { null };
- SqlValidatorNamespace foundNs = lookup.bb.scope.resolve(ImmutableList.of(originalRelName), ancestorScopes, nsIndexes);
-
- assert foundNs != null;
- assert nsIndexes.length == 1;
+ final SqlValidatorScope.ResolvedImpl resolved = new SqlValidatorScope.ResolvedImpl();
+ lookup.bb.scope.resolve(ImmutableList.of(originalRelName), false, resolved);
- SqlValidatorScope ancestorScope = ancestorScopes[0];
+ SqlValidatorScope ancestorScope = resolved.only().scope;
// If the correlated reference is in a scope that's "above" the
// subquery, then this is a correlated subquery.
@@ -2068,23 +2064,6 @@ public class SqlToRelConverter {
return RexUtil.composeConjunction(rexBuilder, list, false);
}
- private static JoinRelType convertJoinType(JoinType joinType) {
- switch (joinType) {
- case COMMA:
- case INNER:
- case CROSS:
- return JoinRelType.INNER;
- case FULL:
- return JoinRelType.FULL;
- case LEFT:
- return JoinRelType.LEFT;
- case RIGHT:
- return JoinRelType.RIGHT;
- default:
- throw Util.unexpected(joinType);
- }
- }
-
/**
* Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query.
*
@@ -2383,20 +2362,11 @@ public class SqlToRelConverter {
return new RelFieldCollation(ordinal + 1, direction, nullDirection);
}
- private static boolean desc(RelFieldCollation.Direction direction) {
- switch (direction) {
- case DESCENDING:
- case STRICTLY_DESCENDING:
- return true;
- default:
- return false;
- }
- }
-
+ @Deprecated // to be removed before 2.0
protected boolean enableDecorrelation() {
// disable subquery decorrelation when needed.
// e.g. if outer joins are not supported.
- return decorrelationEnabled;
+ return config.isDecorrelationEnabled();
}
protected RelNode decorrelateQuery(RelNode rootRel) {
@@ -2404,28 +2374,17 @@ public class SqlToRelConverter {
}
/**
- * Sets whether to trim unused fields as part of the conversion process.
- *
- * @param trim Whether to trim unused fields
- */
- public void setTrimUnusedFields(boolean trim) {
- this.trimUnusedFields = trim;
- }
-
- /**
* Returns whether to trim unused fields as part of the conversion process.
*
* @return Whether to trim unused fields
*/
+ @Deprecated // to be removed before 2.0
public boolean isTrimUnusedFields() {
+ //return config.isTrimUnusedFields();
/* OVERRIDE POINT */
return false;
}
- public void setExpand(boolean expand) {
- this.expand = expand;
- }
-
/**
* Recursively converts a query to a relational expression.
*
@@ -2581,9 +2540,11 @@ public class SqlToRelConverter {
return cluster;
}
- public RelRoot expandView(RelDataType rowType, String queryString, List<String> schemaPath) {
- return viewExpander.expandView(rowType, queryString, schemaPath);
+ @Override
+ public RelRoot expandView(RelDataType rowType, String queryString, List<String> schemaPath, List<String> viewPath) {
+ return viewExpander.expandView(rowType, queryString, schemaPath, viewPath);
}
+
};
}
@@ -2965,131 +2926,407 @@ public class SqlToRelConverter {
final List<RexNode> exprs = new ArrayList<>();
final Collection<String> aliases = new TreeSet<>();
- // Project any system fields. (Must be done before regular select items,
- // because offsets may be affected.)
- final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
- extraSelectItems(bb, select, exprs, fieldNames, aliases, columnMonotonicityList);
+ // Project any system fields. (Must be done before regular select items,
+ // because offsets may be affected.)
+ final List<SqlMonotonicity> columnMonotonicityList = new ArrayList<>();
+ extraSelectItems(bb, select, exprs, fieldNames, aliases, columnMonotonicityList);
+
+ // Project select clause.
+ int i = -1;
+ for (SqlNode expr : selectList) {
+ ++i;
+ exprs.add(bb.convertExpression(expr));
+ fieldNames.add(deriveAlias(expr, aliases, i));
+ }
+
+ // Project extra fields for sorting.
+ for (SqlNode expr : orderList) {
+ ++i;
+ SqlNode expr2 = validator.expandOrderExpr(select, expr);
+ exprs.add(bb.convertExpression(expr2));
+ fieldNames.add(deriveAlias(expr, aliases, i));
+ }
+
+ fieldNames = SqlValidatorUtil.uniquify(fieldNames, catalogReader.isCaseSensitive());
+
+ bb.setRoot(RelOptUtil.createProject(bb.root, exprs, fieldNames), false);
+
+ assert bb.columnMonotonicities.isEmpty();
+ bb.columnMonotonicities.addAll(columnMonotonicityList);
+ for (SqlNode selectItem : selectList) {
+ bb.columnMonotonicities.add(selectItem.getMonotonicity(bb.scope));
+ }
+ }
+
+ /**
+ * Adds extra select items. The default implementation adds nothing; derived
+ * classes may add columns to exprList, nameList, aliasList and
+ * columnMonotonicityList.
+ *
+ * @param bb Blackboard
+ * @param select Select statement being translated
+ * @param exprList List of expressions in select clause
+ * @param nameList List of names, one per column
+ * @param aliasList Collection of aliases that have been used
+ * already
+ * @param columnMonotonicityList List of monotonicity, one per column
+ */
+ protected void extraSelectItems(Blackboard bb, SqlSelect select, List<RexNode> exprList, List<String> nameList, Collection<String> aliasList, List<SqlMonotonicity> columnMonotonicityList) {
+ }
+
+ private String deriveAlias(final SqlNode node, Collection<String> aliases, final int ordinal) {
+ String alias = validator.deriveAlias(node, ordinal);
+ if ((alias == null) || aliases.contains(alias)) {
+ String aliasBase = (alias == null) ? "EXPR$" : alias;
+ for (int j = 0;; j++) {
+ alias = aliasBase + j;
+ if (!aliases.contains(alias)) {
+ break;
+ }
+ }
+ }
+ aliases.add(alias);
+ return alias;
+ }
+
+ /**
+ * Converts a WITH sub-query into a relational expression.
+ */
+ public RelRoot convertWith(SqlWith with, boolean top) {
+ return convertQuery(with.body, false, top);
+ }
+
+ /**
+ * Converts a SELECT statement's parse tree into a relational expression.
+ */
+ public RelNode convertValues(SqlCall values, RelDataType targetRowType) {
+ final SqlValidatorScope scope = validator.getOverScope(values);
+ assert scope != null;
+ final Blackboard bb = createBlackboard(scope, null, false);
+ convertValuesImpl(bb, values, targetRowType);
+ return bb.root;
+ }
+
+ //~ Inner Classes ----------------------------------------------------------
+
+ /**
+ * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a
+ * relational expression.
+ *
+ * @param bb Blackboard
+ * @param values Call to SQL VALUES operator
+ * @param targetRowType Target row type
+ */
+ private void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType targetRowType) {
+ // Attempt direct conversion to LogicalValues; if that fails, deal with
+ // fancy stuff like subqueries below.
+ RelNode valuesRel = convertRowValues(bb, values, values.getOperandList(), true, targetRowType);
+ if (valuesRel != null) {
+ bb.setRoot(valuesRel, true);
+ return;
+ }
+
+ final List<RelNode> unionRels = new ArrayList<>();
+ for (SqlNode rowConstructor1 : values.getOperandList()) {
+ SqlCall rowConstructor = (SqlCall) rowConstructor1;
+ Blackboard tmpBb = createBlackboard(bb.scope, null, false);
+ replaceSubqueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ final List<Pair<RexNode, String>> exps = new ArrayList<>();
+ for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
+ exps.add(Pair.of(tmpBb.convertExpression(operand.e), validator.deriveAlias(operand.e, operand.i)));
+ }
+ RelNode in = (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root;
+ unionRels.add(RelOptUtil.createProject(in, Pair.left(exps), Pair.right(exps), true));
+ }
+
+ if (unionRels.size() == 0) {
+ throw Util.newInternal("empty values clause");
+ } else if (unionRels.size() == 1) {
+ bb.setRoot(unionRels.get(0), true);
+ } else {
+ bb.setRoot(LogicalUnion.create(unionRels, true), true);
+ }
+
+ // REVIEW jvs 22-Jan-2004: should I add
+ // mapScopeToLux.put(validator.getScope(values),bb.root);
+ // ?
+ }
+
+ /**
+ * Interface to define the configuration for a SqlToRelConverter.
+ * Provides methods to set each configuration option.
+ *
+ * @see ConfigBuilder
+ * @see SqlToRelConverter#configBuilder()
+ */
+ public interface Config {
+ /** Default configuration. */
+ Config DEFAULT = configBuilder().build();
+
+ /** Returns the {@code convertTableAccess} option. Controls whether table
+ * access references are converted to physical rels immediately. The
+ * optimizer doesn't like leaf rels to have {@link Convention#NONE}.
+ * However, if we are doing further conversion passes (e.g.
+ * {@link RelStructuredTypeFlattener}), then we may need to defer
+ * conversion. */
+ boolean isConvertTableAccess();
+
+ /** Returns the {@code decorrelationEnabled} option. Controls whether to
+ * disable subquery decorrelation when needed. e.g. if outer joins are not
+ * supported. */
+ boolean isDecorrelationEnabled();
+
+ /** Returns the {@code trimUnusedFields} option. Controls whether to trim
+ * unused fields as part of the conversion process. */
+ boolean isTrimUnusedFields();
+
+ /** Returns the {@code createValuesRel} option. Controls whether instances
+ * of {@link org.apache.calcite.rel.logical.LogicalValues} are generated.
+ * These may not be supported by all physical implementations. */
+ boolean isCreateValuesRel();
+
+ /** Returns the {@code explain} option. Describes whether the current
+ * statement is part of an EXPLAIN PLAN statement. */
+ boolean isExplain();
+
+ /** Returns the {@code expand} option. Controls whether to expand
+ * sub-queries. If false, each sub-query becomes a
+ * {@link org.apache.calcite.rex.RexSubQuery}. */
+ boolean isExpand();
+
+ /** Returns the {@code inSubqueryThreshold} option,
+ * default {@link #DEFAULT_IN_SUBQUERY_THRESHOLD}. Controls the list size
+ * threshold under which {@link #convertInToOr} is used. Lists of this size
+ * or greater will instead be converted to use a join against an inline
+ * table ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than
+ * a predicate. A threshold of 0 forces usage of an inline table in all
+ * cases; a threshold of {@link Integer#MAX_VALUE} forces usage of OR in all
+ * cases. */
+ int getInSubqueryThreshold();
+ }
+
+ /** Deferred lookup. */
+ private static class DeferredLookup {
+ Blackboard bb;
+ String originalRelName;
+
+ DeferredLookup(Blackboard bb, String originalRelName) {
+ this.bb = bb;
+ this.originalRelName = originalRelName;
+ }
+
+ public RexFieldAccess getFieldAccess(CorrelationId name) {
+ return (RexFieldAccess) bb.mapCorrelateToRex.get(name);
+ }
+
+ public String getOriginalRelName() {
+ return originalRelName;
+ }
+ }
+
+ /**
+ * Context to find a relational expression to a field offset.
+ */
+ private static class LookupContext {
+ private final List<Pair<RelNode, Integer>> relOffsetList = new ArrayList<>();
+
+ /**
+ * Creates a LookupContext with multiple input relational expressions.
+ *
+ * @param bb Context for translating this subquery
+ * @param rels Relational expressions
+ * @param systemFieldCount Number of system fields
+ */
+ LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
+ bb.flatten(rels, systemFieldCount, new int[] { 0 }, relOffsetList);
+ }
+
+ /**
+ * Returns the relational expression with a given offset, and the
+ * ordinal in the combined row of its first field.
+ *
+ * <p>For example, in {@code Emp JOIN Dept}, findRel(1) returns the
+ * relational expression for {@code Dept} and offset 6 (because
+ * {@code Emp} has 6 fields, therefore the first field of {@code Dept}
+ * is field 6.
+ *
+ * @param offset Offset of relational expression in FROM clause
+ * @return Relational expression and the ordinal of its first field
+ */
+ Pair<RelNode, Integer> findRel(int offset) {
+ return relOffsetList.get(offset);
+ }
+ }
+
+ /** A sub-query, whether it needs to be translated using 2- or 3-valued
+ * logic. */
+ private static class SubQuery {
+ final SqlNode node;
+ final RelOptUtil.Logic logic;
+ RexNode expr;
+
+ private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
+ this.node = node;
+ this.logic = logic;
+ }
+ }
+
+ /**
+ * Visitor that collects all aggregate functions in a {@link SqlNode} tree.
+ */
+ private static class AggregateFinder extends SqlBasicVisitor<Void> {
+ final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
+
+ @Override
+ public Void visit(SqlCall call) {
+ // ignore window aggregates and ranking functions (associated with OVER operator)
+ if (call.getOperator().getKind() == SqlKind.OVER) {
+ return null;
+ }
+ if (call.getOperator().isAggregator()) {
+ list.add(call);
+ return null;
+ }
+
+ // Don't traverse into sub-queries, even if they contain aggregate
+ // functions.
+ if (call instanceof SqlSelect) {
+ return null;
+ }
+
+ return call.getOperator().acceptCall(this, call);
+ }
+ }
+
+ /** Use of a row as a correlating variable by a given relational
+ * expression. */
+ private static class CorrelationUse {
+ private final CorrelationId id;
+ private final ImmutableBitSet requiredColumns;
+ private final RelNode r;
+
+ CorrelationUse(CorrelationId id, ImmutableBitSet requiredColumns, RelNode r) {
+ this.id = id;
+ this.requiredColumns = requiredColumns;
+ this.r = r;
+ }
+ }
+
+ /** Builder for a {@link Config}. */
+ public static class ConfigBuilder {
+ private boolean convertTableAccess = true;
+ private boolean decorrelationEnabled = true;
+ private boolean trimUnusedFields = false;
+ private boolean createValuesRel = true;
+ private boolean explain;
+ private boolean expand = true;
+ private int inSubqueryThreshold = DEFAULT_IN_SUBQUERY_THRESHOLD;
+
+ private ConfigBuilder() {
+ }
+
+ /** Sets configuration identical to a given {@link Config}. */
+ public ConfigBuilder withConfig(Config config) {
+ this.convertTableAccess = config.isConvertTableAccess();
+ this.decorrelationEnabled = config.isDecorrelationEnabled();
+ this.trimUnusedFields = config.isTrimUnusedFields();
+ this.createValuesRel = config.isCreateValuesRel();
+ this.explain = config.isExplain();
+ this.expand = config.isExpand();
+ this.inSubqueryThreshold = config.getInSubqueryThreshold();
+ return this;
+ }
+
+ public ConfigBuilder withConvertTableAccess(boolean convertTableAccess) {
+ this.convertTableAccess = convertTableAccess;
+ return this;
+ }
+
+ public ConfigBuilder withDecorrelationEnabled(boolean enabled) {
+ this.decorrelationEnabled = enabled;
+ return this;
+ }
+
+ public ConfigBuilder withTrimUnusedFields(boolean trimUnusedFields) {
+ this.trimUnusedFields = trimUnusedFields;
+ return this;
+ }
- // Project select clause.
- int i = -1;
- for (SqlNode expr : selectList) {
- ++i;
- exprs.add(bb.convertExpression(expr));
- fieldNames.add(deriveAlias(expr, aliases, i));
+ public ConfigBuilder withCreateValuesRel(boolean createValuesRel) {
+ this.createValuesRel = createValuesRel;
+ return this;
}
- // Project extra fields for sorting.
- for (SqlNode expr : orderList) {
- ++i;
- SqlNode expr2 = validator.expandOrderExpr(select, expr);
- exprs.add(bb.convertExpression(expr2));
- fieldNames.add(deriveAlias(expr, aliases, i));
+ public ConfigBuilder withExplain(boolean explain) {
+ this.explain = explain;
+ return this;
}
- fieldNames = SqlValidatorUtil.uniquify(fieldNames);
+ public ConfigBuilder withExpand(boolean expand) {
+ this.expand = expand;
+ return this;
+ }
- bb.setRoot(RelOptUtil.createProject(bb.root, exprs, fieldNames), false);
+ public ConfigBuilder withInSubqueryThreshold(int inSubqueryThreshold) {
+ this.inSubqueryThreshold = inSubqueryThreshold;
+ return this;
+ }
- assert bb.columnMonotonicities.isEmpty();
- bb.columnMonotonicities.addAll(columnMonotonicityList);
- for (SqlNode selectItem : selectList) {
- bb.columnMonotonicities.add(selectItem.getMonotonicity(bb.scope));
+ /** Builds a {@link Config}. */
+ public Config build() {
+ return new ConfigImpl(convertTableAccess, decorrelationEnabled, trimUnusedFields, createValuesRel, explain, expand, inSubqueryThreshold);
}
}
- /**
- * Adds extra select items. The default implementation adds nothing; derived
- * classes may add columns to exprList, nameList, aliasList and
- * columnMonotonicityList.
- *
- * @param bb Blackboard
- * @param select Select statement being translated
- * @param exprList List of expressions in select clause
- * @param nameList List of names, one per column
- * @param aliasList Collection of aliases that have been used
- * already
- * @param columnMonotonicityList List of monotonicity, one per column
- */
- protected void extraSelectItems(Blackboard bb, SqlSelect select, List<RexNode> exprList, List<String> nameList, Collection<String> aliasList, List<SqlMonotonicity> columnMonotonicityList) {
- }
+ /** Implementation of {@link Config}.
+ * Called by builder; all values are in private final fields. */
+ private static class ConfigImpl implements Config {
+ private final boolean convertTableAccess;
+ private final boolean decorrelationEnabled;
+ private final boolean trimUnusedFields;
+ private final boolean createValuesRel;
+ private final boolean explain;
+ private final int inSubqueryThreshold;
+ private final boolean expand;
- private String deriveAlias(final SqlNode node, Collection<String> aliases, final int ordinal) {
- String alias = validator.deriveAlias(node, ordinal);
- if ((alias == null) || aliases.contains(alias)) {
- String aliasBase = (alias == null) ? "EXPR$" : alias;
- for (int j = 0;; j++) {
- alias = aliasBase + j;
- if (!aliases.contains(alias)) {
- break;
- }
- }
+ private ConfigImpl(boolean convertTableAccess, boolean decorrelationEnabled, boolean trimUnusedFields, boolean createValuesRel, boolean explain, boolean expand, int inSubqueryThreshold) {
+ this.convertTableAccess = convertTableAccess;
+ this.decorrelationEnabled = decorrelationEnabled;
+ this.trimUnusedFields = trimUnusedFields;
+ this.createValuesRel = createValuesRel;
+ this.explain = explain;
+ this.expand = expand;
+ this.inSubqueryThreshold = inSubqueryThreshold;
}
- aliases.add(alias);
- return alias;
- }
- /**
- * Converts a WITH sub-query into a relational expression.
- */
- public RelRoot convertWith(SqlWith with, boolean top) {
- return convertQuery(with.body, false, top);
- }
+ public boolean isConvertTableAccess() {
+ return convertTableAccess;
+ }
- /**
- * Converts a SELECT statement's parse tree into a relational expression.
- */
- public RelNode convertValues(SqlCall values, RelDataType targetRowType) {
- final SqlValidatorScope scope = validator.getOverScope(values);
- assert scope != null;
- final Blackboard bb = createBlackboard(scope, null, false);
- convertValuesImpl(bb, values, targetRowType);
- return bb.root;
- }
+ public boolean isDecorrelationEnabled() {
+ return decorrelationEnabled;
+ }
- /**
- * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a
- * relational expression.
- *
- * @param bb Blackboard
- * @param values Call to SQL VALUES operator
- * @param targetRowType Target row type
- */
- private void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType targetRowType) {
- // Attempt direct conversion to LogicalValues; if that fails, deal with
- // fancy stuff like subqueries below.
- RelNode valuesRel = convertRowValues(bb, values, values.getOperandList(), true, targetRowType);
- if (valuesRel != null) {
- bb.setRoot(valuesRel, true);
- return;
+ public boolean isTrimUnusedFields() {
+ return trimUnusedFields;
}
- final List<RelNode> unionRels = new ArrayList<>();
- for (SqlNode rowConstructor1 : values.getOperandList()) {
- SqlCall rowConstructor = (SqlCall) rowConstructor1;
- Blackboard tmpBb = createBlackboard(bb.scope, null, false);
- replaceSubqueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
- final List<Pair<RexNode, String>> exps = new ArrayList<>();
- for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
- exps.add(Pair.of(tmpBb.convertExpression(operand.e), validator.deriveAlias(operand.e, operand.i)));
- }
- RelNode in = (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root;
- unionRels.add(RelOptUtil.createProject(in, Pair.left(exps), Pair.right(exps), true));
+ public boolean isCreateValuesRel() {
+ return createValuesRel;
}
- if (unionRels.size() == 0) {
- throw Util.newInternal("empty values clause");
- } else if (unionRels.size() == 1) {
- bb.setRoot(unionRels.get(0), true);
- } else {
- bb.setRoot(LogicalUnion.create(unionRels, true), true);
+ public boolean isExplain() {
+ return explain;
}
- // REVIEW jvs 22-Jan-2004: should I add
- // mapScopeToLux.put(validator.getScope(values),bb.root);
- // ?
- }
+ public boolean isExpand() {
+ return expand;
+ }
- //~ Inner Classes ----------------------------------------------------------
+ public int getInSubqueryThreshold() {
+ return inSubqueryThreshold;
+ }
+ }
/**
* Workspace for translating an individual SELECT statement (or sub-SELECT).
@@ -3100,43 +3337,35 @@ public class SqlToRelConverter {
* statement.
*/
public final SqlValidatorScope scope;
+ final List<RelNode> cursors = new ArrayList<>();
+ final boolean top;
private final Map<String, RexNode> nameToNodeMap;
- public RelNode root;
- private List<RelNode> inputs;
private final Map<CorrelationId, RexFieldAccess> mapCorrelateToRex = new HashMap<>();
-
- final List<RelNode> cursors = new ArrayList<>();
-
/**
* List of <code>IN</code> and <code>EXISTS</code> nodes inside this
* <code>SELECT</code> statement (but not inside sub-queries).
*/
private final Set<SubQuery> subqueryList = Sets.newLinkedHashSet();
-
- private boolean subqueryNeedsOuterJoin;
-
+ /**
+ * Project the groupby expressions out of the root of this sub-select.
+ * Subqueries can reference group by expressions projected from the
+ * "right" to the subquery.
+ */
+ private final Map<RelNode, Map<Integer, Integer>> mapRootRelToFieldProjection = new HashMap<>();
+ private final List<SqlMonotonicity> columnMonotonicities = new ArrayList<>();
+ private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
+ public RelNode root;
/**
* Workspace for building aggregates.
*/
AggConverter agg;
-
/**
* When converting window aggregate, we need to know if the window is
* guaranteed to be non-empty.
*/
SqlWindow window;
-
- /**
- * Project the groupby expressions out of the root of this sub-select.
- * Subqueries can reference group by expressions projected from the
- * "right" to the subquery.
- */
- private final Map<RelNode, Map<Integer, Integer>> mapRootRelToFieldProjection = new HashMap<>();
-
- private final List<SqlMonotonicity> columnMonotonicities = new ArrayList<>();
-
- private final List<RelDataTypeField> systemFieldList = new ArrayList<>();
- final boolean top;
+ private List<RelNode> inputs;
+ private boolean subqueryNeedsOuterJoin;
/**
* Creates a Blackboard.
@@ -3302,22 +3531,22 @@ public class SqlToRelConverter {
}
return Pair.of(node, null);
}
- int[] offsets = { -1 };
- final SqlValidatorScope[] ancestorScopes = { null };
- SqlValidatorNamespace foundNs = scope.resolve(qualified.prefix(), ancestorScopes, offsets);
- if (foundNs == null) {
+ final SqlValidatorScope.ResolvedImpl resolved = new SqlValidatorScope.ResolvedImpl();
+ scope.resolve(qualified.prefix(), false, resolved);
+ if (!(resolved.count() == 1)) {
return null;
}
+ final SqlValidatorScope.Resolve resolve = resolved.only();
+ final SqlValidatorNamespace foundNs = resolve.namespace;
// Found in current query's from list. Find which from item.
// We assume that the order of the from clause items has been
// preserved.
- SqlValidatorScope ancestorScope = ancestorScopes[0];
+ final SqlValidatorScope ancestorScope = resolve.scope;
boolean isParent = ancestorScope != scope;
if ((inputs != null) && !isParent) {
- int offset = offsets[0];
final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size());
- final RexNode node = lookup(offset, rels);
+ final RexNode node = lookup(resolve.path.steps().get(0).i, rels);
if (node == null) {
return null;
} else {
@@ -3331,17 +3560,17 @@ public class SqlToRelConverter {
DeferredLookup lookup = new DeferredLookup(this, qualified.identifier.names.get(0));
final CorrelationId correlId = cluster.createCorrel();
mapCorrelToDeferred.put(correlId, lookup);
- if (offsets[0] < 0) {
+ if (resolve.path.steps().get(0).i < 0) {
return Pair.of(rexBuilder.makeCorrel(foundNs.getRowType(), correlId), null);
} else {
final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
- final ListScope ancestorScope1 = (ListScope) ancestorScopes[0];
+ final ListScope ancestorScope1 = (ListScope) resolve.scope;
final ImmutableMap.Builder<String, Integer> fields = ImmutableMap.builder();
int i = 0;
int offset = 0;
for (SqlValidatorNamespace c : ancestorScope1.getChildren()) {
builder.addAll(c.getRowType().getFieldList());
- if (i == offsets[0]) {
+ if (i == resolve.path.steps().get(0).i) {
for (RelDataTypeField field : c.getRowType().getFieldList()) {
fields.put(field.getName(), field.getIndex() + offset);
}
@@ -3450,7 +3679,7 @@ public class SqlToRelConverter {
// expressions.
final SqlKind kind = expr.getKind();
final SubQuery subQuery;
- if (!expand) {
+ if (!config.isExpand()) {
final SqlCall call;
final SqlNode query;
final RelRoot root;
@@ -3680,25 +3909,6 @@ public class SqlToRelConverter {
}
- /** Deferred lookup. */
- private static class DeferredLookup {
- Blackboard bb;
- String originalRelName;
-
- DeferredLookup(Blackboard bb, String originalRelName) {
- this.bb = bb;
- this.originalRelName = originalRelName;
- }
-
- public RexFieldAccess getFieldAccess(CorrelationId name) {
- return (RexFieldAccess) bb.mapCorrelateToRex.get(name);
- }
-
- public String getOriginalRelName() {
- return originalRelName;
- }
- }
-
/**
* An implementation of DefaultValueFactory which always supplies NULL.
*/
@@ -3751,9 +3961,8 @@ public class SqlToRelConverter {
* </ul>
*/
protected class AggConverter implements SqlVisitor<Void> {
- private final Blackboard bb;
public final AggregatingSelectScope aggregatingSelectScope;
-
+ private final Blackboard bb;
private final Map<String, String> nameMap = Maps.newHashMap();
/**
@@ -3887,9 +4096,6 @@ public class SqlToRelConverter {
// Ignore window aggregates and ranking functions (associated with OVER
// operator). However, do not ignore nested window aggregates.
if (call.getOperator().getKind() == SqlKind.OVER) {
- if (call.operand(0).getKind() == SqlKind.RANK) {
- return null;
- }
// Track aggregate nesting levels only within an OVER operator.
inOver = true;
}
@@ -4090,40 +4296,6 @@ public class SqlToRelConverter {
}
/**
- * Context to find a relational expression to a field offset.
- */
- private static class LookupContext {
- private final List<Pair<RelNode, Integer>> relOffsetList = new ArrayList<>();
-
- /**
- * Creates a LookupContext with multiple input relational expressions.
- *
- * @param bb Context for translating this subquery
- * @param rels Relational expressions
- * @param systemFieldCount Number of system fields
- */
- LookupContext(Blackboard bb, List<RelNode> rels, int systemFieldCount) {
- bb.flatten(rels, systemFieldCount, new int[] { 0 }, relOffsetList);
- }
-
- /**
- * Returns the relational expression with a given offset, and the
- * ordinal in the combined row of its first field.
- *
- * <p>For example, in {@code Emp JOIN Dept}, findRel(1) returns the
- * relational expression for {@code Dept} and offset 6 (because
- * {@code Emp} has 6 fields, therefore the first field of {@code Dept}
- * is field 6.
- *
- * @param offset Offset of relational expression in FROM clause
- * @return Relational expression and the ordinal of its first field
- */
- Pair<RelNode, Integer> findRel(int offset) {
- return relOffsetList.get(offset);
- }
- }
-
- /**
* Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to
* all agg functions.
*
@@ -4260,60 +4432,6 @@ public class SqlToRelConverter {
}
}
}
-
- /** A sub-query, whether it needs to be translated using 2- or 3-valued
- * logic. */
- private static class SubQuery {
- final SqlNode node;
- final RelOptUtil.Logic logic;
- RexNode expr;
-
- private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
- this.node = node;
- this.logic = logic;
- }
- }
-
- /**
- * Visitor that collects all aggregate functions in a {@link SqlNode} tree.
- */
- private static class AggregateFinder extends SqlBasicVisitor<Void> {
- final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO);
-
- @Override
- public Void visit(SqlCall call) {
- // ignore window aggregates and ranking functions (associated with OVER operator)
- if (call.getOperator().getKind() == SqlKind.OVER) {
- return null;
- }
- if (call.getOperator().isAggregator()) {
- list.add(call);
- return null;
- }
-
- // Don't traverse into sub-queries, even if they contain aggregate
- // functions.
- if (call instanceof SqlSelect) {
- return null;
- }
-
- return call.getOperator().acceptCall(this, call);
- }
- }
-
- /** Use of a row as a correlating variable by a given relational
- * expression. */
- private static class CorrelationUse {
- private final CorrelationId id;
- private final ImmutableBitSet requiredColumns;
- private final RelNode r;
-
- CorrelationUse(CorrelationId id, ImmutableBitSet requiredColumns, RelNode r) {
- this.id = id;
- this.requiredColumns = requiredColumns;
- this.r = r;
- }
- }
}
-// End SqlToRelConverter.java
+// End SqlToRelConverter.java
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3bb878f/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3bb878f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c837b10..2ff4c54 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,7 +112,8 @@
<aspectj.version>1.8.9</aspectj.version>
<!-- Calcite Version -->
- <calcite.version>1.8.0</calcite.version>
+ <calcite.version>1.10.0</calcite.version>
+ <avatica.version>1.8.0</avatica.version>
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
@@ -456,7 +457,7 @@
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
- <version>${calcite.version}</version>
+ <version>${avatica.version}</version>
</dependency>
<!-- Workaround for hive 0.14 avatica dependency -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3bb878f/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index b157f34..452170a 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -45,6 +45,7 @@ import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rel.rules.AggregateUnionTransposeRule;
+import org.apache.calcite.rel.rules.DateRangeRules;
import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.rules.JoinCommuteRule;
@@ -172,6 +173,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
planner.removeRule(JoinUnionTransposeRule.LEFT_UNION);
planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
+ planner.removeRule(DateRangeRules.FILTER_INSTANCE);
// distinct count will be split into a separated query that is joined with the left query
planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);