You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vo...@apache.org on 2019/07/05 15:54:03 UTC
[drill] branch master updated: DRILL-7200: Update Calcite to 1.19.0
/ 1.20.0
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 2efa51e DRILL-7200: Update Calcite to 1.19.0 / 1.20.0
2efa51e is described below
commit 2efa51e10e832512feae313c4226c80393dcd842
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Fri Feb 22 18:01:15 2019 +0200
DRILL-7200: Update Calcite to 1.19.0 / 1.20.0
---
common/pom.xml | 5 --
.../store/mapr/db/json/JsonConditionBuilder.java | 10 ++++
.../apache/drill/hbase/HBaseRecordReaderTest.java | 6 +-
.../org/apache/drill/hbase/TestTableGenerator.java | 14 ++---
contrib/storage-hive/core/pom.xml | 2 +-
.../drill/exec/fn/hive/TestInbuiltHiveUDFs.java | 3 +-
docs/dev/Calcite.md | 3 +-
exec/java-exec/pom.xml | 4 +-
exec/java-exec/src/main/codegen/data/Parser.tdd | 8 ++-
.../org/apache/drill/exec/expr/FilterBuilder.java | 9 ++-
.../org/apache/drill/exec/expr/IsPredicate.java | 1 +
.../exec/expr/fn/FunctionGenerationHelper.java | 1 +
.../drill/exec/expr/fn/impl/LastDayFunction.java | 48 ++++++++++++++++
.../exec/planner/common/DrillAggregateRelBase.java | 7 +--
.../planner/common/DrillLateralJoinRelBase.java | 6 +-
.../cost/DrillDefaultRelMetadataProvider.java | 1 +
.../exec/planner/cost/DrillRelMdMaxRowCount.java | 48 ++++++++++++++++
.../exec/planner/logical/DrillAggregateRel.java | 12 ++--
.../exec/planner/logical/DrillAggregateRule.java | 2 +-
.../drill/exec/planner/logical/DrillJoinRel.java | 5 --
.../drill/exec/planner/logical/DrillJoinRule.java | 6 ++
.../exec/planner/logical/DrillLateralJoinRel.java | 17 ++++--
.../DrillProjectPushIntoLateralJoinRule.java | 4 +-
.../planner/logical/DrillReduceAggregatesRule.java | 14 +----
.../exec/planner/logical/DrillRelFactories.java | 4 +-
.../drill/exec/planner/logical/DrillScanRule.java | 2 +-
.../exec/planner/logical/DrillSemiJoinRel.java | 30 +++++-----
.../planner/logical/partition/PruneScanRule.java | 1 -
.../drill/exec/planner/physical/AggPrelBase.java | 3 +-
.../drill/exec/planner/physical/HashAggPrel.java | 8 +--
.../drill/exec/planner/physical/HashAggPrule.java | 3 -
.../drill/exec/planner/physical/JoinPrel.java | 22 +++++++-
.../exec/planner/physical/LateralJoinPrel.java | 10 ++--
.../drill/exec/planner/physical/StreamAggPrel.java | 9 +--
.../exec/planner/physical/StreamAggPrule.java | 5 --
.../sql/DrillCalciteSqlFunctionWrapper.java | 6 ++
.../drill/exec/planner/sql/DrillOperatorTable.java | 15 ++---
.../drill/exec/planner/sql/SqlConverter.java | 28 ++++++++-
.../planner/sql/handlers/DefaultSqlHandler.java | 6 +-
.../exec/planner/sql/handlers/ViewHandler.java | 3 +-
.../sql/parser/UnsupportedOperatorsVisitor.java | 66 +++++++++++-----------
.../java/org/apache/drill/exec/util/Utilities.java | 9 +++
.../impl/limit/TestLateLimit0Optimization.java | 2 +-
.../org/apache/drill/exec/sql/TestViewSupport.java | 24 ++++++++
exec/jdbc-all/pom.xml | 8 +--
logical/pom.xml | 2 +-
pom.xml | 13 ++++-
47 files changed, 351 insertions(+), 164 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index 8c9593c..bc3a9aa 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -46,11 +46,6 @@
</dependency>
<dependency>
- <groupId>com.github.vvysotskyi.drill-calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
index ab3a774..dc531ac 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.mapr.db.json;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
import org.ojai.Value;
@@ -57,6 +58,15 @@ public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void
}
@Override
+ public JsonScanSpec visitSchemaPath(SchemaPath path, Void value) throws RuntimeException {
+ String fieldPath = FieldPathHelper.schemaPath2FieldPath(path).asPathString();
+ QueryCondition cond = MapRDBImpl.newCondition().is(fieldPath, Op.EQUAL, true);
+ return new JsonScanSpec(groupScan.getTableName(),
+ groupScan.getIndexDesc(),
+ cond.build());
+ }
+
+ @Override
public JsonScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
allExpressionsConverted = false;
return null;
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
index 6820c79..d183a70 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
@@ -81,15 +81,15 @@ public class HBaseRecordReaderTest extends BaseHBaseTest {
.baselineValues(
"a1".getBytes(),
mapOf("c3", "23".getBytes()),
- mapOf("c3", "23".getBytes()))
+ mapOf("c1", "21".getBytes(), "c2", "22".getBytes()))
.baselineValues(
"a2".getBytes(),
mapOf("c3", "13".getBytes()),
- mapOf("c3", "13".getBytes()))
+ mapOf("c1", "11".getBytes(), "c2", "12".getBytes()))
.baselineValues(
"a3".getBytes(),
mapOf("c3", "33".getBytes()),
- mapOf("c3", "33".getBytes()))
+ mapOf("c1", "31".getBytes(), "c2", "32".getBytes()))
.go();
}
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
index 3c650db..5e14e09 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
@@ -177,7 +177,7 @@ public class TestTableGenerator {
}
HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(new HColumnDescriptor("f"));
+ desc.addFamily(new HColumnDescriptor("f0"));
desc.addFamily(new HColumnDescriptor("F"));
if (numberRegions > 1) {
admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions - 1));
@@ -188,20 +188,20 @@ public class TestTableGenerator {
BufferedMutator table = conn.getBufferedMutator(tableName);
Put p = new Put("a1".getBytes());
- p.addColumn("f".getBytes(), "c1".getBytes(), "21".getBytes());
- p.addColumn("f".getBytes(), "c2".getBytes(), "22".getBytes());
+ p.addColumn("f0".getBytes(), "c1".getBytes(), "21".getBytes());
+ p.addColumn("f0".getBytes(), "c2".getBytes(), "22".getBytes());
p.addColumn("F".getBytes(), "c3".getBytes(), "23".getBytes());
table.mutate(p);
p = new Put("a2".getBytes());
- p.addColumn("f".getBytes(), "c1".getBytes(), "11".getBytes());
- p.addColumn("f".getBytes(), "c2".getBytes(), "12".getBytes());
+ p.addColumn("f0".getBytes(), "c1".getBytes(), "11".getBytes());
+ p.addColumn("f0".getBytes(), "c2".getBytes(), "12".getBytes());
p.addColumn("F".getBytes(), "c3".getBytes(), "13".getBytes());
table.mutate(p);
p = new Put("a3".getBytes());
- p.addColumn("f".getBytes(), "c1".getBytes(), "31".getBytes());
- p.addColumn("f".getBytes(), "c2".getBytes(), "32".getBytes());
+ p.addColumn("f0".getBytes(), "c1".getBytes(), "31".getBytes());
+ p.addColumn("f0".getBytes(), "c2".getBytes(), "32".getBytes());
p.addColumn("F".getBytes(), "c3".getBytes(), "33".getBytes());
table.mutate(p);
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index 88003ae..3a9d766 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -96,7 +96,7 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.github.vvysotskyi.drill-calcite</groupId>
+ <groupId>${calcite.groupId}</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
<dependency>
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
index 86d11f2..dae2b81 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.fn.hive;
+import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.List;
@@ -149,7 +150,7 @@ public class TestInbuiltHiveUDFs extends HiveTestBase {
.sqlQuery("select last_day(to_date('1994-02-01','yyyy-MM-dd')) as `LAST_DAY` from (VALUES(1))")
.unOrdered()
.baselineColumns("LAST_DAY")
- .baselineValues("1994-02-28")
+ .baselineValues(LocalDate.parse("1994-02-28"))
.go();
}
diff --git a/docs/dev/Calcite.md b/docs/dev/Calcite.md
index 39d5fe4..247fcc6 100644
--- a/docs/dev/Calcite.md
+++ b/docs/dev/Calcite.md
@@ -1,7 +1,7 @@
# Drill-specific commits in Apache Calcite
Currently, Drill uses Apache Calcite with additional changes, required for Drill. All the commits were left after
-update from Calcite `1.4.0` to Calcite `1.15.0` and weren't merged to the Calcite's master yet since there is no consensus on them in Calcite community.
+update from Calcite `1.4.0` to Calcite `1.15.0` (3 commits) and from Calcite `1.18.0` to Calcite `1.20.0` (1 commit) and weren't merged to the Calcite's master yet since there is no consensus on them in Calcite community.
List of Jiras with Drill-specific commits:
@@ -10,6 +10,7 @@ List of Jiras with Drill-specific commits:
|[CALCITE-2018](https://issues.apache.org/jira/browse/CALCITE-2018)|Queries failed with AssertionError: rel has lower cost than best cost of subset|Pull request with the fix was created ([PR-552](https://github.com/apache/calcite/pull/552)), but [CALCITE-2166](https://issues.apache.org/jira/browse/CALCITE-2166) which blocks it was found and is not resolved yet.|
|[CALCITE-2087](https://issues.apache.org/jira/browse/CALCITE-2087)|Add new method to ViewExpander interface to allow passing SchemaPlus.|Pull request into Apache Calcite was created, but it was declined. See conversation in Jira.|
|[CALCITE-1178](https://issues.apache.org/jira/browse/CALCITE-1178)|Allow SqlBetweenOperator to compare DATE and TIMESTAMP|SQL spec does not allow to compare datetime types if they have different `<primary datetime field>`s. Therefore Calcite community won’t accept these changes. Similar issues were reported in [CALCITE-2829](https://issues.apache.org/jira/browse/CALCITE-2829) and in [CALCITE-2745](https://issues.apache.org/jira/browse/CALCITE-2745).|
+|[CALCITE-3121](https://issues.apache.org/jira/browse/CALCITE-3121)|VolcanoPlanner hangs due to removing ORDER BY from sub-query|Pull request was open to revert changes ([PR-1264](https://github.com/apache/calcite/pull/1264)) which remove ORDER BY clause; it wasn't merged, because aforementioned changes only unveiled the issue and no proper solution is available yet.|
# Drill-Calcite repository
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index b2a1899..4d19395 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -207,7 +207,7 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.github.vvysotskyi.drill-calcite</groupId>
+ <groupId>${calcite.groupId}</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
<dependency>
@@ -673,7 +673,7 @@
<configuration>
<artifactItems>
<artifactItem>
- <groupId>com.github.vvysotskyi.drill-calcite</groupId>
+ <groupId>${calcite.groupId}</groupId>
<artifactId>calcite-core</artifactId>
<type>jar</type>
<overWrite>true</overWrite>
diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd
index df97e24..2da27c3 100644
--- a/exec/java-exec/src/main/codegen/data/Parser.tdd
+++ b/exec/java-exec/src/main/codegen/data/Parser.tdd
@@ -866,7 +866,13 @@
# Example: LeftSemiJoin()
joinTypes: [
]
-
+
+ # List of methods for parsing builtin function calls.
+ # Return type of method implementation should be "SqlNode".
+ # Example: DateFunctionCall().
+ builtinFunctionCallMethods: [
+ ]
+
includeCompoundIdentifier: false,
includeBraces: true,
includeAdditionalDeclarations: false,
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java
index 6626cce..08205be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java
@@ -246,7 +246,7 @@ public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<Lo
return handleCompareFunction(funcHolderExpr, value);
}
- if (isIsFunction(funcName)) {
+ if (isIsFunction(funcName) || isNot(funcHolderExpr, funcName)) {
return handleIsFunction(funcHolderExpr, value);
}
@@ -262,6 +262,13 @@ public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<Lo
}
}
+ // shows whether function is simplified IS FALSE
+ private boolean isNot(FunctionHolderExpression holderExpression, String funcName) {
+ return !holderExpression.args.isEmpty()
+ && !(holderExpression.args.get(0) instanceof DrillFuncHolderExpr)
+ && FunctionGenerationHelper.NOT.equals(funcName);
+ }
+
private List<LogicalExpression> generateNewExpressions(List<LogicalExpression> expressions, Set<LogicalExpression> value) {
List<LogicalExpression> newExpressions = new ArrayList<>();
for (LogicalExpression arg : expressions) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java
index 0a7667d..3d4c9bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java
@@ -230,6 +230,7 @@ public class IsPredicate<C extends Comparable<C>> extends LogicalExpressionBase
case FunctionGenerationHelper.IS_NOT_TRUE:
return createIsNotTruePredicate(expr);
case FunctionGenerationHelper.IS_FALSE:
+ case FunctionGenerationHelper.NOT:
return createIsFalsePredicate(expr);
case FunctionGenerationHelper.IS_NOT_FALSE:
return createIsNotFalsePredicate(expr);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
index 67466b5..2871173 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
@@ -53,6 +53,7 @@ public class FunctionGenerationHelper {
public static final String IS_NOT_TRUE = "isnottrue";
public static final String IS_FALSE = "isfalse";
public static final String IS_NOT_FALSE = "isnotfalse";
+ public static final String NOT = "not";
/**
* Finds ordering comparator ("compare_to...") FunctionHolderExpression with
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/LastDayFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/LastDayFunction.java
new file mode 100644
index 0000000..2663325
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/LastDayFunction.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+
+@SuppressWarnings("unused")
+@FunctionTemplate(names = "last_day", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+public class LastDayFunction implements DrillSimpleFunc {
+
+ @Param
+ TimeStampHolder in;
+ @Output
+ DateHolder out;
+
+ @Override
+ public void setup() {
+ }
+
+ @Override
+ public void eval() {
+ java.time.LocalDate date =
+ java.time.Instant.ofEpochMilli(in.value).atZone(java.time.ZoneOffset.UTC).toLocalDate();
+ java.time.LocalDate end = date.withDayOfMonth(date.lengthOfMonth());
+ out.value = end.atStartOfDay(java.time.ZoneOffset.UTC).toInstant().toEpochMilli();
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillAggregateRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillAggregateRelBase.java
index d791556..395158c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillAggregateRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillAggregateRelBase.java
@@ -34,15 +34,14 @@ import org.apache.drill.exec.planner.physical.PrelUtil;
import java.util.List;
-
/**
* Base class for logical and physical Aggregations implemented in Drill
*/
public abstract class DrillAggregateRelBase extends Aggregate implements DrillRelNode {
- public DrillAggregateRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode child, boolean indicator,
- ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
- super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls);
+ public DrillAggregateRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode child,
+ ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ super(cluster, traits, child, groupSet, groupSets, aggCalls);
}
/**
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java
index 69e34d7..f86e9ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.common;
+import org.apache.calcite.rel.core.JoinRelType;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.calcite.plan.RelOptCluster;
@@ -29,7 +30,6 @@ import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.calcite.sql.SemiJoinType;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.drill.exec.ExecConstants;
@@ -47,7 +47,7 @@ public abstract class DrillLateralJoinRelBase extends Correlate implements Drill
final private static double CORRELATE_MEM_COPY_COST = DrillCostBase.MEMORY_TO_CPU_RATIO * DrillCostBase.BASE_CPU_COST;
final public boolean excludeCorrelateColumn;
public DrillLateralJoinRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, boolean excludeCorrelateCol,
- CorrelationId correlationId, ImmutableBitSet requiredColumns, SemiJoinType semiJoinType) {
+ CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType semiJoinType) {
super(cluster, traits, left, right, correlationId, requiredColumns, semiJoinType);
this.excludeCorrelateColumn = excludeCorrelateCol;
}
@@ -73,7 +73,7 @@ public abstract class DrillLateralJoinRelBase extends Correlate implements Drill
case LEFT:
case INNER:
return constructRowType(SqlValidatorUtil.deriveJoinRowType(left.getRowType(),
- removeImplicitField(right.getRowType()), joinType.toJoinType(),
+ removeImplicitField(right.getRowType()), joinType,
getCluster().getTypeFactory(), null,
ImmutableList.of()));
case ANTI:
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillDefaultRelMetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillDefaultRelMetadataProvider.java
index a7d925b..d34647c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillDefaultRelMetadataProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillDefaultRelMetadataProvider.java
@@ -30,5 +30,6 @@ public class DrillDefaultRelMetadataProvider {
.of(DrillRelMdRowCount.SOURCE,
DrillRelMdDistinctRowCount.SOURCE,
DrillRelMdSelectivity.SOURCE,
+ DrillRelMdMaxRowCount.SOURCE,
DefaultRelMetadataProvider.INSTANCE));
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdMaxRowCount.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdMaxRowCount.java
new file mode 100644
index 0000000..ed96025
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/cost/DrillRelMdMaxRowCount.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.cost;
+
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
+import org.apache.calcite.rel.metadata.RelMdMaxRowCount;
+import org.apache.calcite.rel.metadata.RelMetadataProvider;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.util.BuiltInMethod;
+
+public class DrillRelMdMaxRowCount extends RelMdMaxRowCount {
+
+ private static final DrillRelMdMaxRowCount INSTANCE = new DrillRelMdMaxRowCount();
+
+ public static final RelMetadataProvider SOURCE =
+ ReflectiveRelMetadataProvider.reflectiveSource(BuiltInMethod.MAX_ROW_COUNT.method, INSTANCE);
+
+ // The method is overriden because of changes done in CALCITE-2991 and
+ // TODO: should be discarded when CALCITE-1048 is fixed.
+ @Override
+ public Double getMaxRowCount(Aggregate rel, RelMetadataQuery mq) {
+ if (rel.getGroupSet().isEmpty()) {
+ // Aggregate with no GROUP BY always returns 1 row (even on empty table).
+ return 1D;
+ }
+ final Double rowCount = mq.getMaxRowCount(rel.getInput());
+ if (rowCount == null) {
+ return null;
+ }
+ return rowCount * rel.getGroupSets().size();
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
index 6403ac1..80d6043 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
@@ -47,15 +47,15 @@ import java.util.List;
* Aggregation implemented in Drill.
*/
public class DrillAggregateRel extends DrillAggregateRelBase implements DrillRel {
- /** Creates a DrillAggregateRel. */
- public DrillAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, boolean indicator, ImmutableBitSet groupSet,
- List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
- super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls);
+
+ public DrillAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, ImmutableBitSet groupSet,
+ List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ super(cluster, traits, child, groupSet, groupSets, aggCalls);
}
@Override
- public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
- return new DrillAggregateRel(getCluster(), traitSet, input, indicator, groupSet, groupSets, aggCalls);
+ public Aggregate copy(RelTraitSet traitSet, RelNode input, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ return new DrillAggregateRel(getCluster(), traitSet, input, groupSet, groupSets, aggCalls);
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
index 75f806f..8ce07dd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
@@ -51,7 +51,7 @@ public class DrillAggregateRule extends RelOptRule {
final RelTraitSet traits = aggregate.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.DRILL_LOGICAL).simplify());
- call.transformTo(new DrillAggregateRel(aggregate.getCluster(), traits, convertedInput, aggregate.indicator,
+ call.transformTo(new DrillAggregateRel(aggregate.getCluster(), traits, convertedInput,
aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList()));
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java
index 724d5cb..2559d28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java
@@ -189,9 +189,4 @@ public class DrillJoinRel extends DrillJoinRelBase implements DrillRel {
inputs.left, inputs.right, rexCondition, join.getJoinType());
return joinRel;
}
-
- @Override
- public boolean isSemiJoin() {
- return false;
- }
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
index 543a71c..3ad824f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java
@@ -86,6 +86,12 @@ public class DrillJoinRule extends RelOptRule {
}
try {
+ if (join.isSemiJoin()) {
+ RelNode joinRel = new DrillSemiJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, newJoinCondition,
+ leftKeys, rightKeys);
+ call.transformTo(joinRel);
+ return;
+ }
if (!addFilter) {
RelNode joinRel = new DrillJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, newJoinCondition,
join.getJoinType(), leftKeys, rightKeys);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java
index ca03de1..559a2ee 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java
@@ -20,9 +20,10 @@ package org.apache.drill.exec.planner.logical;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.Correlate;
import org.apache.calcite.rel.core.CorrelationId;
-import org.apache.calcite.sql.SemiJoinType;
+import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.drill.common.logical.data.LateralJoin;
import org.apache.drill.common.logical.data.LogicalOperator;
@@ -31,23 +32,27 @@ import org.apache.drill.exec.planner.common.DrillLateralJoinRelBase;
import java.util.ArrayList;
import java.util.List;
-
public class DrillLateralJoinRel extends DrillLateralJoinRelBase implements DrillRel {
- protected DrillLateralJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, boolean includeCorrelateVar,
- CorrelationId correlationId, ImmutableBitSet requiredColumns, SemiJoinType semiJoinType) {
- super(cluster, traits, left, right, includeCorrelateVar, correlationId, requiredColumns, semiJoinType);
+ protected DrillLateralJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, boolean excludeCorrelateCol,
+ CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType semiJoinType) {
+ super(cluster, traits, left, right, excludeCorrelateCol, correlationId, requiredColumns, semiJoinType);
}
@Override
public Correlate copy(RelTraitSet traitSet,
RelNode left, RelNode right, CorrelationId correlationId,
- ImmutableBitSet requiredColumns, SemiJoinType joinType) {
+ ImmutableBitSet requiredColumns, JoinRelType joinType) {
return new DrillLateralJoinRel(this.getCluster(), this.getTraitSet(), left, right, this.excludeCorrelateColumn, correlationId, requiredColumns,
this.getJoinType());
}
@Override
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("exclude correlate column: ", excludeCorrelateColumn);
+ }
+
+ @Override
public LogicalOperator implement(DrillImplementor implementor) {
List<String> fields = new ArrayList<>();
fields.addAll(getInput(0).getRowType().getFieldNames());
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectPushIntoLateralJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectPushIntoLateralJoinRule.java
index 622eb05..d44b916 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectPushIntoLateralJoinRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillProjectPushIntoLateralJoinRule.java
@@ -32,7 +32,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
public class DrillProjectPushIntoLateralJoinRule extends RelOptRule {
public static final DrillProjectPushIntoLateralJoinRule INSTANCE =
@@ -72,11 +71,12 @@ public class DrillProjectPushIntoLateralJoinRule extends RelOptRule {
final RelNode convertedRight = convert(right, right.getTraitSet().plus(DrillRel.DRILL_LOGICAL).simplify());
final RelTraitSet traits = corr.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
+ boolean trivial = DrillRelOptUtil.isTrivialProject(origProj, true);
RelNode relNode = new DrillLateralJoinRel(corr.getCluster(),
traits, convertedLeft, convertedRight, true, corr.getCorrelationId(),
corr.getRequiredColumns(), corr.getJoinType());
- if (!DrillRelOptUtil.isTrivialProject(origProj, true)) {
+ if (!trivial) {
Map<Integer, Integer> mapWithoutCorr = buildMapWithoutCorrColumn(corr, correlationIndex);
List<RexNode> outputExprs = DrillRelOptUtil.transformExprs(origProj.getCluster().getRexBuilder(), origProj.getChildExps(), mapWithoutCorr);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java
index d0711cf..43574aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java
@@ -240,7 +240,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
return oldAggRel.getCluster().getRexBuilder().addAggCall(
oldCall,
oldAggRel.getGroupCount(),
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(getFieldType(
@@ -303,7 +302,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
return rexBuilder.addAggCall(
oldCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
oldArgTypes);
@@ -347,7 +345,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
sumCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(avgInputType));
@@ -356,7 +353,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
countCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(avgInputType));
@@ -386,7 +382,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
countCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(avgInputType));
@@ -449,7 +444,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
sumZeroCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(argType));
@@ -463,7 +457,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
countCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(argType));
@@ -535,7 +528,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
sumArgSquaredAggCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(argType));
@@ -554,7 +546,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
sumArgAggCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(argType));
@@ -571,7 +562,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
rexBuilder.addAggCall(
countArgAggCall,
nGroups,
- oldAggRel.indicator,
newCalls,
aggCallMapping,
ImmutableList.of(argType));
@@ -679,7 +669,7 @@ public class DrillReduceAggregatesRule extends RelOptRule {
List<AggregateCall> newCalls) {
RelOptCluster cluster = inputRel.getCluster();
return new LogicalAggregate(cluster, cluster.traitSetOf(Convention.NONE),
- inputRel, oldAggRel.indicator, oldAggRel.getGroupSet(), oldAggRel.getGroupSets(), newCalls);
+ inputRel, oldAggRel.getGroupSet(), oldAggRel.getGroupSets(), newCalls);
}
private RelDataType getFieldType(RelNode relNode, int i) {
@@ -730,7 +720,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
oldAggRel.getCluster().getRexBuilder()
.addAggCall(sumZeroCall,
oldAggRel.getGroupCount(),
- oldAggRel.indicator,
newAggregateCalls,
aggCallMapping,
ImmutableList.of(argType));
@@ -743,7 +732,6 @@ public class DrillReduceAggregatesRule extends RelOptRule {
oldAggRel.getCluster(),
oldAggRel.getTraitSet(),
oldAggRel.getInput(),
- oldAggRel.indicator,
oldAggRel.getGroupSet(),
oldAggRel.getGroupSets(),
newAggregateCalls));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java
index feccce0..bca08a0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRelFactories.java
@@ -144,9 +144,9 @@ public class DrillRelFactories {
private static class DrillAggregateFactoryImpl implements RelFactories.AggregateFactory {
@Override
- public RelNode createAggregate(RelNode input, boolean indicator, ImmutableBitSet groupSet,
+ public RelNode createAggregate(RelNode input, ImmutableBitSet groupSet,
com.google.common.collect.ImmutableList<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
- return new DrillAggregateRel(input.getCluster(), input.getTraitSet().plus(DRILL_LOGICAL), input, indicator, groupSet, groupSets, aggCalls);
+ return new DrillAggregateRel(input.getCluster(), input.getTraitSet().plus(DRILL_LOGICAL), input, groupSet, groupSets, aggCalls);
}
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java
index 722ea85..da9d707 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRule.java
@@ -28,7 +28,7 @@ public class DrillScanRule extends RelOptRule {
private DrillScanRule() {
super(RelOptHelper.any(EnumerableTableScan.class),
- DrillRelFactories.LOGICAL_BUILDER, "DrillTableRule");
+ DrillRelFactories.LOGICAL_BUILDER, "DrillScanRule");
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSemiJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSemiJoinRel.java
index 527b744..2ad751c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSemiJoinRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillSemiJoinRel.java
@@ -18,26 +18,28 @@
package org.apache.drill.exec.planner.logical;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.SemiJoin;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.Pair;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.logical.data.Join;
import org.apache.drill.common.logical.data.JoinCondition;
import org.apache.drill.common.logical.data.LogicalOperator;
import org.apache.drill.common.logical.data.LogicalSemiJoin;
+import org.apache.drill.exec.planner.common.DrillJoinRelBase;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.List;
-public class DrillSemiJoinRel extends SemiJoin implements DrillJoin, DrillRel {
+public class DrillSemiJoinRel extends DrillJoinRelBase implements DrillJoin, DrillRel {
public DrillSemiJoinRel(
RelOptCluster cluster,
@@ -45,28 +47,29 @@ public class DrillSemiJoinRel extends SemiJoin implements DrillJoin, DrillRel {
RelNode left,
RelNode right,
RexNode condition,
- ImmutableIntList leftKeys,
- ImmutableIntList rightKeys) {
+ List<Integer> leftKeys,
+ List<Integer> rightKeys) {
super(cluster,
traitSet,
left,
right,
condition,
- leftKeys,
- rightKeys);
+ JoinRelType.SEMI);
+ this.leftKeys = leftKeys;
+ this.rightKeys = rightKeys;
}
- public static SemiJoin create(RelNode left, RelNode right, RexNode condition,
- ImmutableIntList leftKeys, ImmutableIntList rightKeys) {
+ public static DrillSemiJoinRel create(RelNode left, RelNode right, RexNode condition,
+ List<Integer> leftKeys, List<Integer> rightKeys) {
final RelOptCluster cluster = left.getCluster();
return new DrillSemiJoinRel(cluster, cluster.traitSetOf(DrillRel.DRILL_LOGICAL), left,
right, condition, leftKeys, rightKeys);
}
@Override
- public SemiJoin copy(RelTraitSet traitSet, RexNode condition,
+ public DrillSemiJoinRel copy(RelTraitSet traitSet, RexNode condition,
RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) {
- Preconditions.checkArgument(joinType == JoinRelType.INNER);
+ Preconditions.checkArgument(joinType == JoinRelType.SEMI);
final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
Preconditions.checkArgument(joinInfo.isEqui());
return new DrillSemiJoinRel(getCluster(), traitSet, left, right, condition,
@@ -99,8 +102,9 @@ public class DrillSemiJoinRel extends SemiJoin implements DrillJoin, DrillRel {
return new LogicalSemiJoin(leftOp, rightOp, conditions, joinType);
}
+ // This method is the same as in Calcite and is here to ensure SemiJoin's behavior
@Override
- public boolean isSemiJoin() {
- return true;
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return planner.getCostFactory().makeTinyCost();
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
index 723cd08..c3470d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
@@ -727,7 +727,6 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule {
Aggregate newAggregate = aggregate.copy(
aggregate.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
newInput,
- aggregate.indicator,
aggregate.getGroupSet(),
aggregate.getGroupSets(),
aggregate.getAggCallList()
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
index f3d527e..977df33 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
@@ -121,12 +121,11 @@ public abstract class AggPrelBase extends DrillAggregateRelBase implements Prel
public AggPrelBase(RelOptCluster cluster,
RelTraitSet traits,
RelNode child,
- boolean indicator,
ImmutableBitSet groupSet,
List<ImmutableBitSet> groupSets,
List<AggregateCall> aggCalls,
OperatorPhase phase) throws InvalidRelException {
- super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls);
+ super(cluster, traits, child, groupSet, groupSets, aggCalls);
this.operPhase = phase;
createKeysAndExprs();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
index acaa5d2..0885f8a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java
@@ -41,19 +41,17 @@ public class HashAggPrel extends AggPrelBase implements Prel{
public HashAggPrel(RelOptCluster cluster,
RelTraitSet traits,
RelNode child,
- boolean indicator,
ImmutableBitSet groupSet,
List<ImmutableBitSet> groupSets,
List<AggregateCall> aggCalls,
OperatorPhase phase) throws InvalidRelException {
- super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls, phase);
+ super(cluster, traits, child, groupSet, groupSets, aggCalls, phase);
}
@Override
- public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ public Aggregate copy(RelTraitSet traitSet, RelNode input, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
try {
- return new HashAggPrel(getCluster(), traitSet, input, indicator, groupSet, groupSets, aggCalls,
- this.getOperatorPhase());
+ return new HashAggPrel(getCluster(), traitSet, input, groupSet, groupSets, aggCalls, this.getOperatorPhase());
} catch (InvalidRelException e) {
throw new AssertionError(e);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java
index dca0edd..7300a80 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrule.java
@@ -131,7 +131,6 @@ public class HashAggPrule extends AggPruleBase {
aggregate.getCluster(),
traits,
newInput,
- aggregate.indicator,
aggregate.getGroupSet(),
aggregate.getGroupSets(),
aggregate.getAggCallList(),
@@ -151,7 +150,6 @@ public class HashAggPrule extends AggPruleBase {
aggregate.getCluster(),
exch.getTraitSet(),
exch,
- aggregate.indicator,
newGroupSet,
newGroupSets,
phase1Agg.getPhase2AggCalls(),
@@ -168,7 +166,6 @@ public class HashAggPrule extends AggPruleBase {
aggregate.getCluster(),
traits,
convertedInput,
- aggregate.indicator,
aggregate.getGroupSet(),
aggregate.getGroupSets(),
aggregate.getAggCallList(),
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java
index 2581fa6..d1c3731 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java
@@ -21,6 +21,8 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexChecker;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
@@ -140,12 +142,12 @@ public abstract class JoinPrel extends DrillJoinRelBase implements Prel {
List<String> rightFields,
List<Integer> leftKeys,
List<Integer> rightKeys) {
- List<RexNode> conjuncts = RelOptUtil.conjunctions(this.getCondition());
- short i=0;
+ List<RexNode> conjuncts = getConjuncts();
+ short i = 0;
for (Pair<Integer, Integer> pair : Pair.zip(leftKeys, rightKeys)) {
final RexNode conditionExpr = conjuncts.get(i++);
- final SqlKind kind = conditionExpr.getKind();
+ SqlKind kind = conditionExpr.getKind();
if (kind != SqlKind.EQUALS && kind != SqlKind.IS_NOT_DISTINCT_FROM) {
throw UserException.unsupportedError()
.message("Unsupported comparator in join condition %s", conditionExpr)
@@ -158,6 +160,20 @@ public abstract class JoinPrel extends DrillJoinRelBase implements Prel {
}
}
+ // todo: remove this method after CALCITE-3174 is resolved
+ private List<RexNode> getConjuncts() {
+ List<RexNode> conjunctions = RelOptUtil.conjunctions(getCondition());
+ RexBuilder rexBuilder = getCluster().getRexBuilder();
+ for (int i = 0; i < conjunctions.size(); i++) {
+ RexNode node = conjunctions.get(i);
+ if (node instanceof RexCall) {
+ conjunctions.set(i,
+ RelOptUtil.collapseExpandedIsNotDistinctFromExpr((RexCall) node, rexBuilder));
+ }
+ }
+ return conjunctions;
+ }
+
public boolean isSemiJoin() {
return isSemiJoin;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
index 18681bb..f01d187 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.physical;
+import org.apache.calcite.rel.core.JoinRelType;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.calcite.plan.RelOptCluster;
@@ -29,7 +30,6 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SemiJoinType;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.commons.collections.ListUtils;
import org.apache.drill.common.expression.SchemaPath;
@@ -49,14 +49,14 @@ public class LateralJoinPrel extends DrillLateralJoinRelBase implements Prel {
protected LateralJoinPrel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, boolean excludeCorrelateCol,
- CorrelationId correlationId, ImmutableBitSet requiredColumns, SemiJoinType semiJoinType) {
+ CorrelationId correlationId, ImmutableBitSet requiredColumns, JoinRelType semiJoinType) {
super(cluster, traits, left, right, excludeCorrelateCol, correlationId, requiredColumns, semiJoinType);
}
@Override
public Correlate copy(RelTraitSet traitSet,
RelNode left, RelNode right, CorrelationId correlationId,
- ImmutableBitSet requiredColumns, SemiJoinType joinType) {
+ ImmutableBitSet requiredColumns, JoinRelType joinType) {
return new LateralJoinPrel(this.getCluster(), this.getTraitSet(), left, right, this.excludeCorrelateColumn, correlationId, requiredColumns,
this.getJoinType());
}
@@ -67,12 +67,12 @@ public class LateralJoinPrel extends DrillLateralJoinRelBase implements Prel {
PhysicalOperator leftPop = ((Prel)left).getPhysicalOperator(creator);
PhysicalOperator rightPop = ((Prel)right).getPhysicalOperator(creator);
- SemiJoinType jtype = this.getJoinType();
+ JoinRelType jtype = this.getJoinType();
List<SchemaPath> excludedColumns = new ArrayList<>();
if (getColumn() != null) {
excludedColumns.add(getColumn());
}
- LateralJoinPOP ljoin = new LateralJoinPOP(leftPop, rightPop, jtype.toJoinType(), DrillLateralJoinRelBase.IMPLICIT_COLUMN, excludedColumns);
+ LateralJoinPOP ljoin = new LateralJoinPOP(leftPop, rightPop, jtype, DrillLateralJoinRelBase.IMPLICIT_COLUMN, excludedColumns);
return creator.addMetadata(this, ljoin);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
index b880b57..ef41ab1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java
@@ -42,23 +42,20 @@ public class StreamAggPrel extends AggPrelBase implements Prel{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StreamAggPrel.class);
-
-
public StreamAggPrel(RelOptCluster cluster,
RelTraitSet traits,
RelNode child,
- boolean indicator,
ImmutableBitSet groupSet,
List<ImmutableBitSet> groupSets,
List<AggregateCall> aggCalls,
OperatorPhase phase) throws InvalidRelException {
- super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls, phase);
+ super(cluster, traits, child, groupSet, groupSets, aggCalls, phase);
}
@Override
- public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
+ public Aggregate copy(RelTraitSet traitSet, RelNode input, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
try {
- return new StreamAggPrel(getCluster(), traitSet, input, indicator, groupSet, groupSets, aggCalls, this.getOperatorPhase());
+ return new StreamAggPrel(getCluster(), traitSet, input, groupSet, groupSets, aggCalls, this.getOperatorPhase());
} catch (InvalidRelException e) {
throw new AssertionError(e);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrule.java
index d566638..0b68014 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrule.java
@@ -85,7 +85,6 @@ public class StreamAggPrule extends AggPruleBase {
aggregate.getCluster(),
traits,
newInput,
- aggregate.indicator,
aggregate.getGroupSet(),
aggregate.getGroupSets(),
aggregate.getAggCallList(),
@@ -104,7 +103,6 @@ public class StreamAggPrule extends AggPruleBase {
aggregate.getCluster(),
singleDistTrait,
exch,
- aggregate.indicator,
newGroupSet,
newGroupSets,
phase1Agg.getPhase2AggCalls(),
@@ -153,7 +151,6 @@ public class StreamAggPrule extends AggPruleBase {
aggregate.getCluster(),
traits,
newInput,
- aggregate.indicator,
aggregate.getGroupSet(),
aggregate.getGroupSets(),
aggregate.getAggCallList(),
@@ -177,7 +174,6 @@ public class StreamAggPrule extends AggPruleBase {
aggregate.getCluster(),
exch.getTraitSet(),
exch,
- aggregate.indicator,
newGroupSet,
newGroupSets,
phase1Agg.getPhase2AggCalls(),
@@ -200,7 +196,6 @@ public class StreamAggPrule extends AggPruleBase {
aggregate.getCluster(),
traits,
convertedInput,
- aggregate.indicator,
aggregate.getGroupSet(),
aggregate.getGroupSets(),
aggregate.getAggCallList(),
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillCalciteSqlFunctionWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillCalciteSqlFunctionWrapper.java
index cf3b866..4c745a1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillCalciteSqlFunctionWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillCalciteSqlFunctionWrapper.java
@@ -21,6 +21,7 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlOperatorBinding;
import org.apache.calcite.sql.SqlSyntax;
@@ -61,6 +62,11 @@ public class DrillCalciteSqlFunctionWrapper extends SqlFunction implements Drill
}
@Override
+ public SqlNode rewriteCall(SqlValidator validator, SqlCall call) {
+ return operator.rewriteCall(validator, call);
+ }
+
+ @Override
public SqlOperator getOperator() {
return operator;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
index eb79a5a..a957e84 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.sql;
+import org.apache.calcite.sql.validate.SqlNameMatcher;
import org.apache.drill.shaded.guava.com.google.common.collect.ArrayListMultimap;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
@@ -101,18 +102,18 @@ public class DrillOperatorTable extends SqlStdOperatorTable {
@Override
public void lookupOperatorOverloads(SqlIdentifier opName, SqlFunctionCategory category,
- SqlSyntax syntax, List<SqlOperator> operatorList) {
+ SqlSyntax syntax, List<SqlOperator> operatorList, SqlNameMatcher nameMatcher) {
if (isInferenceEnabled()) {
- populateFromTypeInference(opName, category, syntax, operatorList);
+ populateFromTypeInference(opName, category, syntax, operatorList, nameMatcher);
} else {
- populateFromWithoutTypeInference(opName, category, syntax, operatorList);
+ populateFromWithoutTypeInference(opName, category, syntax, operatorList, nameMatcher);
}
}
private void populateFromTypeInference(SqlIdentifier opName, SqlFunctionCategory category,
- SqlSyntax syntax, List<SqlOperator> operatorList) {
+ SqlSyntax syntax, List<SqlOperator> operatorList, SqlNameMatcher nameMatcher) {
final List<SqlOperator> calciteOperatorList = Lists.newArrayList();
- inner.lookupOperatorOverloads(opName, category, syntax, calciteOperatorList);
+ inner.lookupOperatorOverloads(opName, category, syntax, calciteOperatorList, nameMatcher);
if (!calciteOperatorList.isEmpty()) {
for (SqlOperator calciteOperator : calciteOperatorList) {
if (calciteToWrapper.containsKey(calciteOperator)) {
@@ -133,8 +134,8 @@ public class DrillOperatorTable extends SqlStdOperatorTable {
}
private void populateFromWithoutTypeInference(SqlIdentifier opName, SqlFunctionCategory category,
- SqlSyntax syntax, List<SqlOperator> operatorList) {
- inner.lookupOperatorOverloads(opName, category, syntax, operatorList);
+ SqlSyntax syntax, List<SqlOperator> operatorList, SqlNameMatcher nameMatcher) {
+ inner.lookupOperatorOverloads(opName, category, syntax, operatorList, nameMatcher);
if (operatorList.isEmpty() && (syntax == SqlSyntax.FUNCTION || syntax == SqlSyntax.FUNCTION_ID) && opName.isSimple()) {
List<SqlOperator> drillOps = drillOperatorsWithoutInferenceMap.get(opName.getSimple().toLowerCase());
if (drillOps != null) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
index d766515..097c7aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
@@ -25,8 +25,10 @@ import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
import org.apache.calcite.util.Static;
import org.apache.commons.lang3.StringUtils;
import org.apache.drill.exec.planner.sql.parser.DrillParserUtil;
@@ -34,6 +36,9 @@ import org.apache.drill.exec.planner.sql.parser.impl.DrillSqlParseException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.metastore.MetadataProviderManager;
import org.apache.drill.metastore.metadata.TableMetadataProvider;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.drill.exec.util.Utilities;
import org.apache.drill.shaded.guava.com.google.common.cache.CacheBuilder;
import org.apache.drill.shaded.guava.com.google.common.cache.CacheLoader;
import org.apache.drill.shaded.guava.com.google.common.cache.LoadingCache;
@@ -120,6 +125,7 @@ public class SqlConverter {
private final RelOptCostFactory costFactory;
private final DrillValidator validator;
private final boolean isInnerQuery;
+ private final boolean isExpandedView;
private final UdfUtilities util;
private final FunctionImplementationRegistry functions;
private final String temporarySchema;
@@ -146,6 +152,7 @@ public class SqlConverter {
this.parserConfig = new DrillParserConfig(settings);
this.sqlToRelConverterConfig = new SqlToRelConverterConfig();
this.isInnerQuery = false;
+ this.isExpandedView = false;
this.typeFactory = new JavaTypeFactoryImpl(DRILL_TYPE_SYSTEM);
this.defaultSchema = context.getNewDefaultSchema();
this.rootSchema = rootSchema(defaultSchema);
@@ -174,6 +181,7 @@ public class SqlConverter {
this.functions = parent.functions;
this.util = parent.util;
this.isInnerQuery = true;
+ this.isExpandedView = true;
this.typeFactory = parent.typeFactory;
this.costFactory = parent.costFactory;
this.settings = parent.settings;
@@ -403,8 +411,21 @@ public class SqlConverter {
new SqlToRelConverter(new Expander(), validator, catalog, cluster, DrillConvertletTable.INSTANCE,
sqlToRelConverterConfig);
- //To avoid unexpected column errors set a value of top to false
- final RelRoot rel = sqlToRelConverter.convertQuery(validatedNode, false, false);
+ RelRoot rel = sqlToRelConverter.convertQuery(validatedNode, false, !isInnerQuery || isExpandedView);
+
+ // If extra expressions used in ORDER BY were added to the project list,
+ // add another project to remove them.
+ if ((!isInnerQuery || isExpandedView) && rel.rel.getRowType().getFieldCount() - rel.fields.size() > 0) {
+ RexBuilder builder = rel.rel.getCluster().getRexBuilder();
+
+ RelNode relNode = rel.rel;
+ List<RexNode> expressions = rel.fields.stream()
+ .map(f -> builder.makeInputRef(relNode, f.left))
+ .collect(Collectors.toList());
+
+ RelNode project = LogicalProject.create(rel.rel, expressions, rel.validatedRowType);
+ rel = RelRoot.of(project, rel.validatedRowType, rel.kind);
+ }
return rel.withRel(sqlToRelConverter.flattenTypes(rel.rel, true));
}
@@ -559,6 +580,9 @@ public class SqlConverter {
private void initCluster() {
cluster = RelOptCluster.create(planner, new DrillRexBuilder(typeFactory));
+ JaninoRelMetadataProvider relMetadataProvider = Utilities.registerJaninoRelMetadataProvider();
+
+ cluster.setMetadataProvider(relMetadataProvider);
}
private static class DrillRexBuilder extends RexBuilder {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index fdaa8cd..8f49961 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
import com.fasterxml.jackson.databind.ser.PropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
+import org.apache.drill.exec.util.Utilities;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
import org.apache.calcite.plan.RelOptCostImpl;
@@ -44,7 +45,6 @@ import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
@@ -73,7 +73,6 @@ import org.apache.drill.exec.physical.impl.join.JoinUtils;
import org.apache.drill.exec.planner.PlannerPhase;
import org.apache.drill.exec.planner.PlannerType;
import org.apache.drill.exec.planner.common.DrillRelOptUtil;
-import org.apache.drill.exec.planner.cost.DrillDefaultRelMetadataProvider;
import org.apache.drill.exec.planner.logical.DrillProjectRel;
import org.apache.drill.exec.planner.logical.DrillRel;
import org.apache.drill.exec.planner.logical.DrillRelFactories;
@@ -407,8 +406,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
final HepPlanner planner = new HepPlanner(hepPgmBldr.build(), context.getPlannerSettings(), true, null,
RelOptCostImpl.FACTORY);
- JaninoRelMetadataProvider relMetadataProvider = JaninoRelMetadataProvider.of(DrillDefaultRelMetadataProvider.INSTANCE);
- RelMetadataQuery.THREAD_PROVIDERS.set(relMetadataProvider);
+ JaninoRelMetadataProvider relMetadataProvider = Utilities.registerJaninoRelMetadataProvider();
// Modify RelMetaProvider for every RelNode in the SQL operator Rel tree.
input.accept(new MetaDataProviderModifier(relMetadataProvider));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
index f5b690a..d524d2d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ViewHandler.java
@@ -66,7 +66,8 @@ public abstract class ViewHandler extends DefaultSqlHandler {
// Disallow temporary tables usage in view definition
config.getConverter().disallowTemporaryTables();
// Store the viewSql as view def SqlNode is modified as part of the resolving the new table definition below.
- final String viewSql = createView.getQuery().toString();
+ // Use of parentheses is forced to ensure expressions are read correctly.
+ final String viewSql = createView.getQuery().toSqlString(null, true).getSql();
final ConvertedRelNode convertedRelNode = validateAndConvert(createView.getQuery());
final RelDataType validatedRowType = convertedRelNode.getValidatedRowType();
final RelNode queryRelNode = convertedRelNode.getConvertedNode();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
index b972841..a8bdd1c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/UnsupportedOperatorsVisitor.java
@@ -79,8 +79,8 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
@Override
public SqlNode visit(SqlDataTypeSpec type) {
- for(String strType : disabledType) {
- if(type.getTypeName().getSimple().equalsIgnoreCase(strType)) {
+ for (String strType : disabledType) {
+ if (type.getTypeName().getSimple().equalsIgnoreCase(strType)) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.DATA_TYPE,
type.getTypeName().getSimple() + " is not supported\n" +
"See Apache Drill JIRA: DRILL-1959");
@@ -94,24 +94,24 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
@Override
public SqlNode visit(SqlCall sqlCall) {
// Inspect the window functions
- if(sqlCall instanceof SqlSelect) {
+ if (sqlCall instanceof SqlSelect) {
SqlSelect sqlSelect = (SqlSelect) sqlCall;
checkGrouping((sqlSelect));
checkRollupCubeGrpSets(sqlSelect);
- for(SqlNode nodeInSelectList : sqlSelect.getSelectList()) {
+ for (SqlNode nodeInSelectList : sqlSelect.getSelectList()) {
// If the window function is used with an alias,
// enter the first operand of AS operator
- if(nodeInSelectList.getKind() == SqlKind.AS
+ if (nodeInSelectList.getKind() == SqlKind.AS
&& (((SqlCall) nodeInSelectList).getOperandList().get(0).getKind() == SqlKind.OVER)) {
nodeInSelectList = ((SqlCall) nodeInSelectList).getOperandList().get(0);
}
- if(nodeInSelectList.getKind() == SqlKind.OVER) {
+ if (nodeInSelectList.getKind() == SqlKind.OVER) {
// Throw exceptions if window functions are disabled
- if(!context.getOptions().getOption(ExecConstants.ENABLE_WINDOW_FUNCTIONS).bool_val) {
+ if (!context.getOptions().getOption(ExecConstants.ENABLE_WINDOW_FUNCTIONS).bool_val) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Window functions are disabled\n" +
"See Apache Drill JIRA: DRILL-2559");
@@ -120,12 +120,12 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
// DRILL-3182, DRILL-3195
SqlCall over = (SqlCall) nodeInSelectList;
- if(over.getOperandList().get(0) instanceof SqlCall) {
+ if (over.getOperandList().get(0) instanceof SqlCall) {
SqlCall function = (SqlCall) over.getOperandList().get(0);
// DRILL-3182
// Window function with DISTINCT qualifier is temporarily disabled
- if(function.getFunctionQuantifier() != null
+ if (function.getFunctionQuantifier() != null
&& function.getFunctionQuantifier().getValue() == SqlSelectKeyword.DISTINCT) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"DISTINCT for window aggregate functions is not currently supported\n" +
@@ -174,7 +174,7 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
// DRILL-3188
// Disable frame which is other than the default
// (i.e., BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
- if(sqlCall instanceof SqlWindow) {
+ if (sqlCall instanceof SqlWindow) {
SqlWindow window = (SqlWindow) sqlCall;
SqlNode lowerBound = window.getLowerBound();
@@ -188,7 +188,7 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
// RANGE UNBOUNDED PRECEDING
// RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
// RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
- if(window.getOrderList().size() != 0
+ if (window.getOrderList().size() != 0
&& !window.isRows()
&& SqlWindow.isUnboundedPreceding(lowerBound)
&& (upperBound == null || SqlWindow.isCurrentRow(upperBound) || SqlWindow.isUnboundedFollowing(upperBound))) {
@@ -214,13 +214,13 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
// When OVER clause doesn't contain an ORDER BY clause, the following are equivalent to the default frame:
// RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
// ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
- if(window.getOrderList().size() == 0
+ if (window.getOrderList().size() == 0
&& SqlWindow.isUnboundedPreceding(lowerBound)
&& SqlWindow.isUnboundedFollowing(upperBound)) {
isSupported = true;
}
- if(!isSupported) {
+ if (!isSupported) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"This type of window frame is currently not supported \n" +
"See Apache Drill JIRA: DRILL-3188");
@@ -228,7 +228,7 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
// DRILL-3189: Disable DISALLOW PARTIAL
- if(!window.isAllowPartial()) {
+ if (!window.isAllowPartial()) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Disallowing partial windows is currently not supported \n" +
"See Apache Drill JIRA: DRILL-3189");
@@ -237,7 +237,7 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
// Disable unsupported Intersect, Except
- if(sqlCall.getKind() == SqlKind.INTERSECT || sqlCall.getKind() == SqlKind.EXCEPT) {
+ if (sqlCall.getKind() == SqlKind.INTERSECT || sqlCall.getKind() == SqlKind.EXCEPT) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.RELATIONAL,
sqlCall.getOperator().getName() + " is not supported\n" +
"See Apache Drill JIRA: DRILL-1921");
@@ -245,11 +245,11 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
// Disable unsupported JOINs
- if(sqlCall.getKind() == SqlKind.JOIN) {
+ if (sqlCall.getKind() == SqlKind.JOIN) {
SqlJoin join = (SqlJoin) sqlCall;
// Block Natural Join
- if(join.isNatural()) {
+ if (join.isNatural()) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.RELATIONAL,
"NATURAL JOIN is not supported\n" +
"See Apache Drill JIRA: DRILL-1986");
@@ -267,8 +267,8 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
// Disable Function
- for(String strOperator : disabledOperators) {
- if(sqlCall.getOperator().isName(strOperator)) {
+ for (String strOperator : disabledOperators) {
+ if (sqlCall.getOperator().isName(strOperator, true)) { // true is passed to preserve previous behavior
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
sqlCall.getOperator().getName() + " is not supported\n" +
"See Apache Drill JIRA: DRILL-2115");
@@ -277,7 +277,7 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
// Disable complex functions incorrect placement
- if(sqlCall instanceof SqlSelect) {
+ if (sqlCall instanceof SqlSelect) {
SqlSelect sqlSelect = (SqlSelect) sqlCall;
for (SqlNode nodeInSelectList : sqlSelect.getSelectList()) {
@@ -299,9 +299,9 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
}
- if(sqlSelect.hasOrderBy()) {
+ if (sqlSelect.hasOrderBy()) {
for (SqlNode sqlNode : sqlSelect.getOrderList()) {
- if(containsFlatten(sqlNode)) {
+ if (containsFlatten(sqlNode)) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Flatten function is not supported in Order By\n" +
"See Apache Drill JIRA: DRILL-2181");
@@ -315,9 +315,9 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
}
- if(sqlSelect.getGroup() != null) {
- for(SqlNode sqlNode : sqlSelect.getGroup()) {
- if(containsFlatten(sqlNode)) {
+ if (sqlSelect.getGroup() != null) {
+ for (SqlNode sqlNode : sqlSelect.getGroup()) {
+ if (containsFlatten(sqlNode)) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Flatten function is not supported in Group By\n" +
"See Apache Drill JIRA: DRILL-2181");
@@ -331,17 +331,17 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
}
- if(sqlSelect.isDistinct()) {
- for(SqlNode column : sqlSelect.getSelectList()) {
- if(column.getKind() == SqlKind.AS) {
- if(containsFlatten(((SqlCall) column).getOperandList().get(0))) {
+ if (sqlSelect.isDistinct()) {
+ for (SqlNode column : sqlSelect.getSelectList()) {
+ if (column.getKind() == SqlKind.AS) {
+ if (containsFlatten(((SqlCall) column).getOperandList().get(0))) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Flatten function is not supported in Distinct\n" +
"See Apache Drill JIRA: DRILL-2181");
throw new UnsupportedOperationException();
}
} else {
- if(containsFlatten(column)) {
+ if (containsFlatten(column)) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Flatten function is not supported in Distinct\n" +
"See Apache Drill JIRA: DRILL-2181");
@@ -352,9 +352,9 @@ public class UnsupportedOperatorsVisitor extends SqlShuttle {
}
}
- if(DrillCalciteWrapperUtility.extractSqlOperatorFromWrapper(sqlCall.getOperator()) instanceof SqlCountAggFunction) {
- for(SqlNode sqlNode : sqlCall.getOperandList()) {
- if(containsFlatten(sqlNode)) {
+ if (DrillCalciteWrapperUtility.extractSqlOperatorFromWrapper(sqlCall.getOperator()) instanceof SqlCountAggFunction) {
+ for (SqlNode sqlNode : sqlCall.getOperandList()) {
+ if (containsFlatten(sqlNode)) {
unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.FUNCTION,
"Flatten function in aggregate functions is not supported\n" +
"See Apache Drill JIRA: DRILL-2181");
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java
index f440cf1..87f2201 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java
@@ -19,11 +19,14 @@ package org.apache.drill.exec.util;
import java.util.Collection;
import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexLiteral;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.planner.cost.DrillDefaultRelMetadataProvider;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.planner.logical.DrillTranslatableTable;
import org.apache.drill.exec.proto.BitControl.QueryContextInformation;
@@ -128,4 +131,10 @@ public class Utilities {
return null;
}
}
+
+ public static JaninoRelMetadataProvider registerJaninoRelMetadataProvider() {
+ JaninoRelMetadataProvider relMetadataProvider = JaninoRelMetadataProvider.of(DrillDefaultRelMetadataProvider.INSTANCE);
+ RelMetadataQuery.THREAD_PROVIDERS.set(relMetadataProvider);
+ return relMetadataProvider;
+ }
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLateLimit0Optimization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLateLimit0Optimization.java
index 94fcab5..22e50bc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLateLimit0Optimization.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLateLimit0Optimization.java
@@ -59,7 +59,7 @@ public class TestLateLimit0Optimization extends BaseTestQuery {
" STRING_BINARY(CONVERT_TO(r_regionkey, 'BIGINT')) as l_be,\n" +
" STRING_BINARY(CONVERT_TO(r_name, 'UTF8')) u8,\n" +
" STRING_BINARY(CONVERT_TO(r_name, 'UTF16')) u16,\n" +
- " STRING_BINARY(CONVERT_TO(r_regionkey, 'INT_HADOOPV')) as l_be\n" +
+ " STRING_BINARY(CONVERT_TO(r_regionkey, 'INT_HADOOPV')) as i_ha\n" +
"FROM cp.`tpch/region.parquet`");
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
index ebe585c..bda294d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
@@ -821,4 +821,28 @@ public class TestViewSupport extends TestBaseViewSupport {
}, null);
}
+ @Test
+ @Category(UnlikelyTest.class)
+ public void testViewContainingWithClauseAndUnionAllInDefinition() throws Exception{
+ String viewName = "test_view";
+ try {
+ String viewDefinition = "create or replace view %s.`%s` as " +
+ "with tbl_un as " +
+ "(" +
+ "(select c_name from cp.`tpch/customer.parquet` limit 1)" +
+ "union all " +
+ "(select c_name from cp.`tpch/customer.parquet` limit 1 offset 1) " +
+ ") " +
+ "select * from tbl_un";
+ test(viewDefinition, DFS_TMP_SCHEMA, viewName);
+ testBuilder()
+ .sqlQuery("select * from %s.`%s`", DFS_TMP_SCHEMA, viewName)
+ .unOrdered()
+ .baselineColumns("c_name")
+ .baselineValuesForSingleColumn("Customer#000000001", "Customer#000000002")
+ .go();
+ } finally {
+ test("drop view if exists %s.`%s`", DFS_TMP_SCHEMA, viewName);
+ }
+ }
}
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index e782d46..a48dabb 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -300,8 +300,8 @@
</includes>
<excludes>
<exclude>io.protostuff:*</exclude>
- <exclude>com.github.vvysotskyi.drill-calcite:calcite-core</exclude>
- <exclude>com.github.vvysotskyi.drill-calcite:calcite-linq4j</exclude>
+ <exclude>${calcite.groupId}:calcite-core</exclude>
+ <exclude>${calcite.groupId}:calcite-linq4j</exclude>
<exclude>org.pentaho:*</exclude>
<exclude>org.msgpack:*</exclude>
<exclude>xerces:*</exclude>
@@ -615,8 +615,8 @@
<excludes>
<exclude>org.slf4j:jcl-over-slf4j</exclude>
<exclude>io.protostuff:*</exclude>
- <exclude>com.github.vvysotskyi.drill-calcite:calcite-core</exclude>
- <exclude>com.github.vvysotskyi.drill-calcite:calcite-linq4j</exclude>
+ <exclude>${calcite.groupId}:calcite-core</exclude>
+ <exclude>${calcite.groupId}:calcite-linq4j</exclude>
<exclude>org.pentaho:*</exclude>
<exclude>org.msgpack:*</exclude>
<exclude>xerces:*</exclude>
diff --git a/logical/pom.xml b/logical/pom.xml
index a685bb2..cd212aa 100644
--- a/logical/pom.xml
+++ b/logical/pom.xml
@@ -59,7 +59,7 @@
</dependency>
<dependency>
- <groupId>com.github.vvysotskyi.drill-calcite</groupId>
+ <groupId>${calcite.groupId}</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
diff --git a/pom.xml b/pom.xml
index 5626bb1..ed39611 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,8 +50,15 @@
<guava.version>19.0</guava.version>
<forkCount>2</forkCount>
<parquet.version>1.10.0</parquet.version>
- <calcite.version>1.18.0-drill-r2</calcite.version>
- <avatica.version>1.13.0</avatica.version>
+ <!--
+ For development purposes to be able to use custom Calcite versions (e.g. not present in jitpack
+ repository or from local repository) update this property to desired value (e.g. org.apache.calcite).
+ In case if new value is org.apache.calcite, one needs to remove `org.apache.calcite:*` from
+ avoid_bad_dependencies plugin found in the file.
+ -->
+ <calcite.groupId>com.github.vvysotskyi.drill-calcite</calcite.groupId>
+ <calcite.version>1.20.0-drill-r0</calcite.version>
+ <avatica.version>1.15.0</avatica.version>
<janino.version>3.0.11</janino.version>
<sqlline.version>1.8.0</sqlline.version>
<jackson.version>2.9.5</jackson.version>
@@ -1039,7 +1046,7 @@
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>com.github.vvysotskyi.drill-calcite</groupId>
+ <groupId>${calcite.groupId}</groupId>
<artifactId>calcite-core</artifactId>
<version>${calcite.version}</version>
<exclusions>