You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/03/18 22:18:50 UTC
[5/7] drill git commit: DRILL-2106: Fix SplitUpComplexExpression rule
to correctly detect last used column reference in the project expression
DRILL-2106: Fix SplitUpComplexExpression rule to correctly detect last used column reference in the project expression
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/47bcacae
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/47bcacae
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/47bcacae
Branch: refs/heads/master
Commit: 47bcacaeed46dd1594d36c5d7e4939d9e4c67bab
Parents: 8faa664
Author: Mehant Baid <me...@gmail.com>
Authored: Sat Mar 14 12:20:30 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Tue Mar 17 18:08:04 2015 -0700
----------------------------------------------------------------------
.../drill/exec/planner/physical/PrelUtil.java | 36 ++++++++++++++++++++
.../visitor/SplitUpComplexExpressions.java | 13 ++++---
.../exec/physical/impl/flatten/TestFlatten.java | 15 ++++++++
.../resources/flatten/drill-2106-result.json | 12 +++++++
4 files changed, 71 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/47bcacae/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
index aa835e6..d904c3a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
@@ -125,6 +125,14 @@ public class PrelUtil {
return new SelectionVectorRemoverPrel(prel);
}
+ public static int getLastUsedColumnReference(List<RexNode> projects) {
+ LastUsedRefVisitor lastUsed = new LastUsedRefVisitor();
+ for (RexNode rex : projects) {
+ rex.accept(lastUsed);
+ }
+ return lastUsed.getLastUsedReference();
+ }
+
public static ProjectPushInfo getColumns(RelDataType rowType, List<RexNode> projects) {
final List<String> fieldNames = rowType.getFieldNames();
if (fieldNames.isEmpty()) {
@@ -241,6 +249,34 @@ public class PrelUtil {
}
}
+ // Simple visitor class to determine the last used reference in the expression
+ private static class LastUsedRefVisitor extends RexVisitorImpl {
+
+ int lastUsedRef = -1;
+
+ protected LastUsedRefVisitor() {
+ super(true);
+ }
+
+ @Override
+ public Void visitInputRef(RexInputRef inputRef) {
+ lastUsedRef = Math.max(lastUsedRef, inputRef.getIndex());
+ return null;
+ }
+
+ @Override
+ public Void visitCall(RexCall call) {
+ for (RexNode operand : call.operands) {
+ operand.accept(this);
+ }
+ return null;
+ }
+
+ public int getLastUsedReference() {
+ return lastUsedRef;
+ }
+ }
+
/** Visitor that finds the set of inputs that are used. */
private static class RefFieldsVisitor extends RexVisitorImpl<PathSegment> {
final Set<SchemaPath> columns = Sets.newLinkedHashSet();
http://git-wip-us.apache.org/repos/asf/drill/blob/47bcacae/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
index 11b6c43..d263ad1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
@@ -82,18 +82,22 @@ public class SplitUpComplexExpressions extends BasePrelVisitor<Prel, Object, Rel
@Override
public Prel visitProject(ProjectPrel project, Object unused) throws RelConversionException {
+ // Apply the rule to the child
+ RelNode originalInput = ((Prel)project.getInput(0)).accept(this, null);
+ project = (ProjectPrel) project.copy(project.getTraitSet(), Lists.newArrayList(originalInput));
+
List<RexNode> exprList = new ArrayList<>();
List<RelDataTypeField> relDataTypes = new ArrayList();
List<RelDataTypeField> origRelDataTypes = new ArrayList();
int i = 0;
+ final int lastColumnReferenced = PrelUtil.getLastUsedColumnReference(project.getProjects());
- ProjectPushInfo columnInfo = PrelUtil.getColumns(project.getInput(0).getRowType(), project.getProjects());
-
- if (columnInfo == null ) {
+ if (lastColumnReferenced == -1) {
return project;
}
- int lastRexInput = columnInfo.desiredFields.size();
+
+ final int lastRexInput = lastColumnReferenced + 1;
RexVisitorComplexExprSplitter exprSplitter = new RexVisitorComplexExprSplitter(factory, funcReg, lastRexInput);
for (RexNode rex : project.getChildExps()) {
@@ -103,7 +107,6 @@ public class SplitUpComplexExpressions extends BasePrelVisitor<Prel, Object, Rel
}
List<RexNode> complexExprs = exprSplitter.getComplexExprs();
- RelNode originalInput = ((Prel)project.getInput(0)).accept(this, null);
ProjectPrel childProject;
List<RexNode> allExprs = new ArrayList();
http://git-wip-us.apache.org/repos/asf/drill/blob/47bcacae/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index dc37079..d64dc91 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -249,4 +249,19 @@ public class TestFlatten extends BaseTestQuery {
.build().run();
}
+ @Test
+ public void testDRILL_2106() throws Exception {
+ testBuilder()
+ .sqlQuery("select rl, flatten(rl) frl from (select `integer`, flatten(rl) as rl from cp.`jsoninput/input2.json`)")
+ .unOrdered()
+ .jsonBaselineFile("flatten/drill-2106-result.json")
+ .go();
+
+ testBuilder()
+ .sqlQuery("select rl, flatten(rl) frl from (select flatten(rl) as rl, `integer` from cp.`jsoninput/input2.json`)")
+ .unOrdered()
+ .jsonBaselineFile("flatten/drill-2106-result.json")
+ .go();
+
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/47bcacae/exec/java-exec/src/test/resources/flatten/drill-2106-result.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/drill-2106-result.json b/exec/java-exec/src/test/resources/flatten/drill-2106-result.json
new file mode 100644
index 0000000..35e3c0d
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/drill-2106-result.json
@@ -0,0 +1,12 @@
+{"rl": [2,1], "frl": 2}
+{"rl": [2,1], "frl": 1}
+{"rl": [4,6], "frl": 4}
+{"rl": [4,6], "frl": 6}
+{"rl": [2,1], "frl": 2}
+{"rl": [2,1], "frl": 1}
+{"rl": [4,6], "frl": 4}
+{"rl": [4,6], "frl": 6}
+{"rl": [2,1], "frl": 2}
+{"rl": [2,1], "frl": 1}
+{"rl": [4,6], "frl": 4}
+{"rl": [4,6], "frl": 6}
\ No newline at end of file