You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2018/07/04 10:39:37 UTC
[drill] branch master updated: DRILL-6546: Allow unnest function
with nested columns and complex expressions
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new cacca92 DRILL-6546: Allow unnest function with nested columns and complex expressions
cacca92 is described below
commit cacca92fde38208828fea71d449ebb67ad9fc10f
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Thu Jun 14 19:32:43 2018 +0300
DRILL-6546: Allow unnest function with nested columns and complex expressions
Fix loss of projected names in right side of correlate when single field is projected
---
.../apache/drill/exec/planner/PlannerPhase.java | 3 +
.../planner/common/DrillLateralJoinRelBase.java | 19 ++-
.../exec/planner/logical/DrillLateralJoinRel.java | 2 +-
.../exec/planner/logical/DrillUnnestRule.java | 11 +-
...rojectComplexRexNodeCorrelateTransposeRule.java | 154 +++++++++++++++++++
.../exec/planner/physical/LateralJoinPrel.java | 5 +-
.../drill/exec/planner/physical/UnnestPrule.java | 5 -
.../physical/visitor/JoinPrelRenameVisitor.java | 14 +-
.../sql/parser/CompoundIdentifierConverter.java | 171 +++++++++++----------
.../impl/lateraljoin/TestE2EUnnestAndLateral.java | 88 +++++++++++
10 files changed, 362 insertions(+), 110 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
index 519d503..e5a3746 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
@@ -37,6 +37,7 @@ import org.apache.drill.exec.planner.logical.DrillJoinRel;
import org.apache.drill.exec.planner.logical.DrillJoinRule;
import org.apache.drill.exec.planner.logical.DrillLimitRule;
import org.apache.drill.exec.planner.logical.DrillMergeProjectRule;
+import org.apache.drill.exec.planner.logical.ProjectComplexRexNodeCorrelateTransposeRule;
import org.apache.drill.exec.planner.logical.DrillProjectLateralJoinTransposeRule;
import org.apache.drill.exec.planner.logical.DrillProjectPushIntoLateralJoinRule;
import org.apache.drill.exec.planner.logical.DrillProjectRule;
@@ -311,6 +312,8 @@ public enum PlannerPhase {
RuleInstance.PROJECT_WINDOW_TRANSPOSE_RULE,
DrillPushProjectIntoScanRule.INSTANCE,
+ ProjectComplexRexNodeCorrelateTransposeRule.INSTANCE,
+
/*
Convert from Calcite Logical to Drill Logical Rules.
*/
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java
index 28e5246..2f895e2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java
@@ -73,7 +73,7 @@ public abstract class DrillLateralJoinRelBase extends Correlate implements Drill
return constructRowType(SqlValidatorUtil.deriveJoinRowType(left.getRowType(),
right.getRowType(), joinType.toJoinType(),
getCluster().getTypeFactory(), null,
- ImmutableList.<RelDataTypeField>of()));
+ ImmutableList.of()));
case ANTI:
case SEMI:
return constructRowType(left.getRowType());
@@ -82,12 +82,19 @@ public abstract class DrillLateralJoinRelBase extends Correlate implements Drill
}
}
- public int getInputSize(int offset, RelNode input) {
- if (this.excludeCorrelateColumn &&
- offset == 0) {
- return input.getRowType().getFieldList().size() - 1;
+ /**
+ * Returns number of fields in {@link RelDataType} for
+ * input rel node with specified ordinal considering value of
+ * {@code excludeCorrelateColumn}.
+ *
+ * @param ordinal ordinal of input rel node
+ * @return number of fields in input's {@link RelDataType}
+ */
+ public int getInputSize(int ordinal) {
+ if (this.excludeCorrelateColumn && ordinal == 0) {
+ return getInput(ordinal).getRowType().getFieldList().size() - 1;
}
- return input.getRowType().getFieldList().size();
+ return getInput(ordinal).getRowType().getFieldList().size();
}
public RelDataType constructRowType(RelDataType inputRowType) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java
index aa6ccb0..4356d49 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java
@@ -50,7 +50,7 @@ public class DrillLateralJoinRel extends DrillLateralJoinRelBase implements Dril
public LogicalOperator implement(DrillImplementor implementor) {
final List<String> fields = getRowType().getFieldNames();
assert DrillJoinRel.isUnique(fields);
- final int leftCount = getInputSize(0,left);
+ final int leftCount = getInputSize(0);
final LogicalOperator leftOp = DrillJoinRel.implementInput(implementor, 0, 0, left, this);
final LogicalOperator rightOp = DrillJoinRel.implementInput(implementor, 1, leftCount, right, this);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java
index 762eb46..ce0cd3c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java
@@ -24,6 +24,8 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.core.Uncollect;
import org.apache.calcite.rel.logical.LogicalProject;
import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
public class DrillUnnestRule extends RelOptRule {
public static final RelOptRule INSTANCE = new DrillUnnestRule();
@@ -38,11 +40,14 @@ public class DrillUnnestRule extends RelOptRule {
public void onMatch(RelOptRuleCall call) {
final Uncollect uncollect = call.rel(0);
final LogicalProject project = call.rel(1);
- final LogicalValues values = call.rel(2);
+ RexNode projectedNode = project.getProjects().iterator().next();
+ if (projectedNode.getKind() != SqlKind.FIELD_ACCESS) {
+ return;
+ }
final RelTraitSet traits = uncollect.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
- DrillUnnestRel unnest = new DrillUnnestRel(uncollect.getCluster(), traits, uncollect.getRowType(),
- project.getProjects().iterator().next());
+ DrillUnnestRel unnest = new DrillUnnestRel(uncollect.getCluster(),
+ traits, uncollect.getRowType(), projectedNode);
call.transformTo(unnest);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ProjectComplexRexNodeCorrelateTransposeRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ProjectComplexRexNodeCorrelateTransposeRule.java
new file mode 100644
index 0000000..a979d5b
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ProjectComplexRexNodeCorrelateTransposeRule.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Correlate;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.Uncollect;
+import org.apache.calcite.rel.logical.LogicalCorrelate;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.trace.CalciteTrace;
+import org.apache.drill.common.exceptions.UserException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Rule that moves non-{@link RexFieldAccess} rex node from project below {@link Uncollect}
+ * to the left side of the {@link Correlate}.
+ */
+public class ProjectComplexRexNodeCorrelateTransposeRule extends RelOptRule {
+
+ public static final RelOptRule INSTANCE = new ProjectComplexRexNodeCorrelateTransposeRule();
+
+ public ProjectComplexRexNodeCorrelateTransposeRule() {
+ super(operand(LogicalCorrelate.class,
+ operand(RelNode.class, any()),
+ operand(Uncollect.class, operand(LogicalProject.class, any()))),
+ DrillRelFactories.LOGICAL_BUILDER,
+ "ProjectComplexRexNodeCorrelateTransposeRule");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final Correlate correlate = call.rel(0);
+ final Uncollect uncollect = call.rel(2);
+ final LogicalProject project = call.rel(3);
+
+ // uncollect requires project with single expression
+ RexNode projectedNode = project.getProjects().iterator().next();
+
+ // check that the expression is complex call
+ if (!(projectedNode instanceof RexFieldAccess)) {
+ RelBuilder builder = call.builder();
+ RexBuilder rexBuilder = builder.getRexBuilder();
+
+ builder.push(correlate.getLeft());
+
+ // creates project with complex expr on top of the left side
+ List<RexNode> leftProjExprs = new ArrayList<>();
+
+ String complexFieldName = correlate.getRowType().getFieldNames()
+ .get(correlate.getRowType().getFieldNames().size() - 1);
+
+ List<String> fieldNames = new ArrayList<>();
+ for (RelDataTypeField field : correlate.getLeft().getRowType().getFieldList()) {
+ leftProjExprs.add(rexBuilder.makeInputRef(correlate.getLeft(), field.getIndex()));
+ fieldNames.add(field.getName());
+ }
+ fieldNames.add(complexFieldName);
+ List<RexNode> topProjectExpressions = new ArrayList<>(leftProjExprs);
+
+ // adds complex expression with replaced correlation
+ // to the projected list from the left
+ leftProjExprs.add(projectedNode.accept(new RexFieldAccessReplacer(builder)));
+
+ RelNode leftProject = builder.project(leftProjExprs, fieldNames)
+ .build();
+
+ CorrelationId correlationId = correlate.getCluster().createCorrel();
+ RexCorrelVariable rexCorrel =
+ (RexCorrelVariable) rexBuilder.makeCorrel(
+ leftProject.getRowType(),
+ correlationId);
+ builder.push(project.getInput());
+ RelNode rightProject = builder.project(
+ ImmutableList.of(rexBuilder.makeFieldAccess(rexCorrel, leftProjExprs.size() - 1)),
+ ImmutableList.of(complexFieldName))
+ .build();
+
+ int requiredColumnsCount = correlate.getRequiredColumns().cardinality();
+ if (requiredColumnsCount != 1) {
+ throw UserException.planError()
+ .message("Required columns count for Correlate operator " +
+ "differs from the expected value:\n" +
+ "Expected columns count is %s, but actual is %s",
+ 1, requiredColumnsCount)
+ .build(CalciteTrace.getPlannerTracer());
+ }
+
+ RelNode newUncollect = uncollect.copy(uncollect.getTraitSet(), rightProject);
+ Correlate newCorrelate = correlate.copy(uncollect.getTraitSet(), leftProject, newUncollect,
+ correlationId, ImmutableBitSet.of(leftProjExprs.size() - 1), correlate.getJoinType());
+ builder.push(newCorrelate);
+
+ switch(correlate.getJoinType()) {
+ case LEFT:
+ case INNER:
+ // adds field from the right input of correlate to the top project
+ topProjectExpressions.add(
+ rexBuilder.makeInputRef(newCorrelate, topProjectExpressions.size() + 1));
+ // fall through
+ case ANTI:
+ case SEMI:
+ builder.project(topProjectExpressions, correlate.getRowType().getFieldNames());
+ }
+
+ call.transformTo(builder.build());
+ }
+ }
+
+ /**
+ * Visitor for RexNode which replaces {@link RexFieldAccess}
+ * with a reference to the field used in {@link RexFieldAccess}.
+ */
+ private static class RexFieldAccessReplacer extends RexShuttle {
+ private final RelBuilder builder;
+
+ public RexFieldAccessReplacer(RelBuilder builder) {
+ this.builder = builder;
+ }
+
+ @Override
+ public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+ return builder.field(fieldAccess.getField().getName());
+ }
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
index b55076b..b10eff0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
@@ -88,11 +88,12 @@ public class LateralJoinPrel extends DrillLateralJoinRelBase implements Prel {
* Check to make sure that the fields of the inputs are the same as the output field names.
* If not, insert a project renaming them.
*/
- public RelNode getLateralInput(int offset, RelNode input) {
+ public RelNode getLateralInput(int ordinal, RelNode input) {
+ int offset = ordinal == 0 ? 0 : getInputSize(0);
Preconditions.checkArgument(DrillJoinRelBase.uniqueFieldNames(input.getRowType()));
final List<String> fields = getRowType().getFieldNames();
final List<String> inputFields = input.getRowType().getFieldNames();
- final List<String> outputFields = fields.subList(offset, offset + getInputSize(offset, input));
+ final List<String> outputFields = fields.subList(offset, offset + getInputSize(ordinal));
if (ListUtils.subtract(outputFields, inputFields).size() != 0) {
// Ensure that input field names are the same as output field names.
// If there are duplicate field names on left and right, fields will get
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java
index 48f4ea9..544a628 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.planner.physical;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexNode;
import org.apache.drill.exec.planner.logical.DrillUnnestRel;
import org.apache.drill.exec.planner.logical.RelOptHelper;
@@ -34,10 +33,6 @@ public class UnnestPrule extends Prule {
public void onMatch(RelOptRuleCall call) {
final DrillUnnestRel unnest = call.rel(0);
RexNode ref = unnest.getRef();
- if (ref instanceof RexFieldAccess) {
- final RexFieldAccess field = (RexFieldAccess)ref;
- field.getField().getName();
- }
UnnestPrel unnestPrel = new UnnestPrel(unnest.getCluster(),
unnest.getTraitSet().plus(Prel.DRILL_PHYSICAL), unnest.getRowType(), ref);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java
index 850f0bd..3a2529b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.physical.visitor;
+import java.util.ArrayList;
import java.util.List;
import org.apache.drill.exec.planner.physical.JoinPrel;
@@ -75,16 +76,11 @@ public class JoinPrelRenameVisitor extends BasePrelVisitor<Prel, Void, RuntimeEx
public Prel visitLateral(LateralJoinPrel prel, Void value) throws RuntimeException {
List<RelNode> children = getChildren(prel);
+ List<RelNode> reNamedChildren = new ArrayList<>();
- final int leftCount = prel.getInputSize(0,children.get(0));
-
- List<RelNode> reNamedChildren = Lists.newArrayList();
-
- RelNode left = prel.getLateralInput(0, children.get(0));
- RelNode right = prel.getLateralInput(leftCount, children.get(1));
-
- reNamedChildren.add(left);
- reNamedChildren.add(right);
+ for (int i = 0; i < children.size(); i++) {
+ reNamedChildren.add(prel.getLateralInput(i, children.get(i)));
+ }
return preparePrel(prel, reNamedChildren);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
index 4d0f34c..ded85c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlJoin;
+import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
@@ -31,30 +32,68 @@ import org.apache.calcite.sql.util.SqlShuttle;
import org.apache.calcite.sql.util.SqlVisitor;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
/**
- * Implementation of {@link SqlVisitor} that converts bracketed compound {@link SqlIdentifier} to bracket-less compound
- * {@link SqlIdentifier} (also known as {@link DrillCompoundIdentifier}) to provide ease of use while querying complex
- * types.
+ * Implementation of {@link SqlVisitor} that converts bracketed compound {@link SqlIdentifier}
+ * to bracket-less compound {@link SqlIdentifier} (also known as {@link DrillCompoundIdentifier})
+ * to provide ease of use while querying complex types.
* <p/>
* For example, this visitor converts {@code a['b'][4]['c']} to {@code a.b[4].c}
*/
public class CompoundIdentifierConverter extends SqlShuttle {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompoundIdentifierConverter.class);
+ /**
+ * This map stores the rules that instruct each SqlCall class which data field needs
+ * to be rewritten if that data field is a {@link DrillCompoundIdentifier}.
+ * <p/>
+ * <ul>
+ * <li>Key : Each rule corresponds to a {@link SqlCall} class;
+ * <li>Value: It is an array of {@link RewriteType}, each being associated with a data field
+ * in that class.
+ * </ul>
+ * <p/>
+ * For example, there are four data fields (query, orderList, offset, fetch)
+ * in {@link SqlOrderBy}. Since only orderList needs to be written,
+ * {@link RewriteType[]} should be {@code arrayOf(D, E, D, D)}.
+ */
+ private static final Map<Class<? extends SqlCall>, RewriteType[]> REWRITE_RULES;
+
+ static {
+ final RewriteType E = RewriteType.ENABLE;
+ final RewriteType D = RewriteType.DISABLE;
+
+ // Every element of the array corresponds to the item in the list
+ // returned by getOperandList() method for concrete SqlCall implementation.
+ REWRITE_RULES = ImmutableMap.<Class<? extends SqlCall>, RewriteType[]>builder()
+ .put(SqlSelect.class, arrayOf(D, E, D, E, E, E, E, E, D, D))
+ .put(SqlCreateTable.class, arrayOf(D, D, D, E, D, D))
+ .put(SqlCreateView.class, arrayOf(D, E, E, D))
+ .put(DrillSqlDescribeTable.class, arrayOf(D, D, E))
+ .put(SqlDropView.class, arrayOf(D, D))
+ .put(SqlShowFiles.class, arrayOf(D))
+ .put(SqlShowSchemas.class, arrayOf(D, D))
+ .put(SqlUseSchema.class, arrayOf(D))
+ .put(SqlJoin.class, arrayOf(D, D, D, D, D, E))
+ .put(SqlOrderBy.class, arrayOf(D, E, D, D))
+ .put(SqlDropTable.class, arrayOf(D, D))
+ .put(SqlRefreshMetadata.class, arrayOf(D))
+ .put(SqlSetOption.class, arrayOf(D, D, D))
+ .put(SqlCreateFunction.class, arrayOf(D))
+ .put(SqlDropFunction.class, arrayOf(D))
+ .build();
+ }
private boolean enableComplex = true;
@Override
public SqlNode visit(SqlIdentifier id) {
- if(id instanceof DrillCompoundIdentifier){
- if(enableComplex){
- return ((DrillCompoundIdentifier) id).getAsSqlNode();
- }else{
- return ((DrillCompoundIdentifier) id).getAsCompoundIdentifier();
+ if (id instanceof DrillCompoundIdentifier) {
+ DrillCompoundIdentifier compoundIdentifier = (DrillCompoundIdentifier) id;
+ if (enableComplex) {
+ return compoundIdentifier.getAsSqlNode();
+ } else {
+ return compoundIdentifier.getAsCompoundIdentifier();
}
-
- }else{
+ } else {
return id;
}
}
@@ -64,22 +103,46 @@ public class CompoundIdentifierConverter extends SqlShuttle {
// Handler creates a new copy of 'call' only if one or more operands
// change.
ArgHandler<SqlNode> argHandler = new ComplexExpressionAware(call);
+ boolean localEnableComplex = enableComplex;
+ // for the case of UNNEST call set enableComplex to true
+ // to convert DrillCompoundIdentifier to the item call.
+ if (call.getKind() == SqlKind.UNNEST) {
+ enableComplex = true;
+ }
call.getOperator().acceptCall(this, call, false, argHandler);
+ enableComplex = localEnableComplex;
return argHandler.result();
}
+ /**
+ * Constructs array which contains specified parameters.
+ */
+ private static RewriteType[] arrayOf(RewriteType... types) {
+ return types;
+ }
+
+ enum RewriteType {
+ UNCHANGED, DISABLE, ENABLE
+ }
+
+ /**
+ * Argument handler which accepts {@link CompoundIdentifierConverter}
+ * for every operand of {@link SqlCall} and constructs new {@link SqlCall}
+ * if one or more operands changed.
+ */
+ private class ComplexExpressionAware implements ArgHandler<SqlNode> {
- private class ComplexExpressionAware implements ArgHandler<SqlNode> {
- boolean update;
- SqlNode[] clonedOperands;
- RewriteType[] rewriteTypes;
private final SqlCall call;
+ private final SqlNode[] clonedOperands;
+ private final RewriteType[] rewriteTypes;
+
+ private boolean update;
public ComplexExpressionAware(SqlCall call) {
this.call = call;
this.update = false;
final List<SqlNode> operands = call.getOperandList();
- this.clonedOperands = operands.toArray(new SqlNode[operands.size()]);
+ this.clonedOperands = operands.toArray(new SqlNode[0]);
rewriteTypes = REWRITE_RULES.get(call.getClass());
}
@@ -106,13 +169,13 @@ public class CompoundIdentifierConverter extends SqlShuttle {
}
boolean localEnableComplex = enableComplex;
- if(rewriteTypes != null){
- switch(rewriteTypes[i]){
- case DISABLE:
- enableComplex = false;
- break;
- case ENABLE:
- enableComplex = true;
+ if (rewriteTypes != null) {
+ switch (rewriteTypes[i]) {
+ case DISABLE:
+ enableComplex = false;
+ break;
+ case ENABLE:
+ enableComplex = true;
}
}
SqlNode newOperand = operand.accept(CompoundIdentifierConverter.this);
@@ -124,64 +187,4 @@ public class CompoundIdentifierConverter extends SqlShuttle {
return newOperand;
}
}
-
- static final Map<Class<? extends SqlCall>, RewriteType[]> REWRITE_RULES;
-
- enum RewriteType {
- UNCHANGED, DISABLE, ENABLE;
- }
-
- static {
- final RewriteType E =RewriteType.ENABLE;
- final RewriteType D =RewriteType.DISABLE;
- final RewriteType U =RewriteType.UNCHANGED;
-
- /*
- This map stores the rules that instruct each SqlCall class which data field needs
- to be rewritten if that data field is a CompoundIdentifier
-
- Key : Each rule corresponds to a SqlCall class;
- value: It is an array of RewriteType, each being associated with a data field
- in that class.
-
- For example, there are four data fields (query, orderList, offset, fetch)
- in org.eigenbase.sql.SqlOrderBy. Since only orderList needs to be written,
- RewriteType[] should be R(D, E, D, D).
- */
- Map<Class<? extends SqlCall>, RewriteType[]> rules = Maps.newHashMap();
-
- //SqlNodeList keywordList,
- //SqlNodeList selectList,
- //SqlNode fromClause,
- //SqlNode whereClause,
- //SqlNodeList groupBy,
- //SqlNode having,
- //SqlNodeList windowDecls,
- //SqlNodeList orderBy,
- //SqlNode offset,
- //SqlNode fetch,
- rules.put(SqlSelect.class, R(D, E, D, E, E, E, E, E, D, D));
- rules.put(SqlCreateTable.class, R(D, D, D, E, D, D));
- rules.put(SqlCreateView.class, R(D, E, E, D));
- rules.put(DrillSqlDescribeTable.class, R(D, D, E));
- rules.put(SqlDropView.class, R(D, D));
- rules.put(SqlShowFiles.class, R(D));
- rules.put(SqlShowSchemas.class, R(D, D));
- rules.put(SqlUseSchema.class, R(D));
- rules.put(SqlJoin.class, R(D, D, D, D, D, E));
- rules.put(SqlOrderBy.class, R(D, E, D, D));
- rules.put(SqlDropTable.class, R(D, D));
- rules.put(SqlRefreshMetadata.class, R(D));
- rules.put(SqlSetOption.class, R(D, D, D));
- rules.put(SqlCreateFunction.class, R(D));
- rules.put(SqlDropFunction.class, R(D));
- REWRITE_RULES = ImmutableMap.copyOf(rules);
- }
-
- // Each type in the input arguments refers to
- // each data field in the class
- private static RewriteType[] R(RewriteType... types){
- return types;
- }
-
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
index 6bf3f9a..98f051e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
@@ -21,6 +21,7 @@ import org.apache.drill.categories.OperatorTest;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
+import org.apache.drill.test.TestBuilder;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -167,6 +168,93 @@ public class TestE2EUnnestAndLateral extends ClusterTest {
}
@Test
+ public void testUnnestWithItem() throws Exception {
+ String sql = "select u.item from\n" +
+ "cp.`lateraljoin/nested-customer.parquet` c," +
+ "unnest(c.orders['items']) as u(item)\n" +
+ "limit 1";
+
+ testBuilder()
+ .sqlQuery(sql)
+ .unOrdered()
+ .baselineColumns("item")
+ .baselineValues(
+ TestBuilder.mapOf("i_name", "paper towel",
+ "i_number", 2.0,
+ "i_supplier", "oregan"))
+ .go();
+ }
+
+ @Test
+ public void testUnnestWithFunctionCall() throws Exception {
+ String sql = "select u.ord.o_amount o_amount from\n" +
+ "cp.`lateraljoin/nested-customer.parquet` c," +
+ "unnest(convert_fromjson(convert_tojson(c.orders))) as u(ord)\n" +
+ "limit 1";
+
+ testBuilder()
+ .sqlQuery(sql)
+ .unOrdered()
+ .baselineColumns("o_amount")
+ .baselineValues(4.5)
+ .go();
+ }
+
+ @Test
+ public void testUnnestWithMap() throws Exception {
+ String sql = "select u.item from\n" +
+ "cp.`lateraljoin/nested-customer.parquet` c," +
+ "unnest(c.orders.items) as u(item)\n" +
+ "limit 1";
+
+ testBuilder()
+ .sqlQuery(sql)
+ .unOrdered()
+ .baselineColumns("item")
+ .baselineValues(
+ TestBuilder.mapOf("i_name", "paper towel",
+ "i_number", 2.0,
+ "i_supplier", "oregan"))
+ .go();
+ }
+
+ @Test
+ public void testMultiUnnestWithMap() throws Exception {
+ String sql = "select u.item from\n" +
+ "cp.`lateraljoin/nested-customer.parquet` c," +
+ "unnest(c.orders.items) as u(item)," +
+ "unnest(c.orders.items) as u1(item1)\n" +
+ "limit 1";
+
+ testBuilder()
+ .sqlQuery(sql)
+ .unOrdered()
+ .baselineColumns("item")
+ .baselineValues(
+ TestBuilder.mapOf("i_name", "paper towel",
+ "i_number", 2.0,
+ "i_supplier", "oregan"))
+ .go();
+ }
+
+ @Test
+ public void testSingleUnnestCol() throws Exception {
+ String sql =
+ "select t.orders.o_id as id " +
+ "from (select u.orders from\n" +
+ "cp.`lateraljoin/nested-customer.parquet` c," +
+ "unnest(c.orders) as u(orders)\n" +
+ "limit 1) t";
+
+ testBuilder()
+ .sqlQuery(sql)
+ .unOrdered()
+ .baselineColumns("id")
+ .baselineValues(1.0)
+ .go();
+ }
+
+ @Test
public void testNestedUnnest() throws Exception {
String Sql = "select * from (select customer.orders as orders from cp.`lateraljoin/nested-customer.parquet` customer ) t1," +
" lateral ( select t.ord.items as items from unnest(t1.orders) t(ord) ) t2, unnest(t2.items) t3(item) ";