You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by GitBox <gi...@apache.org> on 2018/08/04 01:01:36 UTC
[GitHub] ilooner closed pull request #1417: DRILL-6645: Transform TopN in
Lateral Unnest pipeline to Sort and Limit.
ilooner closed pull request #1417: DRILL-6645: Transform TopN in Lateral Unnest pipeline to Sort and Limit.
URL: https://github.com/apache/drill/pull/1417
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
index a4f51f315eb..ca68a7d1a44 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java
@@ -189,7 +189,7 @@ public boolean needsFinalColumnReordering() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
List<Integer> groupingCols = Lists.newArrayList();
groupingCols.add(0);
for (int groupingCol : groupSet.asList()) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
index 1c9112c5673..33c29448432 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
@@ -85,7 +85,7 @@ public boolean needsFinalColumnReordering() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
RexBuilder builder = this.getCluster().getRexBuilder();
// right shift the previous field indices.
return (Prel) this.copy(this.traitSet, children.get(0), DrillRelOptUtil.transformExpr(builder,
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
index 057cfaed2c4..ccbff17f86a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java
@@ -111,7 +111,7 @@ public boolean needsFinalColumnReordering() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
return new LimitPrel(this.getCluster(), this.traitSet, children.get(0), getOffset(), getFetch(), isPushDown(), true);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
index b72aff70b9c..01d8e9c33be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
@@ -56,7 +56,13 @@ public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits,
SelectionVectorMode getEncoding();
boolean needsFinalColumnReordering();
- default Prel addImplicitRowIDCol(List<RelNode> children) {
+ /**
+ * If the operator is in Lateral/Unnest pipeline, then it generates a new operator which knows how to process
+ * the rows accordingly during execution.
+ * eg: TopNPrel -> SortPrel and LimitPrel
+ * Other operators like FilterPrel, ProjectPrel etc will add an implicit row id to the output.
+ */
+ default Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
throw new UnsupportedOperationException("Adding Implicit RowID column is not supported for " +
this.getClass().getSimpleName() + " operator ");
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
index 0a9e8bf1aeb..4d5de20d583 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
@@ -136,7 +136,7 @@ public boolean needsFinalColumnReordering() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
RelDataTypeFactory typeFactory = this.getCluster().getTypeFactory();
RexBuilder builder = this.getCluster().getRexBuilder();
List<RexNode> projects = Lists.newArrayList();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
index a4cd9211d0c..3fad0172842 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorRemoverPrel.java
@@ -55,7 +55,7 @@ public SelectionVectorMode getEncoding() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
return (Prel) this.copy(this.traitSet, children);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
index 686e04a4031..aa7158a683d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
@@ -124,7 +124,7 @@ public boolean needsFinalColumnReordering() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
List<RelFieldCollation> relFieldCollations = Lists.newArrayList();
relFieldCollations.add(new RelFieldCollation(0,
RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
index 9bdcad06b9d..3e407f74678 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
@@ -18,11 +18,14 @@
package org.apache.drill.exec.planner.physical;
import java.io.IOException;
+import java.math.BigDecimal;
import java.util.List;
import com.google.common.collect.Lists;
import org.apache.calcite.rel.RelCollationImpl;
import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.config.TopN;
import org.apache.drill.exec.planner.cost.DrillCostBase;
@@ -101,7 +104,7 @@ public SelectionVectorMode getEncoding() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
List<RelFieldCollation> relFieldCollations = Lists.newArrayList();
relFieldCollations.add(new RelFieldCollation(0,
RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST));
@@ -115,6 +118,17 @@ public Prel addImplicitRowIDCol(List<RelNode> children) {
.replace(this.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE))
.replace(collationTrait)
.replace(DRILL_PHYSICAL);
- return (Prel) this.copy(traits, children);
+ return transformTopNToSortAndLimit(children, traits, collationTrait);
+ }
+
+ private Prel transformTopNToSortAndLimit(List<RelNode> children, RelTraitSet traits, RelCollation collationTrait) {
+ SortPrel sortprel = new SortPrel(this.getCluster(), traits, children.get(0), collationTrait);
+ RexNode offset = this.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.valueOf(0),
+ this.getCluster().getTypeFactory().createSqlType(SqlTypeName.INTEGER));
+ RexNode limit = this.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.valueOf(this.limit),
+ this.getCluster().getTypeFactory().createSqlType(SqlTypeName.INTEGER));
+ //SMEX is not needed here because Lateral/Unnest pipeline doesn't support exchanges.
+ LimitPrel limitPrel = new LimitPrel(this.getCluster(), traits, sortprel, offset, limit, false, true);
+ return limitPrel;
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
index 23311383242..274f27a2e0b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
@@ -86,7 +86,7 @@ public boolean needsFinalColumnReordering() {
}
@Override
- public Prel addImplicitRowIDCol(List<RelNode> children) {
+ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
RelDataTypeFactory typeFactory = this.getCluster().getTypeFactory();
List<String> fieldNames = new ArrayList<>();
List<RelDataType> fieldTypes = new ArrayList<>();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
index 469220252eb..dc4af5b08ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
@@ -43,7 +43,7 @@ public static Prel insertRowID(Prel prel){
public Prel visitPrel(Prel prel, Boolean isRightOfLateral) throws RuntimeException {
List<RelNode> children = getChildren(prel, isRightOfLateral);
if (isRightOfLateral) {
- return prel.addImplicitRowIDCol(children);
+ return prel.prepareForLateralUnnestPipeline(children);
} else {
return (Prel) prel.copy(prel.getTraitSet(), children);
}
@@ -61,7 +61,7 @@ public Prel visitPrel(Prel prel, Boolean isRightOfLateral) throws RuntimeExcepti
@Override
public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws RuntimeException {
List<RelNode> children = Lists.newArrayList();
- children.add(((Prel)prel.getInput(0)).accept(this, false));
+ children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
return (Prel) prel.copy(prel.getTraitSet(), children);
@@ -69,6 +69,6 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws RuntimeExce
@Override
public Prel visitUnnest(UnnestPrel prel, Boolean value) throws RuntimeException {
- return prel.addImplicitRowIDCol(null);
+ return prel.prepareForLateralUnnestPipeline(null);
}
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services