You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2020/04/06 17:56:08 UTC
[hive] 02/02: HIVE-23135: Add RelDistribution trait to
HiveSortExchange (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 182dac38a472da9fff8be1acb9d250ad5faaa87a
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Mon Apr 6 10:49:08 2020 -0700
HIVE-23135: Add RelDistribution trait to HiveSortExchange (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
---
.../hadoop/hive/ql/optimizer/calcite/TraitsUtil.java | 4 +++-
.../ql/optimizer/calcite/cost/HiveVolcanoPlanner.java | 6 +++++-
.../calcite/reloperators/HiveSortExchange.java | 10 +++++-----
.../rules/HiveProjectSortExchangeTransposeRule.java | 17 +++++++++--------
4 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
index 8b82241..c41a8c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
@@ -23,6 +23,7 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelDistributions;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
public class TraitsUtil {
@@ -32,7 +33,8 @@ public class TraitsUtil {
}
public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) {
- return cluster.traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY);
+ return cluster.traitSet()
+ .replace(HiveRelNode.CONVENTION).replace(RelCollations.EMPTY).replace(RelDistributions.ANY);
}
public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster, RelTraitSet traitsFromInput) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java
index 5dcbff6..208fb15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java
@@ -28,10 +28,10 @@ import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelDistributionTraitDef;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
@@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules;
*/
public class HiveVolcanoPlanner extends VolcanoPlanner {
private static final boolean ENABLE_COLLATION_TRAIT = true;
+ private static final boolean ENABLE_DISTRIBUTION_TRAIT = true;
private final boolean isHeuristic;
@@ -62,6 +63,9 @@ public class HiveVolcanoPlanner extends VolcanoPlanner {
if (ENABLE_COLLATION_TRAIT) {
planner.addRelTraitDef(RelCollationTraitDef.INSTANCE);
}
+ if (ENABLE_DISTRIBUTION_TRAIT) {
+ planner.addRelTraitDef(RelDistributionTraitDef.INSTANCE);
+ }
return planner;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
index 8c46e44..ce794cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
@@ -61,13 +61,13 @@ public final class HiveSortExchange extends SortExchange implements HiveRelNode
RelOptCluster cluster = input.getCluster();
distribution = RelDistributionTraitDef.INSTANCE.canonize(distribution);
collation = RelCollationTraitDef.INSTANCE.canonize(collation);
- RelTraitSet traitSet = getTraitSet(collation, cluster);
+ RelTraitSet traitSet = getTraitSet(cluster, collation, distribution);
return new HiveSortExchange(cluster, traitSet, input, distribution, collation, keys);
}
- private static RelTraitSet getTraitSet(RelCollation collation, RelOptCluster cluster) {
- // add distribution
- return TraitsUtil.getDefaultTraitSet(cluster).replace(collation);
+ private static RelTraitSet getTraitSet(
+ RelOptCluster cluster, RelCollation collation, RelDistribution distribution) {
+ return TraitsUtil.getDefaultTraitSet(cluster).replace(collation).replace(distribution);
}
public static HiveSortExchange create(RelNode input,
@@ -75,7 +75,7 @@ public final class HiveSortExchange extends SortExchange implements HiveRelNode
RelOptCluster cluster = input.getCluster();
distribution = RelDistributionTraitDef.INSTANCE.canonize(distribution);
collation = RelCollationTraitDef.INSTANCE.canonize(collation);
- RelTraitSet traitSet = getTraitSet(collation, cluster);
+ RelTraitSet traitSet = getTraitSet(cluster, collation, distribution);
RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(collation.getFieldCollations()));
ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
index 06cbe3d..d40d5c4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
@@ -29,6 +29,9 @@ import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelDistribution;
+import org.apache.calcite.rel.RelDistributionTraitDef;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.SortExchange;
@@ -83,19 +86,17 @@ public final class HiveProjectSortExchangeTransposeRule extends RelOptRule {
return;
}
- RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster());
- RelCollation newCollation = newTraitSet.canonize(RelCollationImpl.of(fieldCollations));
- newTraitSet = newTraitSet.replace(newCollation);
+ RelCollation newCollation = RelCollationTraitDef.INSTANCE.canonize(RelCollationImpl.of(fieldCollations));
List<Integer> newDistributionKeys = getNewRelDistributionKeys(project, sortExchange.getDistribution());
+ RelDistribution newDistribution = RelDistributionTraitDef.INSTANCE.canonize(
+ new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys));
+ RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster())
+ .replace(newCollation).replace(newDistribution);
// New operators
final RelNode newProject = project.copy(sortExchange.getInput().getTraitSet(),
ImmutableList.of(sortExchange.getInput()));
- final SortExchange newSort = sortExchange.copy(
- newTraitSet,
- newProject,
- new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys),
- newCollation);
+ final SortExchange newSort = sortExchange.copy(newTraitSet, newProject, newDistribution, newCollation);
call.transformTo(newSort);
}