You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2020/04/06 17:56:06 UTC
[hive] branch master updated (2de6ae8 -> 182dac3)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from 2de6ae8 HIVE-23136 Do not compare q test result if test.output.overwrite is specified (Miklos Gergely, reviewed by Laszlo Bodor)
new 1fcbe91 HIVE-23119: Test sort_acid should be run by TestMiniLlapLocalCliDriver only (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
new 182dac3 HIVE-23135: Add RelDistribution trait to HiveSortExchange (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
itests/src/test/resources/testconfiguration.properties | 3 +--
.../hadoop/hive/ql/optimizer/calcite/TraitsUtil.java | 4 +++-
.../ql/optimizer/calcite/cost/HiveVolcanoPlanner.java | 6 +++++-
.../calcite/reloperators/HiveSortExchange.java | 10 +++++-----
.../rules/HiveProjectSortExchangeTransposeRule.java | 17 +++++++++--------
5 files changed, 23 insertions(+), 17 deletions(-)
[hive] 01/02: HIVE-23119: Test sort_acid should be run by
TestMiniLlapLocalCliDriver only (Krisztian Kasa,
reviewed by Jesus Camacho Rodriguez)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 1fcbe9157b4704a8152eccfce94916bd9e29e2e6
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Mon Apr 6 10:44:09 2020 -0700
HIVE-23119: Test sort_acid should be run by TestMiniLlapLocalCliDriver only (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
---
itests/src/test/resources/testconfiguration.properties | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index af8e961..e99ce7b 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -15,7 +15,6 @@ disabled.query.files=cbo_rp_subq_in.q,\
sample2.q,\
sample4.q,\
root_dir_external_table.q,\
- sort_acid.q,\
input31.q
@@ -208,7 +207,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
selectDistinctStar.q,\
select_dummy_source.q,\
skewjoin.q,\
- sort_acid.q,\
stats_noscan_1.q,\
stats_only_null.q,\
subquery_exists.q,\
@@ -792,6 +790,7 @@ minillaplocal.query.files=\
sharedworkext.q,\
sharedworkresidual.q,\
smb_cache.q,\
+ sort_acid.q,\
special_character_in_tabnames_1.q,\
sqlmerge.q,\
sqlmerge_stats.q,\
[hive] 02/02: HIVE-23135: Add RelDistribution trait to
HiveSortExchange (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 182dac38a472da9fff8be1acb9d250ad5faaa87a
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Mon Apr 6 10:49:08 2020 -0700
HIVE-23135: Add RelDistribution trait to HiveSortExchange (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
---
.../hadoop/hive/ql/optimizer/calcite/TraitsUtil.java | 4 +++-
.../ql/optimizer/calcite/cost/HiveVolcanoPlanner.java | 6 +++++-
.../calcite/reloperators/HiveSortExchange.java | 10 +++++-----
.../rules/HiveProjectSortExchangeTransposeRule.java | 17 +++++++++--------
4 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
index 8b82241..c41a8c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
@@ -23,6 +23,7 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelDistributions;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
public class TraitsUtil {
@@ -32,7 +33,8 @@ public class TraitsUtil {
}
public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) {
- return cluster.traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY);
+ return cluster.traitSet()
+ .replace(HiveRelNode.CONVENTION).replace(RelCollations.EMPTY).replace(RelDistributions.ANY);
}
public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster, RelTraitSet traitsFromInput) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java
index 5dcbff6..208fb15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java
@@ -28,10 +28,10 @@ import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelDistributionTraitDef;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
@@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules;
*/
public class HiveVolcanoPlanner extends VolcanoPlanner {
private static final boolean ENABLE_COLLATION_TRAIT = true;
+ private static final boolean ENABLE_DISTRIBUTION_TRAIT = true;
private final boolean isHeuristic;
@@ -62,6 +63,9 @@ public class HiveVolcanoPlanner extends VolcanoPlanner {
if (ENABLE_COLLATION_TRAIT) {
planner.addRelTraitDef(RelCollationTraitDef.INSTANCE);
}
+ if (ENABLE_DISTRIBUTION_TRAIT) {
+ planner.addRelTraitDef(RelDistributionTraitDef.INSTANCE);
+ }
return planner;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
index 8c46e44..ce794cb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
@@ -61,13 +61,13 @@ public final class HiveSortExchange extends SortExchange implements HiveRelNode
RelOptCluster cluster = input.getCluster();
distribution = RelDistributionTraitDef.INSTANCE.canonize(distribution);
collation = RelCollationTraitDef.INSTANCE.canonize(collation);
- RelTraitSet traitSet = getTraitSet(collation, cluster);
+ RelTraitSet traitSet = getTraitSet(cluster, collation, distribution);
return new HiveSortExchange(cluster, traitSet, input, distribution, collation, keys);
}
- private static RelTraitSet getTraitSet(RelCollation collation, RelOptCluster cluster) {
- // add distribution
- return TraitsUtil.getDefaultTraitSet(cluster).replace(collation);
+ private static RelTraitSet getTraitSet(
+ RelOptCluster cluster, RelCollation collation, RelDistribution distribution) {
+ return TraitsUtil.getDefaultTraitSet(cluster).replace(collation).replace(distribution);
}
public static HiveSortExchange create(RelNode input,
@@ -75,7 +75,7 @@ public final class HiveSortExchange extends SortExchange implements HiveRelNode
RelOptCluster cluster = input.getCluster();
distribution = RelDistributionTraitDef.INSTANCE.canonize(distribution);
collation = RelCollationTraitDef.INSTANCE.canonize(collation);
- RelTraitSet traitSet = getTraitSet(collation, cluster);
+ RelTraitSet traitSet = getTraitSet(cluster, collation, distribution);
RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(collation.getFieldCollations()));
ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
index 06cbe3d..d40d5c4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
@@ -29,6 +29,9 @@ import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelDistribution;
+import org.apache.calcite.rel.RelDistributionTraitDef;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.SortExchange;
@@ -83,19 +86,17 @@ public final class HiveProjectSortExchangeTransposeRule extends RelOptRule {
return;
}
- RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster());
- RelCollation newCollation = newTraitSet.canonize(RelCollationImpl.of(fieldCollations));
- newTraitSet = newTraitSet.replace(newCollation);
+ RelCollation newCollation = RelCollationTraitDef.INSTANCE.canonize(RelCollationImpl.of(fieldCollations));
List<Integer> newDistributionKeys = getNewRelDistributionKeys(project, sortExchange.getDistribution());
+ RelDistribution newDistribution = RelDistributionTraitDef.INSTANCE.canonize(
+ new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys));
+ RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster())
+ .replace(newCollation).replace(newDistribution);
// New operators
final RelNode newProject = project.copy(sortExchange.getInput().getTraitSet(),
ImmutableList.of(sortExchange.getInput()));
- final SortExchange newSort = sortExchange.copy(
- newTraitSet,
- newProject,
- new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys),
- newCollation);
+ final SortExchange newSort = sortExchange.copy(newTraitSet, newProject, newDistribution, newCollation);
call.transformTo(newSort);
}