You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/05/05 15:23:29 UTC
hive git commit: HIVE-17978: Shared work optimizer may leave useless
operator branches in the plan (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master a84fff8ab -> 0fc67416e
HIVE-17978: Shared work optimizer may leave useless operator branches in the plan (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0fc67416
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0fc67416
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0fc67416
Branch: refs/heads/master
Commit: 0fc67416e7b19ca9b9e7a7c019809223a60e3d6b
Parents: a84fff8
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu May 3 22:42:41 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Sat May 5 08:21:49 2018 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/SharedWorkOptimizer.java | 33 +++++++++++++-------
1 file changed, 21 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0fc67416/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index b60512b..aff5520 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -909,8 +909,8 @@ public class SharedWorkOptimizer extends Transform {
}
}
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache,
+ Sets.union(discardableInputOps, discardableOps)));
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps,
discardableInputOps));
return new SharedResult(retainableOps, discardableOps, discardableInputOps,
@@ -947,11 +947,7 @@ public class SharedWorkOptimizer extends Transform {
.get((TableScanOperator) op);
for (Operator<?> dppSource : c) {
// Remove the branches
- Operator<?> currentOp = dppSource;
- while (currentOp.getNumChild() <= 1) {
- dppBranches.add(currentOp);
- currentOp = currentOp.getParentOperators().get(0);
- }
+ removeBranch(dppSource, dppBranches, ops);
}
}
}
@@ -971,11 +967,7 @@ public class SharedWorkOptimizer extends Transform {
findAscendantWorkOperators(pctx, optimizerCache, dppSource);
if (!Collections.disjoint(ascendants, discardedOps)) {
// Remove branch
- Operator<?> currentOp = dppSource;
- while (currentOp.getNumChild() <= 1) {
- dppBranches.add(currentOp);
- currentOp = currentOp.getParentOperators().get(0);
- }
+ removeBranch(dppSource, dppBranches, ops);
}
}
}
@@ -983,6 +975,23 @@ public class SharedWorkOptimizer extends Transform {
return dppBranches;
}
+ private static void removeBranch(Operator<?> currentOp, Set<Operator<?>> branchesOps,
+ Set<Operator<?>> discardableOps) {
+ if (currentOp.getNumChild() > 1) {
+ for (Operator<?> childOp : currentOp.getChildOperators()) {
+ if (!branchesOps.contains(childOp) && !discardableOps.contains(childOp)) {
+ return;
+ }
+ }
+ }
+ branchesOps.add(currentOp);
+ if (currentOp.getParentOperators() != null) {
+ for (Operator<?> parentOp : currentOp.getParentOperators()) {
+ removeBranch(parentOp, branchesOps, discardableOps);
+ }
+ }
+ }
+
private static List<Operator<?>> compareAndGatherOps(ParseContext pctx,
Operator<?> op1, Operator<?> op2) throws SemanticException {
List<Operator<?>> result = new ArrayList<>();