You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/29 21:23:33 UTC
svn commit: r1628264 -
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Author: gunther
Date: Mon Sep 29 19:23:33 2014
New Revision: 1628264
URL: http://svn.apache.org/r1628264
Log:
HIVE-8281: NPE with dynamic partition pruning on Tez (Gunther Hagleitner, reviewed by Vikram Dixit K)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1628264&r1=1628263&r2=1628264&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Mon Sep 29 19:23:33 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
@@ -619,15 +620,31 @@ public class ConvertJoinMapJoin implemen
return mapJoinOp;
}
- private boolean hasDynamicPartitionBroadcast(Operator<?> op) {
- if (op instanceof AppMasterEventOperator && op.getConf() instanceof DynamicPruningEventDesc) {
- return true;
- }
- for (Operator<?> c : op.getChildOperators()) {
- if (hasDynamicPartitionBroadcast(c)) {
- return true;
+ private boolean hasDynamicPartitionBroadcast(Operator<?> parent) {
+ boolean hasDynamicPartitionPruning = false;
+
+ for (Operator<?> op: parent.getChildOperators()) {
+ while (op != null) {
+ if (op instanceof AppMasterEventOperator && op.getConf() instanceof DynamicPruningEventDesc) {
+ // found dynamic partition pruning operator
+ hasDynamicPartitionPruning = true;
+ break;
+ }
+
+ if (op instanceof ReduceSinkOperator || op instanceof FileSinkOperator) {
+ // crossing reduce sink or file sink means the pruning isn't for this parent.
+ break;
+ }
+
+ if (op.getChildOperators().size() != 1) {
+ // dynamic partition pruning pipeline doesn't have multiple children
+ break;
+ }
+
+ op = op.getChildOperators().get(0);
}
}
- return false;
+
+ return hasDynamicPartitionPruning;
}
}