You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/03/19 00:51:31 UTC
svn commit: r1667641 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java
ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
Author: gunther
Date: Wed Mar 18 23:51:31 2015
New Revision: 1667641
URL: http://svn.apache.org/r1667641
Log:
HIVE-9984: JoinReorder's getOutputSize is exponential (Gopal V and Gunther Hagleitner)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1667641&r1=1667640&r2=1667641&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Mar 18 23:51:31 2015
@@ -2034,7 +2034,9 @@ public class HiveConf extends Configurat
SPARK_RPC_CHANNEL_LOG_LEVEL("hive.spark.client.channel.log.level", null,
"Channel logging level for remote Spark driver. One of {DEBUG, ERROR, INFO, TRACE, WARN}."),
SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5",
- "Name of the SASL mechanism to use for authentication.");
+ "Name of the SASL mechanism to use for authentication."),
+ NWAYJOINREORDER("hive.reorder.nway.joins", true,
+ "Runs reordering of tables within single n-way join (i.e.: picks streamtable)");
public final String varname;
private final String defaultExpr;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java?rev=1667641&r1=1667640&r2=1667641&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java Wed Mar 18 23:51:31 2015
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.optimizer;
import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -36,6 +38,8 @@ import org.apache.hadoop.hive.ql.plan.Op
* implemented, this transformation can also be done based on costs.
*/
public class JoinReorder implements Transform {
+
+ private final Map<Operator<?>, Integer> cache = new IdentityHashMap<Operator<?>, Integer>();
/**
* Estimate the size of the output based on the STREAMTABLE hints. To do so
* the whole tree is traversed. Possible sizes: 0: the operator and its
@@ -49,8 +53,25 @@ public class JoinReorder implements Tran
* @return The estimated size - 0 (no streamed tables), 1 (streamed tables in
* subtree) or 2 (a streamed table)
*/
+
private int getOutputSize(Operator<? extends OperatorDesc> operator,
Set<String> bigTables) {
+
+ // memoize decorator for getOutputSizeInternal
+ if (cache.containsKey(operator)) {
+ return cache.get(operator);
+ }
+
+ int result = getOutputSizeInternal(operator, bigTables);
+
+ cache.put(operator, result);
+
+ return result;
+ }
+
+ private int getOutputSizeInternal(Operator<? extends OperatorDesc> operator,
+ Set<String> bigTables) {
+
// If a join operator contains a big subtree, there is a chance that its
// output is also big, so the output size is 1 (medium)
if (operator instanceof JoinOperator) {
@@ -74,6 +95,7 @@ public class JoinReorder implements Tran
int maxSize = 0;
if (operator.getParentOperators() != null) {
for (Operator<? extends OperatorDesc> o : operator.getParentOperators()) {
+ // recurse into memoized decorator
int current = getOutputSize(o, bigTables);
if (current > maxSize) {
maxSize = current;
@@ -151,8 +173,10 @@ public class JoinReorder implements Tran
* @param pactx
* current parse context
*/
+ @Override
public ParseContext transform(ParseContext pactx) throws SemanticException {
Set<String> bigTables = getBigTables(pactx);
+ cache.clear();
for (JoinOperator joinOp : pactx.getJoinOps()) {
reorder(joinOp, bigTables);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=1667641&r1=1667640&r2=1667641&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Wed Mar 18 23:51:31 2015
@@ -128,7 +128,11 @@ public class Optimizer {
}
transformations.add(new UnionProcessor());
- transformations.add(new JoinReorder());
+
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.NWAYJOINREORDER)) {
+ transformations.add(new JoinReorder());
+ }
+
if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) &&
HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") &&
HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) &&