You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/03/19 00:51:31 UTC

svn commit: r1667641 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java

Author: gunther
Date: Wed Mar 18 23:51:31 2015
New Revision: 1667641

URL: http://svn.apache.org/r1667641
Log:
HIVE-9984: JoinReorder's getOutputSize is exponential (Gopal V and Gunther Hagleitner)

Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1667641&r1=1667640&r2=1667641&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Mar 18 23:51:31 2015
@@ -2034,7 +2034,9 @@ public class HiveConf extends Configurat
     SPARK_RPC_CHANNEL_LOG_LEVEL("hive.spark.client.channel.log.level", null,
       "Channel logging level for remote Spark driver.  One of {DEBUG, ERROR, INFO, TRACE, WARN}."),
     SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5",
-      "Name of the SASL mechanism to use for authentication.");
+      "Name of the SASL mechanism to use for authentication."),
+    NWAYJOINREORDER("hive.reorder.nway.joins", true,
+      "Runs reordering of tables within single n-way join (i.e.: picks streamtable)");
 
     public final String varname;
     private final String defaultExpr;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java?rev=1667641&r1=1667640&r2=1667641&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java Wed Mar 18 23:51:31 2015
@@ -19,6 +19,8 @@
 package org.apache.hadoop.hive.ql.optimizer;
 
 import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -36,6 +38,8 @@ import org.apache.hadoop.hive.ql.plan.Op
  * implemented, this transformation can also be done based on costs.
  */
 public class JoinReorder implements Transform {
+
+  private final Map<Operator<?>, Integer> cache = new IdentityHashMap<Operator<?>, Integer>();
   /**
    * Estimate the size of the output based on the STREAMTABLE hints. To do so
    * the whole tree is traversed. Possible sizes: 0: the operator and its
@@ -49,8 +53,25 @@ public class JoinReorder implements Tran
    * @return The estimated size - 0 (no streamed tables), 1 (streamed tables in
    *         subtree) or 2 (a streamed table)
    */
+
   private int getOutputSize(Operator<? extends OperatorDesc> operator,
       Set<String> bigTables) {
+
+    // memoize decorator for getOutputSizeInternal
+    if (cache.containsKey(operator)) {
+      return cache.get(operator);
+    }
+
+    int result = getOutputSizeInternal(operator, bigTables);
+
+    cache.put(operator, result);
+
+    return result;
+  }
+
+  private int getOutputSizeInternal(Operator<? extends OperatorDesc> operator,
+      Set<String> bigTables) {
+
     // If a join operator contains a big subtree, there is a chance that its
     // output is also big, so the output size is 1 (medium)
     if (operator instanceof JoinOperator) {
@@ -74,6 +95,7 @@ public class JoinReorder implements Tran
     int maxSize = 0;
     if (operator.getParentOperators() != null) {
       for (Operator<? extends OperatorDesc> o : operator.getParentOperators()) {
+        // recurse into memoized decorator
         int current = getOutputSize(o, bigTables);
         if (current > maxSize) {
           maxSize = current;
@@ -151,8 +173,10 @@ public class JoinReorder implements Tran
    * @param pactx
    *          current parse context
    */
+  @Override
   public ParseContext transform(ParseContext pactx) throws SemanticException {
     Set<String> bigTables = getBigTables(pactx);
+    cache.clear();
 
     for (JoinOperator joinOp : pactx.getJoinOps()) {
       reorder(joinOp, bigTables);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=1667641&r1=1667640&r2=1667641&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Wed Mar 18 23:51:31 2015
@@ -128,7 +128,11 @@ public class Optimizer {
     }
 
     transformations.add(new UnionProcessor());
-    transformations.add(new JoinReorder());
+
+    if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.NWAYJOINREORDER)) {
+      transformations.add(new JoinReorder());
+    }
+
     if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) &&
         HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") &&
         HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) &&