You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2014/10/09 20:35:01 UTC
svn commit: r1630549 - in
/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql:
optimizer/stats/annotation/StatsRulesProcFactory.java stats/StatsUtils.java
Author: prasanthj
Date: Thu Oct 9 18:35:00 2014
New Revision: 1630549
URL: http://svn.apache.org/r1630549
Log:
HIVE-8354: HIVE-7156 introduced required dependency on tez (Prasanth J reviewed by Brock Noland)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java?rev=1630549&r1=1630548&r2=1630549&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java Thu Oct 9 18:35:00 2014
@@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.exec.Re
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.tez.DagUtils;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
@@ -889,16 +888,12 @@ public class StatsRulesProcFactory {
GroupByDesc.Mode mode = desc.getMode();
if (mode.equals(GroupByDesc.Mode.HASH)) {
- float hashAggMem = conf.getFloatVar(
- HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
- float hashAggMaxThreshold = conf.getFloatVar(
- HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
-
- // get memory for container. May be use mapreduce.map.java.opts instead?
- long totalMemory =
- DagUtils.getContainerResource(conf).getMemory() * 1000L * 1000L;
- long maxMemHashAgg = Math
- .round(totalMemory * hashAggMem * hashAggMaxThreshold);
+ float hashAggMem = conf.getFloatVar(HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
+ float hashAggMaxThreshold = conf.getFloatVar(HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
+
+ // get available map memory
+ long totalMemory = StatsUtils.getAvailableMemory(conf) * 1000L * 1000L;
+ long maxMemHashAgg = Math.round(totalMemory * hashAggMem * hashAggMaxThreshold);
// estimated number of rows will be product of NDVs
long numEstimatedRows = 1;
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1630549&r1=1630548&r2=1630549&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Thu Oct 9 18:35:00 2014
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
@@ -86,6 +87,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
@@ -1350,4 +1352,11 @@ public class StatsUtils {
}
}
+
+ public static long getAvailableMemory(Configuration conf) {
+ int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ?
+ HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) :
+ conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB);
+ return memory;
+ }
}