You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2015/05/29 02:51:20 UTC
[73/82] [abbrv] hive git commit: HIVE-10711 Tez HashTableLoader
attempts to allocate more memory than available (Mostafa Mokhtar and Jason
Dere via Alexander Pivovarov)
HIVE-10711 Tez HashTableLoader attempts to allocate more memory than available (Mostafa Mokhtar and Jason Dere via Alexander Pivovarov)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1293f3d3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1293f3d3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1293f3d3
Branch: refs/heads/llap
Commit: 1293f3d389b73eea672805c49270124f44cbd687
Parents: ac25506
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Wed May 27 23:30:39 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Wed May 27 23:30:39 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/HashTableLoader.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1293f3d3/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 536b92c..7e67d40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.tez;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
import java.util.Collections;
import java.util.Map;
@@ -82,6 +83,15 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
// TODO remove this after memory manager is in
long noConditionalTaskThreshold = HiveConf.getLongVar(
hconf, HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD);
+ long processMaxMemory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+ if (noConditionalTaskThreshold > processMaxMemory) {
+ float hashtableMemoryUsage = HiveConf.getFloatVar(
+ hconf, HiveConf.ConfVars.HIVEHASHTABLEFOLLOWBYGBYMAXMEMORYUSAGE);
+ LOG.warn("noConditionalTaskThreshold value of " + noConditionalTaskThreshold +
+ " is greater than the max memory size of " + processMaxMemory);
+ // Don't want to attempt to grab more memory than we have available .. percentage is a bit arbitrary
+ noConditionalTaskThreshold = (long) (processMaxMemory * hashtableMemoryUsage);
+ }
// Only applicable to n-way Hybrid Grace Hash Join
HybridHashTableConf nwayConf = null;