You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rb...@apache.org on 2017/01/11 07:57:51 UTC
hive git commit: HIVE-15565: LLAP: GroupByOperator flushes hash table
too frequently (Rajesh Balamohan, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 93848252a -> c219ce581
HIVE-15565: LLAP: GroupByOperator flushes hash table too frequently (Rajesh Balamohan, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c219ce58
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c219ce58
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c219ce58
Branch: refs/heads/master
Commit: c219ce5815c4ceb52c079a021386975a49ca6355
Parents: 9384825
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Wed Jan 11 13:27:36 2017 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Wed Jan 11 13:27:36 2017 +0530
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/GroupByOperator.java | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c219ce58/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
index 46f0ecd..d98cea9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
@@ -407,7 +407,11 @@ public class GroupByOperator extends Operator<GroupByDesc> {
computeMaxEntriesHashAggr();
}
memoryMXBean = ManagementFactory.getMemoryMXBean();
- maxMemory = isTez ? getConf().getMaxMemoryAvailable() : memoryMXBean.getHeapMemoryUsage().getMax();
+ if (isLlap || !isTez) {
+ maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
+ } else {
+ maxMemory = getConf().getMaxMemoryAvailable();
+ }
memoryThreshold = this.getConf().getMemoryThreshold();
LOG.info("isTez: {} isLlap: {} numExecutors: {} maxMemory: {}", isTez, isLlap, numExecutors, maxMemory);
}
@@ -423,12 +427,13 @@ public class GroupByOperator extends Operator<GroupByDesc> {
**/
private void computeMaxEntriesHashAggr() throws HiveException {
float memoryPercentage = this.getConf().getGroupByMemoryUsage();
- if (isTez) {
- maxHashTblMemory = (long) (memoryPercentage * getConf().getMaxMemoryAvailable());
- } else {
+ if (isLlap || !isTez) {
maxHashTblMemory = (long) (memoryPercentage * Runtime.getRuntime().maxMemory());
+ } else {
+ maxHashTblMemory = (long) (memoryPercentage * getConf().getMaxMemoryAvailable());
}
- LOG.info("Max hash table memory: {} bytes", maxHashTblMemory);
+ LOG.info("Max hash table memory: memoryPercentage:{}, maxHashTblMemory:{} bytes",
+ memoryPercentage, maxHashTblMemory);
estimateRowSize();
}