You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ju...@apache.org on 2015/07/20 18:59:42 UTC
parquet-mr git commit: PARQUET-340: MemoryManager: max memory can be
truncated
Repository: parquet-mr
Updated Branches:
refs/heads/master 1dd5cec08 -> 83406b73e
PARQUET-340: MemoryManager: max memory can be truncated
Using float will cause the max heap limit to be limited to 2147483647
due to math.round(float) if used with a large heap. This should be a double
precision to prevent rounding to an int32 before storing into a long.
Author: Chris Bannister <c....@gmail.com>
Closes #246 from Zariel/default-mem-truncated and squashes the following commits:
bf375f6 [Chris Bannister] MemoryManager: ensure max memory is not truncated
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/83406b73
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/83406b73
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/83406b73
Branch: refs/heads/master
Commit: 83406b73e70a251eec5daae34f1bd8d554cdddec
Parents: 1dd5cec
Author: Chris Bannister <c....@gmail.com>
Authored: Mon Jul 20 09:59:29 2015 -0700
Committer: julien <ju...@twitter.com>
Committed: Mon Jul 20 09:59:29 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/parquet/hadoop/MemoryManager.java | 2 +-
.../src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83406b73/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
index f997d08..0c56bb2 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
@@ -57,7 +57,7 @@ public class MemoryManager {
memoryPoolRatio = ratio;
minMemoryAllocation = minAllocation;
- totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
+ totalMemoryPool = Math.round((double) ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
() * ratio);
LOG.debug(String.format("Allocated total memory pool is: %,d", totalMemoryPool));
}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83406b73/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
index bbe7443..fb71b86 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
@@ -54,7 +54,7 @@ public class TestMemoryManager {
@Before
public void setUp() {
GroupWriteSupport.setSchema(MessageTypeParser.parseMessageType(writeSchema),conf);
- expectPoolSize = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
+ expectPoolSize = Math.round((double) ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
() * MemoryManager.DEFAULT_MEMORY_POOL_RATIO);
rowGroupSize = (int) Math.floor(expectPoolSize / 2);
conf.setInt(ParquetOutputFormat.BLOCK_SIZE, rowGroupSize);