You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/11/08 00:27:33 UTC
svn commit: r1637482 - in /pig/branches/branch-0.14: CHANGES.txt
src/org/apache/pig/impl/util/SpillableMemoryManager.java
Author: daijy
Date: Fri Nov 7 23:27:33 2014
New Revision: 1637482
URL: http://svn.apache.org/r1637482
Log:
PIG-4299: SpillableMemoryManager assumes tenured heap incorrectly
Modified:
pig/branches/branch-0.14/CHANGES.txt
pig/branches/branch-0.14/src/org/apache/pig/impl/util/SpillableMemoryManager.java
Modified: pig/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1637482&r1=1637481&r2=1637482&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Fri Nov 7 23:27:33 2014
@@ -101,6 +101,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4299: SpillableMemoryManager assumes tenured heap incorrectly (prkommireddi via daijy)
+
PIG-4298: Descending order-by is broken in some cases when key is bytearrays (cheolsoo)
PIG-4263: Move tez local mode unit tests to a separate target (daijy)
Modified: pig/branches/branch-0.14/src/org/apache/pig/impl/util/SpillableMemoryManager.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/impl/util/SpillableMemoryManager.java?rev=1637482&r1=1637481&r2=1637482&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/impl/util/SpillableMemoryManager.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/impl/util/SpillableMemoryManager.java Fri Nov 7 23:27:33 2014
@@ -88,30 +88,28 @@ public class SpillableMemoryManager impl
private SpillableMemoryManager() {
((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(this, null, null);
List<MemoryPoolMXBean> mpbeans = ManagementFactory.getMemoryPoolMXBeans();
- MemoryPoolMXBean biggestHeap = null;
- long biggestSize = 0;
+ MemoryPoolMXBean tenuredHeap = null;
+ long tenuredHeapSize = 0;
long totalSize = 0;
- for (MemoryPoolMXBean b: mpbeans) {
- log.debug("Found heap (" + b.getName() +
- ") of type " + b.getType());
- if (b.getType() == MemoryType.HEAP) {
- /* Here we are making the leap of faith that the biggest
- * heap is the tenured heap
- */
- long size = b.getUsage().getMax();
+ for (MemoryPoolMXBean pool : mpbeans) {
+ log.debug("Found heap (" + pool.getName() + ") of type " + pool.getType());
+ if (pool.getType() == MemoryType.HEAP) {
+ long size = pool.getUsage().getMax();
totalSize += size;
- if (size > biggestSize) {
- biggestSize = size;
- biggestHeap = b;
+ // CMS Old Gen or "tenured" is the only heap that supports
+ // setting usage threshold.
+ if (pool.isUsageThresholdSupported()) {
+ tenuredHeapSize = size;
+ tenuredHeap = pool;
}
}
}
extraGCSpillSizeThreshold = (long) (totalSize * extraGCThresholdFraction);
- if (biggestHeap == null) {
+ if (tenuredHeap == null) {
throw new RuntimeException("Couldn't find heap");
}
log.debug("Selected heap to monitor (" +
- biggestHeap.getName() + ")");
+ tenuredHeap.getName() + ")");
// we want to set both collection and usage threshold alerts to be
// safe. In some local tests after a point only collection threshold
@@ -123,11 +121,11 @@ public class SpillableMemoryManager impl
/* We set the threshold to be 50% of tenured since that is where
* the GC starts to dominate CPU time according to Sun doc */
- biggestHeap.setCollectionUsageThreshold((long)(biggestSize * collectionMemoryThresholdFraction));
+ tenuredHeap.setCollectionUsageThreshold((long)(tenuredHeapSize * collectionMemoryThresholdFraction));
// we set a higher threshold for usage threshold exceeded notification
// since this is more likely to be effective sooner and we do not
// want to be spilling too soon
- biggestHeap.setUsageThreshold((long)(biggestSize * memoryThresholdFraction));
+ tenuredHeap.setUsageThreshold((long)(tenuredHeapSize * memoryThresholdFraction));
}
public static SpillableMemoryManager getInstance() {