You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by kl...@apache.org on 2016/03/04 18:30:16 UTC
[04/14] incubator-geode git commit: GEODE-996: OffheapStats -
fragmentation percentage calculation has to be revisited.
GEODE-996: OffheapStats - fragmentation percentage calculation has to be revisited.
Offheap fragmentation stat is computed based on
available fragments compared to max possbile fragments
which describes how fragmented the current free memory is.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/c18f5156
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/c18f5156
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/c18f5156
Branch: refs/heads/feature/GEODE-949-2
Commit: c18f5156a0982fcbec6f288b3c88c101bcf10736
Parents: 00b4c84
Author: Sai Boorlagadda <sb...@pivotal.io>
Authored: Fri Feb 26 13:46:32 2016 -0800
Committer: Sai Boorlagadda <sb...@pivotal.io>
Committed: Tue Mar 1 14:21:31 2016 -0800
----------------------------------------------------------------------
.../internal/offheap/FreeListManager.java | 39 +++++----
.../internal/offheap/OffHeapStorage.java | 2 +-
.../internal/offheap/FreeListManagerTest.java | 83 ++++++++++++++++++++
.../management/OffHeapManagementDUnitTest.java | 54 ++++++++++---
4 files changed, 149 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c18f5156/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
index b816eb9..3859d58 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
@@ -437,7 +437,7 @@ public class FreeListManager {
this.ma.getStats().setLargestFragment(largestFragment);
this.ma.getStats().setFragments(tmp.size());
- updateFragmentation(largestFragment);
+ this.ma.getStats().setFragmentation(getFragmentation());
return result;
} // sync
@@ -473,21 +473,30 @@ public class FreeListManager {
}
}
- private void updateFragmentation(long largestFragment) {
- long freeSize = getFreeMemory();
-
- // Calculate free space fragmentation only if there is free space available.
- if(freeSize > 0) {
- long numerator = freeSize - largestFragment;
-
- double percentage = (double) numerator / (double) freeSize;
- percentage *= 100d;
-
- int wholePercentage = (int) Math.rint(percentage);
- this.ma.getStats().setFragmentation(wholePercentage);
+ protected int getFragmentCount() {
+ return this.fragmentList.size();
+ }
+
+ protected int getFragmentation() {
+ if(getUsedMemory() == 0) {
+ //when no memory is used then there is no fragmentation
+ return 0;
} else {
- // No free space? Then we have no free space fragmentation.
- this.ma.getStats().setFragmentation(0);
+ int availableFragments = getFragmentCount();
+ if (availableFragments == 0) {
+ //zero fragments means no free memory then no fragmentation
+ return 0;
+ } else if (availableFragments == 1) {
+ //free memory is available as one fragment, so no fragmentation
+ return 0;
+ } else {
+ //more than 1 fragment is available so freeMemory is > ObjectChunk.MIN_CHUNK_SIZE
+ long freeMemory = getFreeMemory();
+ assert freeMemory > ObjectChunk.MIN_CHUNK_SIZE;
+ long maxPossibleFragments = freeMemory / ObjectChunk.MIN_CHUNK_SIZE;
+ double fragmentation = ((double) availableFragments /(double) maxPossibleFragments) * 100d;
+ return (int) Math.rint(fragmentation);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c18f5156/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java
index bb5cdeb..3156067 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/offheap/OffHeapStorage.java
@@ -71,7 +71,7 @@ public class OffHeapStorage implements OffHeapMemoryStats {
final String usedMemoryDesc = "The amount of off-heap memory, in bytes, that is being used to store data.";
final String compactionsDesc = "The total number of times off-heap memory has been compacted.";
final String compactionTimeDesc = "The total time spent compacting off-heap memory.";
- final String fragmentationDesc = "The percentage of off-heap memory fragmentation. Updated every time a compaction is performed.";
+ final String fragmentationDesc = "The percentage of off-heap free memory that is fragmented. Updated every time a compaction is performed.";
final String fragmentsDesc = "The number of fragments of free off-heap memory. Updated every time a compaction is done.";
final String freeMemoryDesc = "The amount of off-heap memory, in bytes, that is not being used.";
final String largestFragmentDesc = "The largest fragment of memory found by the last compaction of off heap memory. Updated every time a compaction is done.";
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c18f5156/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
index 64032cc..9bfb2eb 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
@@ -739,6 +739,89 @@ public class FreeListManagerTest {
LogWriter lw = mock(LogWriter.class);
this.freeListManager.logOffHeapState(lw, 1024);
}
+
+ @Test
+ public void fragmentationShouldBeZeroIfNumberOfFragmentsIsZero() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(10);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+
+ FreeListManager spy = spy(this.freeListManager);
+
+ when(spy.getFragmentCount()).thenReturn(0);
+
+ assertThat(spy.getFragmentation()).isZero();
+ }
+
+ @Test
+ public void fragmentationShouldBeZeroIfNumberOfFragmentsIsOne() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(10);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+
+ FreeListManager spy = spy(this.freeListManager);
+
+ when(spy.getFragmentCount()).thenReturn(1);
+
+ assertThat(spy.getFragmentation()).isZero();
+ }
+
+ @Test
+ public void fragmentationShouldBeZeroIfUsedMemoryIsZero() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(10);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+
+ FreeListManager spy = spy(this.freeListManager);
+
+ when(spy.getUsedMemory()).thenReturn(0L);
+
+ assertThat(spy.getFragmentation()).isZero();
+ }
+
+ @Test
+ public void fragmentationShouldBe100IfAllFreeMemoryIsFragmentedAsMinChunks() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(10);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+
+ FreeListManager spy = spy(this.freeListManager);
+
+ when(spy.getUsedMemory()).thenReturn(1L);
+ when(spy.getFragmentCount()).thenReturn(2);
+ when(spy.getFreeMemory()).thenReturn((long)ObjectChunk.MIN_CHUNK_SIZE * 2);
+
+ assertThat(spy.getFragmentation()).isEqualTo(100);
+ }
+
+ @Test
+ public void fragmentationShouldBeRoundedToNearestInteger() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(10);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+
+ FreeListManager spy = spy(this.freeListManager);
+
+ when(spy.getUsedMemory()).thenReturn(1L);
+ when(spy.getFragmentCount()).thenReturn(4);
+ when(spy.getFreeMemory()).thenReturn((long)ObjectChunk.MIN_CHUNK_SIZE * 8);
+
+ assertThat(spy.getFragmentation()).isEqualTo(50); //Math.rint(50.0)
+
+ when(spy.getUsedMemory()).thenReturn(1L);
+ when(spy.getFragmentCount()).thenReturn(3);
+ when(spy.getFreeMemory()).thenReturn((long)ObjectChunk.MIN_CHUNK_SIZE * 8);
+
+ assertThat(spy.getFragmentation()).isEqualTo(38); //Math.rint(37.5)
+
+ when(spy.getUsedMemory()).thenReturn(1L);
+ when(spy.getFragmentCount()).thenReturn(6);
+ when(spy.getFreeMemory()).thenReturn((long)ObjectChunk.MIN_CHUNK_SIZE * 17);
+
+ assertThat(spy.getFragmentation()).isEqualTo(35); //Math.rint(35.29)
+
+ when(spy.getUsedMemory()).thenReturn(1L);
+ when(spy.getFragmentCount()).thenReturn(6);
+ when(spy.getFreeMemory()).thenReturn((long)ObjectChunk.MIN_CHUNK_SIZE * 9);
+
+ assertThat(spy.getFragmentation()).isEqualTo(67); //Math.rint(66.66)
+ }
+
/**
* Just like Fragment except that the first time allocate is called
* it returns false indicating that the allocate failed.
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c18f5156/geode-core/src/test/java/com/gemstone/gemfire/management/OffHeapManagementDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/management/OffHeapManagementDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/management/OffHeapManagementDUnitTest.java
index acafada..3d06e11 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/management/OffHeapManagementDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/management/OffHeapManagementDUnitTest.java
@@ -36,6 +36,7 @@ import com.gemstone.gemfire.cache30.CacheTestCase;
import com.gemstone.gemfire.distributed.internal.DistributionConfig;
import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
+import com.gemstone.gemfire.internal.offheap.ObjectChunk;
import com.gemstone.gemfire.internal.offheap.OffHeapMemoryStats;
import com.gemstone.gemfire.internal.offheap.OffHeapStorage;
import com.gemstone.gemfire.management.internal.MBeanJMXAdapter;
@@ -237,21 +238,51 @@ public class OffHeapManagementDUnitTest extends CacheTestCase {
// Make sure our starting off heap stats are correct
assertOffHeapMetricsOnVm(vm, TOTAL_MEMORY, 0, 0, 0);
- // After allocating large chunck we should still have no fragmentation
- doPutOnVm(vm, KEY, new byte[HALF_TOTAL_MEMORY], OFF_HEAP_REGION_NAME, false);
+ // After allocating large chunk (equal to total memory)
+ // we should still have no fragmentation
+ int largeChunk = (int) TOTAL_MEMORY - ObjectChunk.OFF_HEAP_HEADER_SIZE;
+ doPutOnVm(vm, KEY, new byte[largeChunk], OFF_HEAP_REGION_NAME, false);
+ // No compaction has run, so fragmentation should be zero
+ assertFragmentationStatOnVm(vm,0,ASSERT_OP.EQUAL);
+
+ // Allocate more memory to trigger compaction
+ doPutOnVm(vm, KEY, new byte[ALLOCATION_SIZE], OFF_HEAP_REGION_NAME, true);
+ // When total memory is used no fragmentation
assertFragmentationStatOnVm(vm,0,ASSERT_OP.EQUAL);
// After freeing all memory we should have no fragmentation
doDestroyOnVm(vm, KEY, OFF_HEAP_REGION_NAME);
assertFragmentationStatOnVm(vm,0,ASSERT_OP.EQUAL);
- // Consume all off-heap memory using an allocation size
- int numAllocations = doConsumeOffHeapMemoryOnVm(vm,ALLOCATION_SIZE);
- assertTrue(numAllocations > 0);
+ // Allocate HALF_TOTAL_MEMORY twice and release one to create one fragment
+ int halfChunk = HALF_TOTAL_MEMORY - ObjectChunk.OFF_HEAP_HEADER_SIZE;
+ doPutOnVm(vm, KEY + "0", new byte[halfChunk], OFF_HEAP_REGION_NAME, false);
+ doPutOnVm(vm, KEY + "1", new byte[halfChunk], OFF_HEAP_REGION_NAME, false);
+ doDestroyOnVm(vm, KEY + "0", OFF_HEAP_REGION_NAME);
- // Randomly free 3 allocations to produce off-heap gaps
- doFreeOffHeapMemoryOnVm(vm, numAllocations, 3);
-
+ // Allocate largeChunk to trigger compaction and fragmentation should be zero
+ // as all free memory is available as one fragment
+ doPutOnVm(vm, KEY + "1", new byte[largeChunk], OFF_HEAP_REGION_NAME, true);
+ assertFragmentationStatOnVm(vm,0,ASSERT_OP.EQUAL);
+
+ // Consume the available fragment as below
+ // [16][262120][16][262120][16] = [524288] (HALF_TOTAL_MEMORY)
+ int smallChunk = ObjectChunk.MIN_CHUNK_SIZE - ObjectChunk.OFF_HEAP_HEADER_SIZE;
+ int mediumChunk = 262112; //(262120 - ObjectChunk.OFF_HEAP_HEADER_SIZE)
+ doPutOnVm(vm, KEY + "S1", new byte[smallChunk], OFF_HEAP_REGION_NAME, false);
+ doPutOnVm(vm, KEY + "M1", new byte[mediumChunk], OFF_HEAP_REGION_NAME, false);
+ doPutOnVm(vm, KEY + "S2", new byte[smallChunk], OFF_HEAP_REGION_NAME, false);
+ doPutOnVm(vm, KEY + "M2", new byte[mediumChunk], OFF_HEAP_REGION_NAME, false);
+ doPutOnVm(vm, KEY + "S3", new byte[smallChunk], OFF_HEAP_REGION_NAME, false);
+
+ // free small chunks to create gaps
+ doDestroyOnVm(vm, KEY + "S1", OFF_HEAP_REGION_NAME);
+ doDestroyOnVm(vm, KEY + "S2", OFF_HEAP_REGION_NAME);
+ doDestroyOnVm(vm, KEY + "S3", OFF_HEAP_REGION_NAME);
+
+ // Now free memory should be 48 so allocate a 40 byte object
+ doPutOnVm(vm, KEY + "newKey", new byte[40], OFF_HEAP_REGION_NAME, true);
+
/*
* Setup a fragmentation attribute monitor
*/
@@ -259,12 +290,9 @@ public class OffHeapManagementDUnitTest extends CacheTestCase {
setupOffHeapMonitorOnVm(vm,"OffHeapFragmentation",0,0);
clearNotificationListenerOnVm(vm);
}
-
- // Allocate enough memory to force compaction which will update fragmenation stat
- doPutOnVm(vm,KEY, new byte[NEW_ALLOCATION_SIZE], OFF_HEAP_REGION_NAME, true);
-
+
// Make sure we have some fragmentation
- assertFragmentationStatOnVm(vm, 0, ASSERT_OP.GREATER_THAN);
+ assertFragmentationStatOnVm(vm, 100, ASSERT_OP.EQUAL);
// Make sure our fragmentation monitor was triggered
waitForNotificationListenerOnVm(vm, 5000, 500, true);