You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ds...@apache.org on 2016/02/18 00:11:07 UTC
[32/33] incubator-geode git commit: finish up FreeListManagerTest
finish up FreeListManagerTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/81c9476e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/81c9476e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/81c9476e
Branch: refs/heads/feature/GEODE-831
Commit: 81c9476ec533d0a2bcc69632994c3a34b9b661f4
Parents: 95e23f0
Author: Darrel Schneider <ds...@pivotal.io>
Authored: Wed Feb 17 14:51:10 2016 -0800
Committer: Darrel Schneider <ds...@pivotal.io>
Committed: Wed Feb 17 14:51:10 2016 -0800
----------------------------------------------------------------------
.../internal/offheap/FreeListManager.java | 60 +++---
.../internal/offheap/FreeListManagerTest.java | 181 ++++++++++++++++++-
2 files changed, 207 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/81c9476e/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
index 951e8a8..8efefbd 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/FreeListManager.java
@@ -234,14 +234,18 @@ public class FreeListManager {
private void logOffHeapState(int chunkSize) {
if (InternalDistributedSystem.getAnyInstance() != null) {
LogWriter lw = InternalDistributedSystem.getAnyInstance().getLogWriter();
- OffHeapMemoryStats stats = this.ma.getStats();
- lw.info("OutOfOffHeapMemory allocating size of " + chunkSize + ". allocated=" + this.allocatedSize.get() + " compactions=" + this.compactCount.get() + " objects=" + stats.getObjects() + " free=" + stats.getFreeMemory() + " fragments=" + stats.getFragments() + " largestFragment=" + stats.getLargestFragment() + " fragmentation=" + stats.getFragmentation());
- logFragmentState(lw);
- logTinyState(lw);
- logHugeState(lw);
+ logOffHeapState(lw, chunkSize);
}
}
+ void logOffHeapState(LogWriter lw, int chunkSize) {
+ OffHeapMemoryStats stats = this.ma.getStats();
+ lw.info("OutOfOffHeapMemory allocating size of " + chunkSize + ". allocated=" + this.allocatedSize.get() + " compactions=" + this.compactCount.get() + " objects=" + stats.getObjects() + " free=" + stats.getFreeMemory() + " fragments=" + stats.getFragments() + " largestFragment=" + stats.getLargestFragment() + " fragmentation=" + stats.getFragmentation());
+ logFragmentState(lw);
+ logTinyState(lw);
+ logHugeState(lw);
+ }
+
private void logHugeState(LogWriter lw) {
for (Chunk c: this.hugeChunkSet) {
lw.info("Free huge of size " + c.getSize());
@@ -264,7 +268,7 @@ public class FreeListManager {
}
}
- private final AtomicInteger compactCount = new AtomicInteger();
+ protected final AtomicInteger compactCount = new AtomicInteger();
/*
* Set this to "true" to perform data integrity checks on allocated and reused Chunks. This may clobber
* performance so turn on only when necessary.
@@ -311,6 +315,7 @@ public class FreeListManager {
boolean compact(int chunkSize) {
final long startCompactionTime = this.ma.getStats().startCompaction();
final int countPreSync = this.compactCount.get();
+ afterCompactCountFetched();
try {
synchronized (this) {
if (this.compactCount.get() != countPreSync) {
@@ -329,7 +334,6 @@ public class FreeListManager {
long addr = l.poll();
while (addr != 0) {
int idx = Arrays.binarySearch(sorted, 0, sortedSize, addr);
- //System.out.println("DEBUG addr=" + addr + " size=" + Chunk.getSize(addr) + " idx="+idx + " sortedSize=" + sortedSize);
idx = -idx;
idx--;
if (idx == sortedSize) {
@@ -442,6 +446,12 @@ public class FreeListManager {
}
}
+ /**
+ * Unit tests override this method to get better test coverage
+ */
+ protected void afterCompactCountFetched() {
+ }
+
static void verifyOffHeapAlignment(int tinyMultiple) {
if (tinyMultiple <= 0 || (tinyMultiple & 3) != 0) {
throw new IllegalStateException("gemfire.OFF_HEAP_ALIGNMENT must be a multiple of 8.");
@@ -542,7 +552,7 @@ public class FreeListManager {
}
}
- private Chunk allocateFromFragment(final int fragIdx, final int chunkSize) {
+ Chunk allocateFromFragment(final int fragIdx, final int chunkSize) {
if (fragIdx >= this.fragmentList.size()) return null;
final Fragment fragment;
try {
@@ -573,11 +583,7 @@ public class FreeListManager {
// We did the allocate!
this.lastFragmentAllocation.set(fragIdx);
Chunk result = new GemFireChunk(fragment.getMemoryAddress()+oldOffset, chunkSize+extraSize);
-
- if(this.validateMemoryWithFill) {
- result.validateFill();
- }
-
+ checkDataIntegrity(result);
return result;
} else {
Chunk result = basicAllocate(chunkSize, false);
@@ -603,12 +609,7 @@ public class FreeListManager {
long memAddr = clq.poll();
if (memAddr != 0) {
Chunk result = new GemFireChunk(memAddr);
-
- // Data integrity check.
- if(this.validateMemoryWithFill) {
- result.validateFill();
- }
-
+ checkDataIntegrity(result);
result.readyForAllocation();
return result;
}
@@ -627,11 +628,7 @@ public class FreeListManager {
if (result != null) {
if (result.getSize() - (HUGE_MULTIPLE - Chunk.OFF_HEAP_HEADER_SIZE) < size) {
// close enough to the requested size; just return it.
-
- // Data integrity check.
- if(this.validateMemoryWithFill) {
- result.validateFill();
- }
+ checkDataIntegrity(result);
result.readyForAllocation();
return result;
} else {
@@ -647,6 +644,11 @@ public class FreeListManager {
}
}
+ private void checkDataIntegrity(Chunk data) {
+ if (this.validateMemoryWithFill) {
+ data.validateFill();
+ }
+ }
/**
* Used by the FreeListManager to easily search its
* ConcurrentSkipListSet. This is not a real chunk
@@ -697,15 +699,21 @@ public class FreeListManager {
if (clq != null) {
clq.offer(addr);
} else {
- clq = new SyncChunkStack();
+ clq = createFreeListForEmptySlot(freeLists, idx);
clq.offer(addr);
if (!freeLists.compareAndSet(idx, null, clq)) {
clq = freeLists.get(idx);
clq.offer(addr);
}
}
-
}
+ /**
+ * Tests override this method to simulate concurrent modification
+ */
+ protected SyncChunkStack createFreeListForEmptySlot(AtomicReferenceArray<SyncChunkStack> freeLists, int idx) {
+ return new SyncChunkStack();
+ }
+
private void freeHuge(long addr, int cSize) {
this.hugeChunkSet.add(new GemFireChunk(addr)); // TODO make this a collection of longs
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/81c9476e/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
index 5f3a546..0de4ea7 100644
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/FreeListManagerTest.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicReferenceArray;
import org.junit.After;
import org.junit.AfterClass;
@@ -32,6 +33,7 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.gemstone.gemfire.LogWriter;
import com.gemstone.gemfire.OutOfOffHeapMemoryException;
import com.gemstone.gemfire.test.junit.categories.UnitTest;
@@ -44,7 +46,7 @@ public class FreeListManagerTest {
private final int DEFAULT_SLAB_SIZE = 1024*1024*5;
private final SimpleMemoryAllocatorImpl ma = mock(SimpleMemoryAllocatorImpl.class);
private final OffHeapMemoryStats stats = mock(OffHeapMemoryStats.class);
- private FreeListManager freeListManager;
+ private TestableFreeListManager freeListManager;
@BeforeClass
@@ -67,12 +69,15 @@ public class FreeListManagerTest {
}
}
- private static FreeListManager createFreeListManager(SimpleMemoryAllocatorImpl ma, AddressableMemoryChunk[] slabs) {
+ private static TestableFreeListManager createFreeListManager(SimpleMemoryAllocatorImpl ma, AddressableMemoryChunk[] slabs) {
return new TestableFreeListManager(ma, slabs);
}
private void setUpSingleSlabManager() {
- UnsafeMemoryChunk slab = new UnsafeMemoryChunk(DEFAULT_SLAB_SIZE);
+ setUpSingleSlabManager(DEFAULT_SLAB_SIZE);
+ }
+ private void setUpSingleSlabManager(int slabSize) {
+ UnsafeMemoryChunk slab = new UnsafeMemoryChunk(slabSize);
this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {slab});
}
@@ -190,6 +195,22 @@ public class FreeListManagerTest {
}
@Test
+ public void freeTinyMemoryWithTwoTinyFreeListsEqualToChunkSize() {
+ setUpSingleSlabManager();
+ int dataSize = 10;
+
+ Chunk c = this.freeListManager.allocate(dataSize);
+ Chunk.release(c.getMemoryAddress(), this.freeListManager);
+
+ int dataSize2 = 100;
+
+ Chunk c2 = this.freeListManager.allocate(dataSize2);
+ Chunk.release(c2.getMemoryAddress(), this.freeListManager);
+
+ assertThat(this.freeListManager.getFreeTinyMemory()).isEqualTo(computeExpectedSize(dataSize)+computeExpectedSize(dataSize2));
+ }
+
+ @Test
public void freeHugeMemoryDefault() {
setUpSingleSlabManager();
@@ -275,7 +296,7 @@ public class FreeListManagerTest {
for (Chunk c: chunks) {
Chunk.release(c.getMemoryAddress(), this.freeListManager);
}
-
+ this.freeListManager.firstCompact = false;
assertThat(this.freeListManager.compact(DEFAULT_SLAB_SIZE+1)).isFalse();
}
@@ -328,7 +349,7 @@ public class FreeListManagerTest {
public void compactAfterAllocatingAll() {
setUpSingleSlabManager();
Chunk c = freeListManager.allocate(DEFAULT_SLAB_SIZE-8);
-
+ this.freeListManager.firstCompact = false;
assertThat(this.freeListManager.compact(1)).isFalse();
// call compact twice for extra code coverage
assertThat(this.freeListManager.compact(1)).isFalse();
@@ -336,13 +357,75 @@ public class FreeListManagerTest {
}
@Test
+ public void afterAllocatingAllOneSizeCompactToAllocateDifferentSize() {
+ setUpSingleSlabManager();
+ ArrayList<Chunk> chunksToFree = new ArrayList<>();
+ ArrayList<Chunk> chunksToFreeLater = new ArrayList<>();
+ int ALLOCATE_COUNT = 1000;
+ Chunk bigChunk = freeListManager.allocate(DEFAULT_SLAB_SIZE-8-(ALLOCATE_COUNT*32)-256-256);
+ for (int i=0; i < ALLOCATE_COUNT; i++) {
+ Chunk c = freeListManager.allocate(24);
+ if (i%3 != 2) {
+ chunksToFree.add(c);
+ } else {
+ chunksToFreeLater.add(c);
+ }
+ }
+ Chunk c1 = freeListManager.allocate(64-8);
+ Chunk c2 = freeListManager.allocate(64-8);
+ Chunk c3 = freeListManager.allocate(64-8);
+ Chunk c4 = freeListManager.allocate(64-8);
+
+ Chunk mediumChunk1 = freeListManager.allocate(128-8);
+ Chunk mediumChunk2 = freeListManager.allocate(128-8);
+
+ Chunk.release(bigChunk.getMemoryAddress(), freeListManager);
+ int s = chunksToFree.size();
+ for (int i=s/2; i >=0; i--) {
+ Chunk c = chunksToFree.get(i);
+ Chunk.release(c.getMemoryAddress(), freeListManager);
+ }
+ for (int i=(s/2)+1; i < s; i++) {
+ Chunk c = chunksToFree.get(i);
+ Chunk.release(c.getMemoryAddress(), freeListManager);
+ }
+ Chunk.release(c3.getMemoryAddress(), freeListManager);
+ Chunk.release(c1.getMemoryAddress(), freeListManager);
+ Chunk.release(c2.getMemoryAddress(), freeListManager);
+ Chunk.release(c4.getMemoryAddress(), freeListManager);
+ Chunk.release(mediumChunk1.getMemoryAddress(), freeListManager);
+ Chunk.release(mediumChunk2.getMemoryAddress(), freeListManager);
+ this.freeListManager.firstCompact = false;
+ assertThat(freeListManager.compact(DEFAULT_SLAB_SIZE-(ALLOCATE_COUNT*32))).isFalse();
+ for (int i=0; i < ((256*2)/96); i++) {
+ Chunk.release(chunksToFreeLater.get(i).getMemoryAddress(), freeListManager);
+ }
+ assertThat(freeListManager.compact(DEFAULT_SLAB_SIZE-(ALLOCATE_COUNT*32))).isTrue();
+ }
+
+ @Test
+ public void afterAllocatingAndFreeingCompact() {
+ int slabSize = 1024*3;
+ setUpSingleSlabManager(slabSize);
+ Chunk bigChunk1 = freeListManager.allocate(slabSize/3-8);
+ Chunk bigChunk2 = freeListManager.allocate(slabSize/3-8);
+ Chunk bigChunk3 = freeListManager.allocate(slabSize/3-8);
+ this.freeListManager.firstCompact = false;
+ assertThat(freeListManager.compact(1)).isFalse();
+ Chunk.release(bigChunk3.getMemoryAddress(), freeListManager);
+ Chunk.release(bigChunk2.getMemoryAddress(), freeListManager);
+ Chunk.release(bigChunk1.getMemoryAddress(), freeListManager);
+ assertThat(freeListManager.compact(slabSize)).isTrue();
+ }
+
+ @Test
public void compactWithEmptyTinyFreeList() {
setUpSingleSlabManager();
Fragment originalFragment = this.freeListManager.getFragmentList().get(0);
Chunk c = freeListManager.allocate(16);
Chunk.release(c.getMemoryAddress(), this.freeListManager);
c = freeListManager.allocate(16);
-
+ this.freeListManager.firstCompact = false;
assertThat(this.freeListManager.compact(1)).isTrue();
assertThat(this.freeListManager.getFragmentList()).hasSize(1);
Fragment compactedFragment = this.freeListManager.getFragmentList().get(0);
@@ -586,7 +669,7 @@ public class FreeListManagerTest {
@Test
public void orderBlocksContainsTinyFree() {
- UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(64);
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(96);
long address = chunk.getMemoryAddress();
this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
Chunk c = this.freeListManager.allocate(24);
@@ -595,9 +678,67 @@ public class FreeListManagerTest {
List<MemoryBlock> ob = this.freeListManager.getOrderedBlocks();
assertThat(ob).hasSize(3);
-// assertThat(ob.get(0).getMemoryAddress()).isEqualTo(address);
}
+ @Test
+ public void allocatedBlocksEmptyIfNoAllocations() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(10);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+ List<MemoryBlock> ob = this.freeListManager.getAllocatedBlocks();
+ assertThat(ob).hasSize(0);
+ }
+
+ @Test
+ public void allocatedBlocksEmptyAfterFree() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(96);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+ Chunk c = this.freeListManager.allocate(24);
+ Chunk.release(c.getMemoryAddress(), this.freeListManager);
+ List<MemoryBlock> ob = this.freeListManager.getAllocatedBlocks();
+ assertThat(ob).hasSize(0);
+ }
+
+ @Test
+ public void allocatedBlocksHasAllocatedChunk() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(96);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+ Chunk c = this.freeListManager.allocate(24);
+ List<MemoryBlock> ob = this.freeListManager.getAllocatedBlocks();
+ assertThat(ob).hasSize(1);
+ assertThat(ob.get(0).getMemoryAddress()).isEqualTo(c.getMemoryAddress());
+ }
+
+ @Test
+ public void allocatedBlocksHasBothAllocatedChunks() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(96);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+ Chunk c = this.freeListManager.allocate(24);
+ Chunk c2 = this.freeListManager.allocate(33);
+ List<MemoryBlock> ob = this.freeListManager.getAllocatedBlocks();
+ assertThat(ob).hasSize(2);
+ }
+
+ @Test
+ public void allocateFromFragmentWithBadIndexesReturnsNull() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(96);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk});
+ assertThat(this.freeListManager.allocateFromFragment(-1, 32)).isNull();
+ assertThat(this.freeListManager.allocateFromFragment(1, 32)).isNull();
+ }
+
+ @Test
+ public void testLogging() {
+ UnsafeMemoryChunk chunk = new UnsafeMemoryChunk(32);
+ UnsafeMemoryChunk chunk2 = new UnsafeMemoryChunk(1024*1024*5);
+ this.freeListManager = createFreeListManager(ma, new UnsafeMemoryChunk[] {chunk, chunk2});
+ Chunk c = this.freeListManager.allocate(24);
+ Chunk c2 = this.freeListManager.allocate(1024*1024);
+ Chunk.release(c.getMemoryAddress(), this.freeListManager);
+ Chunk.release(c2.getMemoryAddress(), this.freeListManager);
+
+ LogWriter lw = mock(LogWriter.class);
+ this.freeListManager.logOffHeapState(lw, 1024);
+ }
/**
* Just like Fragment except that the first time allocate is called
* it returns false indicating that the allocate failed.
@@ -619,11 +760,35 @@ public class FreeListManagerTest {
}
}
private static class TestableFreeListManager extends FreeListManager {
+ private boolean firstTime = true;
+
@Override
protected Fragment createFragment(long addr, int size) {
return new TestableFragment(addr, size);
}
+ @Override
+ protected SyncChunkStack createFreeListForEmptySlot(AtomicReferenceArray<SyncChunkStack> freeLists, int idx) {
+ if (this.firstTime) {
+ this.firstTime = false;
+ SyncChunkStack clq = super.createFreeListForEmptySlot(freeLists, idx);
+ if (!freeLists.compareAndSet(idx, null, clq)) {
+ fail("this should never happen. Indicates a concurrent modification");
+ }
+ }
+ return super.createFreeListForEmptySlot(freeLists, idx);
+ }
+
+ public boolean firstCompact = true;
+ @Override
+ protected void afterCompactCountFetched() {
+ if (this.firstCompact) {
+ this.firstCompact = false;
+ // Force compact into thinking a concurrent compaction happened.
+ this.compactCount.incrementAndGet();
+ }
+ }
+
public TestableFreeListManager(SimpleMemoryAllocatorImpl ma, AddressableMemoryChunk[] slabs) {
super(ma, slabs);
}