You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bs...@apache.org on 2015/10/01 19:58:44 UTC
[03/26] incubator-geode git commit: GEODE-306: support printing of
off-heap compressed nodes
GEODE-306: support printing of off-heap compressed nodes
Before when a chunk of off-heap compressed memory tried to
get the type of the data stored in the chunk it needed to
decompress the data. Since it did not know what decompressor
to use it threw an UnsupportedOperationException.
Now for a compressed chunk it returns the DataType as either
a "compressed object of size xxx" or "compressed byte[xxx]".
It does not do a decompression.
Unit test coverage was added.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/f0f61766
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/f0f61766
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/f0f61766
Branch: refs/heads/feature/GEODE-77
Commit: f0f61766d0fa9737369b6910d94d1668e011299b
Parents: 141512c
Author: Darrel Schneider <ds...@pivotal.io>
Authored: Tue Sep 1 16:40:43 2015 -0700
Committer: Darrel Schneider <ds...@pivotal.io>
Committed: Wed Sep 2 09:41:06 2015 -0700
----------------------------------------------------------------------
.../offheap/SimpleMemoryAllocatorImpl.java | 8 +++-
.../offheap/OffHeapValidationJUnitTest.java | 45 +++++++++++++++-----
2 files changed, 41 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f0f61766/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
index 29319e0..7cf1656 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
@@ -3341,7 +3341,13 @@ public final class SimpleMemoryAllocatorImpl implements MemoryAllocator, MemoryI
}
if (!isSerialized()) {
// byte array
- return "byte[" + ((Chunk)this.block).getDataSize() + "]";
+ if (isCompressed()) {
+ return "compressed byte[" + ((Chunk)this.block).getDataSize() + "]";
+ } else {
+ return "byte[" + ((Chunk)this.block).getDataSize() + "]";
+ }
+ } else if (isCompressed()) {
+ return "compressed object of size " + ((Chunk)this.block).getDataSize();
}
//Object obj = EntryEventImpl.deserialize(((Chunk)this.block).getRawBytes());
byte[] bytes = ((Chunk)this.block).getRawBytes();
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f0f61766/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
index 7991872..1b40f2a 100755
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
@@ -5,10 +5,13 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import java.io.IOException;
import java.io.Serializable;
import java.sql.Timestamp;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -33,9 +36,13 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.gemstone.gemfire.DataSerializer;
import com.gemstone.gemfire.cache.CacheFactory;
import com.gemstone.gemfire.cache.Region;
import com.gemstone.gemfire.cache.RegionShortcut;
+import com.gemstone.gemfire.compression.SnappyCompressor;
+import com.gemstone.gemfire.internal.HeapDataOutputStream;
+import com.gemstone.gemfire.internal.Version;
import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
import com.gemstone.gemfire.internal.cache.LocalRegion;
import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl.Chunk;
@@ -87,7 +94,7 @@ public class OffHeapValidationJUnitTest {
}
@Test
- public void testMemoryInspection() {
+ public void testMemoryInspection() throws IOException {
// validate initial state
MemoryAllocator allocator = this.cache.getOffHeapStore();
assertNotNull(allocator);
@@ -113,6 +120,7 @@ public class OffHeapValidationJUnitTest {
// create off-heap region
Region<Object, Object> region = this.cache.createRegionFactory(getRegionShortcut()).setOffHeap(true).create(getRegionName());
+ Region<Object, Object> compressedRegion = this.cache.createRegionFactory(getRegionShortcut()).setOffHeap(true).setCompressor(SnappyCompressor.getDefaultInstance()).create(getRegionName()+"Compressed");
// perform some ops
List<ExpectedValues> expected = new ArrayList<ExpectedValues>();
@@ -120,8 +128,10 @@ public class OffHeapValidationJUnitTest {
// Chunk.OFF_HEAP_HEADER_SIZE + 4 ?
putString(region, expected);
+ putCompressedString(compressedRegion, expected);
putDate(region, expected);
putByteArray(region, expected);
+ putCompressedByteArray(compressedRegion, expected);
putByteArrayArray(region, expected);
putShortArray(region, expected);
putStringArray(region, expected);
@@ -189,8 +199,11 @@ public class OffHeapValidationJUnitTest {
assertTrue(obj instanceof String);
assertEquals("this is a string", (String)obj);
}
- if (values.dataType.contains("[")) { //for (int j = 0; j < ((byte[])values.dataValue).length; j++) {
- // TODO
+ if ((values.dataType.contains("byte [") && values.dataType.lastIndexOf('[') == values.dataType.indexOf('[')) || values.dataType.startsWith("compressed")) {
+ assertTrue("for dataType=" + values.dataType + " expected " + Arrays.toString((byte[])values.dataValue) + " but was " + Arrays.toString((byte[])block.getDataValue()),
+ Arrays.equals((byte[])values.dataValue, (byte[])block.getDataValue()));
+ } else if (values.dataType.contains("[")) {
+ // TODO: multiple dimension arrays or non-byte arrays
} else if (values.dataValue instanceof Collection) {
int diff = joint((Collection<?>)values.dataValue, (Collection<?>)block.getDataValue());
assertEquals(i + ":" + values.dataType, 0, diff);
@@ -216,14 +229,6 @@ public class OffHeapValidationJUnitTest {
}
- @Test
- public void testCompaction() {
- // create fragmented state
- // validate fragmented
- // perform compaction
- // validate freed fragments
- }
-
/**
* Returns -1 if c1 is missing an element in c2, 1 if c2 is missing an element
* in c1, or 0 is they contain the exact same elements.
@@ -289,6 +294,17 @@ public class OffHeapValidationJUnitTest {
expected.add(new ExpectedValues(value, value.length()*2, "java.lang.String", -1, getMemoryAddress(region, key), 1, 0, false, true));
}
+ private void putCompressedString(Region<Object, Object> region, List<ExpectedValues> expected) throws IOException {
+ String key = "keyString";
+ String value = "this is a string";
+ region.put(key, value);
+ HeapDataOutputStream hdos = new HeapDataOutputStream(Version.CURRENT);
+ DataSerializer.writeObject(value, hdos);
+ byte[] uncompressedBytes = hdos.toByteArray();
+ byte[] expectedValue = SnappyCompressor.getDefaultInstance().compress(uncompressedBytes);
+ expected.add(new ExpectedValues(expectedValue, 32, "compressed object of size " + expectedValue.length, -1, getMemoryAddress(region, key), 1, 0, true, true));
+ }
+
private void putDate(Region<Object, Object> region, List<ExpectedValues> expected) {
String key = "keyDate";
Date value = new Date();
@@ -302,6 +318,13 @@ public class OffHeapValidationJUnitTest {
region.put(key, value);
expected.add(new ExpectedValues(value, 24, "byte[10]", -1, getMemoryAddress(region, key), 1, 0, false, false));
}
+ private void putCompressedByteArray(Region<Object, Object> region, List<ExpectedValues> expected) throws IOException {
+ String key = "keyByteArray";
+ byte[] value = new byte[10];
+ region.put(key, value);
+ byte[] expectedValue = SnappyCompressor.getDefaultInstance().compress(value);
+ expected.add(new ExpectedValues(expectedValue, 24, "compressed byte[" + expectedValue.length + "]", -1, getMemoryAddress(region, key), 1, 0, true, false));
+ }
private void putByteArrayArray(Region<Object, Object> region, List<ExpectedValues> expected) {
String key = "keyByteArrayArray";