You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/09/11 18:05:45 UTC
incubator-tinkerpop git commit: Added bufferSize configuration option
to gryo serialization.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/tp30 ac2e443a2 -> 6a333d7f7
Added bufferSize configuration option to gryo serialization.
Without this setting "Buffer too small" KryoException could get raised if a single Request/Response for Gremlin Server exceeded 4096 (the default value set by Kryo for Output instances).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6a333d7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6a333d7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6a333d7f
Branch: refs/heads/tp30
Commit: 6a333d7f7c0e3d5e9c800cd4ae71a46771b8c296
Parents: ac2e443
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Sep 11 12:04:30 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Sep 11 12:04:30 2015 -0400
----------------------------------------------------------------------
docs/src/gremlin-applications.asciidoc | 1 +
.../driver/ser/GryoMessageSerializerV1d0.java | 13 +--
.../ser/GryoMessageSerializerV1d0Test.java | 86 +++++++++++++++++---
.../server/GremlinDriverIntegrateTest.java | 4 +-
4 files changed, 86 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a333d7f/docs/src/gremlin-applications.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/gremlin-applications.asciidoc b/docs/src/gremlin-applications.asciidoc
index cf63054..51c2219 100644
--- a/docs/src/gremlin-applications.asciidoc
+++ b/docs/src/gremlin-applications.asciidoc
@@ -784,6 +784,7 @@ It has the MIME type of `application/vnd.gremlin-v1.0+gryo` and the following co
[width="100%",cols="3,10,^2",options="header"]
|=========================================================
|Key |Description |Default
+|bufferSize |The maximum size of the Kryo buffer for use on a single object being serialized. Increasing this value will correct `KryoException` errors that complain of "Buffer too small". |_4096_
|serializeResultToString |When set to `true`, results are serialized by first calling `toString()` on each object in the result list resulting in an extended MIME Type of `application/vnd.gremlin-v1.0+gryo-stringd`. When set to `false` Kryo-based serialization is applied. |_false_
|useMapperFromGraph |Specifies the name of the `Graph` (from the `graphs` `Map` in the configuration file) from which to plugin any custom serializers that are tied to it. |_none_
|ioRegistries |A list of `IoRegistry` implementations to be applied to the serializer.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a333d7f/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
index 04742ac..64680b1 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
@@ -68,11 +68,13 @@ public final class GryoMessageSerializerV1d0 implements MessageSerializer {
private static final String TOKEN_CUSTOM = "custom";
private static final String TOKEN_SERIALIZE_RESULT_TO_STRING = "serializeResultToString";
private static final String TOKEN_USE_MAPPER_FROM_GRAPH = "useMapperFromGraph";
+ private static final String TOKEN_BUFFER_SIZE = "bufferSize";
- private boolean serializeToString;
+ private boolean serializeToString = false;
+ private int bufferSize = 4096;
/**
- * Creates an instance with a standard {@link org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper} instance. Note that this instance
+ * Creates an instance with a standard {@link GryoMapper} instance. Note that this instance
* will be overriden by {@link #configure} is called.
*/
public GryoMessageSerializerV1d0() {
@@ -80,7 +82,7 @@ public final class GryoMessageSerializerV1d0 implements MessageSerializer {
}
/**
- * Creates an instance with a provided mapper configured {@link org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper} instance. Note that this instance
+ * Creates an instance with a provided mapper configured {@link GryoMapper} instance. Note that this instance
* will be overriden by {@link #configure} is called.
*/
public GryoMessageSerializerV1d0(final GryoMapper kryo) {
@@ -114,6 +116,7 @@ public final class GryoMessageSerializerV1d0 implements MessageSerializer {
addCustomClasses(config, builder);
this.serializeToString = Boolean.parseBoolean(config.getOrDefault(TOKEN_SERIALIZE_RESULT_TO_STRING, "false").toString());
+ this.bufferSize = Integer.parseInt(config.getOrDefault(TOKEN_BUFFER_SIZE, "4096").toString());
this.gryoMapper = builder.create();
}
@@ -223,7 +226,7 @@ public final class GryoMessageSerializerV1d0 implements MessageSerializer {
try {
final Kryo kryo = kryoThreadLocal.get();
try (final OutputStream baos = new ByteArrayOutputStream()) {
- final Output output = new Output(baos);
+ final Output output = new Output(baos, bufferSize);
// request id - if present
kryo.writeObjectOrNull(output, responseMessage.getRequestId() != null ? responseMessage.getRequestId() : null, UUID.class);
@@ -287,7 +290,7 @@ public final class GryoMessageSerializerV1d0 implements MessageSerializer {
try {
final Kryo kryo = kryoThreadLocal.get();
try (final OutputStream baos = new ByteArrayOutputStream()) {
- final Output output = new Output(baos);
+ final Output output = new Output(baos, bufferSize);
final String mimeType = serializeToString ? MIME_TYPE_STRINGD : MIME_TYPE;
output.writeByte(mimeType.length());
output.write(mimeType.getBytes(UTF8));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a333d7f/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
index 8e377b8..e9b51a1 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.driver.ser;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
@@ -36,6 +37,7 @@ import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.apache.tinkerpop.shaded.kryo.KryoException;
import org.junit.Test;
import java.util.ArrayList;
@@ -44,10 +46,12 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
/**
* Serializer tests that cover non-lossy serialization/deserialization methods.
@@ -72,7 +76,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeIterable() throws Exception {
+ public void shouldSerializeIterable() throws Exception {
final ArrayList<Integer> list = new ArrayList<>();
list.add(1);
list.add(100);
@@ -87,7 +91,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeIterableToString() throws Exception {
+ public void shouldSerializeIterableToString() throws Exception {
final ArrayList<Integer> list = new ArrayList<>();
list.add(1);
list.add(100);
@@ -102,7 +106,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeIterableToStringWithNull() throws Exception {
+ public void shouldSerializeIterableToStringWithNull() throws Exception {
final ArrayList<Integer> list = new ArrayList<>();
list.add(1);
list.add(null);
@@ -119,7 +123,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeIterableWithNull() throws Exception {
+ public void shouldSerializeIterableWithNull() throws Exception {
final ArrayList<Integer> list = new ArrayList<>();
list.add(1);
list.add(null);
@@ -136,7 +140,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeMap() throws Exception {
+ public void shouldSerializeMap() throws Exception {
final Map<String, Object> map = new HashMap<>();
final Map<String, String> innerMap = new HashMap<>();
innerMap.put("a", "b");
@@ -159,7 +163,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeEdge() throws Exception {
+ public void shouldSerializeEdge() throws Exception {
final Graph g = TinkerGraph.open();
final Vertex v1 = g.addVertex();
final Vertex v2 = g.addVertex();
@@ -187,7 +191,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeTree() throws Exception {
+ public void shouldSerializeTree() throws Exception {
final Graph g = TinkerFactory.createModern();
final Tree t = g.traversal().V().out().out().tree().by("name").next();
@@ -211,7 +215,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeVertexWithEmbeddedMap() throws Exception {
+ public void shouldSerializeVertexWithEmbeddedMap() throws Exception {
final Graph g = TinkerGraph.open();
final Vertex v = g.addVertex();
final Map<String, Object> map = new HashMap<>();
@@ -251,7 +255,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeToMapWithElementForKey() throws Exception {
+ public void shouldSerializeToMapWithElementForKey() throws Exception {
final TinkerGraph graph = TinkerFactory.createClassic();
final GraphTraversalSource g = graph.traversal();
final Map<Vertex, Integer> map = new HashMap<>();
@@ -274,7 +278,7 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeFullResponseMessage() throws Exception {
+ public void shouldSerializeFullResponseMessage() throws Exception {
final UUID id = UUID.randomUUID();
final Map<String, Object> metaData = new HashMap<>();
@@ -307,7 +311,42 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void serializeFullRequestMessage() throws Exception {
+ public void shouldHaveTooSmallBufferToSerializeResponseMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ put("bufferSize", 1);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ try {
+ binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
+ fail("Should have a buffer size that is too small");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(KryoException.class));
+ }
+ }
+
+ @Test
+ public void shouldSerializeFullRequestMessage() throws Exception {
final UUID id = UUID.randomUUID();
final RequestMessage request = RequestMessage.build("try")
@@ -326,6 +365,31 @@ public class GryoMessageSerializerV1d0Test {
assertEquals("this", deserialized.getArgs().get("test"));
}
+ @Test
+ public void shouldHaveTooSmallBufferToSerializeRequestMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ put("bufferSize", 1);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ try {
+ binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
+ fail("Should have a buffer size that is too small");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(KryoException.class));
+ }
+ }
+
private void assertCommon(final ResponseMessage response) {
assertEquals(requestId, response.getRequestId());
assertEquals(ResponseStatusCode.SUCCESS, response.getStatus().getCode());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a333d7f/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinDriverIntegrateTest.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinDriverIntegrateTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinDriverIntegrateTest.java
index 326d746..0057d6d 100644
--- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinDriverIntegrateTest.java
+++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinDriverIntegrateTest.java
@@ -318,7 +318,7 @@ public class GremlinDriverIntegrateTest extends AbstractGremlinServerIntegration
IntStream.range(0, requests).forEach(ix -> {
refs[ix] = new AtomicReference();
client.submitAsync("Thread.sleep(5000);[1,2,3,4,5,6,7,8,9]").thenAccept(rs ->
- rs.all().thenAccept(refs[ix]::set).thenRun(latch::countDown));
+ rs.all().thenAccept(refs[ix]::set).thenRun(latch::countDown));
});
// countdown should have reached zero as results should have eventually been all returned and processed
@@ -482,7 +482,7 @@ public class GremlinDriverIntegrateTest extends AbstractGremlinServerIntegration
client.submit("'" + fatty + "'").all().get();
fail("Should throw an exception.");
} catch (Exception re) {
- Throwable root = ExceptionUtils.getRootCause(re);
+ final Throwable root = ExceptionUtils.getRootCause(re);
assertTrue(root.getMessage().equals("Max frame length of 1 has been exceeded."));
} finally {
cluster.close();