You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2017/05/23 15:03:18 UTC
tinkerpop git commit: Improved memory usage when deserializing
TinkerGraph from GraphSON
Repository: tinkerpop
Updated Branches:
refs/heads/tp32 afa3432e4 -> 7ea387518
Improved memory usage when deserializing TinkerGraph from GraphSON
Streamed vertices/edges rather than reading them all into memory at once. CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7ea38751
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7ea38751
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7ea38751
Branch: refs/heads/tp32
Commit: 7ea38751871f6eddd4c4bcfe8ac82300a08f5a7d
Parents: afa3432
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 23 11:01:55 2017 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 23 11:01:55 2017 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../structure/TinkerIoRegistryV2d0.java | 40 +++++++++-----------
2 files changed, 18 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7ea38751/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 7057aac..be8059d 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
TinkerPop 3.2.5 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Reduced memory usage for TinkerGraph deserialization in GraphSON by streaming vertices and edges.
* Now using Groovy `[...]` map notation in `GroovyTranslator` instead of `new LinkedHashMap(){{ }}`.
* Maintained type information on `Traversal.promise()`.
* Propagated exception to `Future` instead of calling thread in `RemoteConnection`.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7ea38751/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
index 727931c..394216b 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java
@@ -26,7 +26,6 @@ import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONUtil;
import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
@@ -37,12 +36,11 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
import org.apache.tinkerpop.shaded.jackson.core.JsonParser;
import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException;
+import org.apache.tinkerpop.shaded.jackson.core.JsonToken;
import org.apache.tinkerpop.shaded.jackson.databind.DeserializationContext;
import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider;
import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer;
-import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer;
import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
-import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
import org.apache.tinkerpop.shaded.kryo.io.Input;
@@ -53,7 +51,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
/**
@@ -201,26 +198,23 @@ public final class TinkerIoRegistryV2d0 extends AbstractIoRegistry {
conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality", "list");
final TinkerGraph graph = TinkerGraph.open(conf);
- final List<? extends Edge> edges;
- final List<? extends Vertex> vertices;
-
- jsonParser.nextToken();
- final Map<String, Object> graphData = deserializationContext.readValue(jsonParser, Map.class);
- vertices = (List<DetachedVertex>) graphData.get(GraphSONTokens.VERTICES);
- edges = (List<DetachedEdge>) graphData.get(GraphSONTokens.EDGES);
-
-
- vertices.forEach(e -> {
- if (e instanceof DetachedVertex) {
- ((DetachedVertex)e).attach(Attachable.Method.getOrCreate(graph));
+ while (jsonParser.nextToken() != JsonToken.END_OBJECT) {
+ if (jsonParser.getCurrentName().equals("vertices")) {
+ while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+ if (jsonParser.currentToken() == JsonToken.START_OBJECT) {
+ final DetachedVertex v = (DetachedVertex) deserializationContext.readValue(jsonParser, Vertex.class);
+ v.attach(Attachable.Method.getOrCreate(graph));
+ }
+ }
+ } else if (jsonParser.getCurrentName().equals("edges")) {
+ while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+ if (jsonParser.currentToken() == JsonToken.START_OBJECT) {
+ final DetachedEdge e = (DetachedEdge) deserializationContext.readValue(jsonParser, Edge.class);
+ e.attach(Attachable.Method.getOrCreate(graph));
+ }
+ }
}
- });
-
- edges.forEach(e -> {
- if (e instanceof DetachedEdge) {
- ((DetachedEdge) e).attach(Attachable.Method.getOrCreate(graph));
- }
- });
+ }
return graph;
}