You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/04/24 17:00:04 UTC
incubator-tinkerpop git commit: Add batching and transaction support
to readGraph() in gryo.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/refactor-io a92e1cead -> f5d5fcb14
Add batching and transaction support to readGraph() in gryo.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f5d5fcb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f5d5fcb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f5d5fcb1
Branch: refs/heads/refactor-io
Commit: f5d5fcb14c6f9bd359222417b10a460aee05271d
Parents: a92e1ce
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 24 10:59:40 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 24 10:59:40 2015 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoReader.java | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f5d5fcb1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
index 921c32d..ec9e512 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
@@ -37,8 +37,9 @@ import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
/**
@@ -100,10 +101,19 @@ public class GryoReader implements GraphReader {
// dual pass - create all vertices and store to cache the ids. then create edges. as long as we don't
// have vertex labels in the output we can't do this single pass
final Map<StarGraph.StarVertex,Vertex> cache = new HashMap<>();
- IteratorUtils.iterate(new VertexInputIterator(new Input(inputStream), attachable ->
- cache.put((StarGraph.StarVertex) attachable.get(), attachable.attach(Attachable.Method.create(graphToWriteTo)))));
- cache.entrySet().forEach(kv -> kv.getKey().edges(Direction.OUT)
- .forEachRemaining(e -> ((StarGraph.StarEdge) e).attach(Attachable.Method.create(kv.getValue()))));
+ final AtomicLong counter = new AtomicLong(0);
+ final boolean supportsTx = graphToWriteTo.features().graph().supportsTransactions();
+ IteratorUtils.iterate(new VertexInputIterator(new Input(inputStream), attachable -> {
+ final Vertex v = cache.put((StarGraph.StarVertex) attachable.get(), attachable.attach(Attachable.Method.create(graphToWriteTo)));
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+ return v;
+ }));
+ cache.entrySet().forEach(kv -> kv.getKey().edges(Direction.OUT).forEachRemaining(e -> {
+ ((StarGraph.StarEdge) e).attach(Attachable.Method.create(kv.getValue()));
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+ }));
}
@Override