You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/04/28 16:21:30 UTC
[1/3] incubator-tinkerpop git commit: Dropped BatchGraph from
LegacyGraphSONReader.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 1a3a346d1 -> 4a1aa0203
Dropped BatchGraph from LegacyGraphSONReader.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/3d361e5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/3d361e5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/3d361e5b
Branch: refs/heads/master
Commit: 3d361e5b5e25ad1ba9bf4044b9a6655513bb1857
Parents: 1a3a346
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Apr 28 10:15:00 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Apr 28 10:15:00 2015 -0400
----------------------------------------------------------------------
.../io/graphson/LegacyGraphSONReader.java | 49 +++++++++++++-------
1 file changed, 31 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3d361e5b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java
index 0009432..947dabe 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/LegacyGraphSONReader.java
@@ -42,6 +42,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
/**
@@ -63,17 +64,14 @@ public class LegacyGraphSONReader implements GraphReader {
@Override
public void readGraph(final InputStream inputStream, final Graph graphToWriteTo) throws IOException {
- final BatchGraph<?> graph;
- try {
- // will throw an exception if not constructed properly
- graph = BatchGraph.build(graphToWriteTo)
- .bufferSize(batchSize).create();
- } catch (Exception ex) {
- throw new IOException("Could not instantiate BatchGraph wrapper", ex);
- }
+ final Map<Object,Vertex> cache = new HashMap<>();
+ final AtomicLong counter = new AtomicLong(0);
+ final boolean supportsTx = graphToWriteTo.features().graph().supportsTransactions();
+ final boolean supportsEdgeIds = graphToWriteTo.features().edge().supportsUserSuppliedIds();
+ final boolean supportsVertexIds = graphToWriteTo.features().vertex().supportsUserSuppliedIds();
final JsonFactory factory = mapper.getFactory();
- final GraphSONUtility graphson = new GraphSONUtility(graph);
+ final GraphSONUtility graphson = new GraphSONUtility(graphToWriteTo, supportsVertexIds, supportsEdgeIds, cache);
try (JsonParser parser = factory.createParser(inputStream)) {
if (parser.nextToken() != JsonToken.START_OBJECT)
@@ -93,15 +91,21 @@ public class LegacyGraphSONReader implements GraphReader {
while (parser.nextToken() != JsonToken.END_ARRAY) {
final JsonNode node = parser.readValueAsTree();
graphson.vertexFromJson(node);
+
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
}
break;
case GraphSONTokens.EDGES:
parser.nextToken();
while (parser.nextToken() != JsonToken.END_ARRAY) {
final JsonNode node = parser.readValueAsTree();
- final Vertex inV = graph.vertices(GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokens._IN_V))).next();
- final Vertex outV = graph.vertices(GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokens._OUT_V))).next();
- GraphSONUtility.edgeFromJson(node, outV, inV);
+ final Vertex inV = cache.get(GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokens._IN_V)));
+ final Vertex outV = cache.get(GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokens._OUT_V)));
+ graphson.edgeFromJson(node, outV, inV);
+
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
}
break;
default:
@@ -109,7 +113,7 @@ public class LegacyGraphSONReader implements GraphReader {
}
}
- graph.tx().commit();
+ if (supportsTx) graphToWriteTo.tx().commit();
} catch (Exception ex) {
throw new IOException(ex);
}
@@ -193,35 +197,44 @@ public class LegacyGraphSONReader implements GraphReader {
}
}
- public static class GraphSONUtility {
+ static class GraphSONUtility {
private static final String EMPTY_STRING = "";
private final Graph g;
+ private final boolean supportsVertexIds;
+ private final boolean supportsEdgeIds;
+ private final Map<Object,Vertex> cache;
- public GraphSONUtility(final Graph g) {
+ public GraphSONUtility(final Graph g, final boolean supportsVertexIds, final boolean supportsEdgeIds,
+ final Map<Object,Vertex> cache) {
this.g = g;
+ this.supportsVertexIds = supportsVertexIds;
+ this.supportsEdgeIds = supportsEdgeIds;
+ this.cache = cache;
}
public Vertex vertexFromJson(final JsonNode json) throws IOException {
final Map<String, Object> props = readProperties(json);
final Object vertexId = getTypedValueFromJsonNode(json.get(GraphSONTokens._ID));
- final Vertex v = g.addVertex(T.id, vertexId);
+ final Vertex v = supportsVertexIds ? g.addVertex(T.id, vertexId) : g.addVertex();
+ cache.put(vertexId, v);
for (Map.Entry<String, Object> entry : props.entrySet()) {
+ // todo: cardinality
v.property(VertexProperty.Cardinality.list, entry.getKey(), entry.getValue());
}
return v;
}
- public static Edge edgeFromJson(final JsonNode json, final Vertex out, final Vertex in) throws IOException {
+ public Edge edgeFromJson(final JsonNode json, final Vertex out, final Vertex in) throws IOException {
final Map<String, Object> props = GraphSONUtility.readProperties(json);
final Object edgeId = getTypedValueFromJsonNode(json.get(GraphSONTokens._ID));
final JsonNode nodeLabel = json.get(GraphSONTokens._LABEL);
final String label = nodeLabel == null ? EMPTY_STRING : nodeLabel.textValue();
- final Edge e = out.addEdge(label, in, T.id, edgeId);
+ final Edge e = supportsEdgeIds ? out.addEdge(label, in, T.id, edgeId) : out.addEdge(label, in) ;
for (Map.Entry<String, Object> entry : props.entrySet()) {
e.property(entry.getKey(), entry.getValue());
}
[2/3] incubator-tinkerpop git commit: Minor adjustments to logic when
adding a vertex to the cache.
Posted by sp...@apache.org.
Minor adjustments to logic when adding a vertex to the cache.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/be1411a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/be1411a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/be1411a8
Branch: refs/heads/master
Commit: be1411a8d1e20e297a134418d3563622879bd928
Parents: 3d361e5
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Apr 28 10:15:12 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Apr 28 10:15:12 2015 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/graphml/GraphMLReader.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/be1411a8/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
index e229e05..89aed88 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
@@ -226,10 +226,10 @@ public class GraphMLReader implements GraphReader {
if (cache.containsKey(id)) {
return cache.get(id);
} else {
- if (supportsIds)
- return cache.put(id, graphToWriteTo.addVertex(ElementHelper.upsert(args, T.id, id)));
- else
- return cache.put(id, graphToWriteTo.addVertex(args));
+ final Object [] argsReady = supportsIds ? ElementHelper.upsert(args, T.id, id) : args;
+ final Vertex v = graphToWriteTo.addVertex(argsReady);
+ cache.put(id, v);
+ return v;
}
}
[3/3] incubator-tinkerpop git commit: Fixed up batching in readers
and ensured a final commit at the end.
Posted by sp...@apache.org.
Fixed up batching in readers and ensured a final commit at the end.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4a1aa020
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4a1aa020
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4a1aa020
Branch: refs/heads/master
Commit: 4a1aa0203e826e43f8b0edd0d542c86c556eb715
Parents: be1411a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Apr 28 10:20:57 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Apr 28 10:20:57 2015 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/graphml/GraphMLReader.java | 12 ++++++++++--
.../gremlin/structure/io/graphson/GraphSONReader.java | 2 ++
.../tinkerpop/gremlin/structure/io/gryo/GryoReader.java | 2 ++
3 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4a1aa020/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
index 89aed88..88ac149 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLReader.java
@@ -155,6 +155,9 @@ public class GraphMLReader implements GraphReader {
edgeOutVertex = findOrCreate(vertexIdOut, graphToWriteTo, supportsVertexIds, cache);
edgeInVertex = findOrCreate(vertexIdIn, graphToWriteTo, supportsVertexIds, cache);
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+
isInEdge = true;
edgeProps = new HashMap<>();
@@ -191,6 +194,9 @@ public class GraphMLReader implements GraphReader {
findOrCreate(currentVertexId, graphToWriteTo, supportsVertexIds, cache, ElementHelper.upsert(propsAsArray, T.label, currentVertexLabel));
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+
vertexId = null;
vertexLabel = null;
vertexProps = null;
@@ -200,6 +206,9 @@ public class GraphMLReader implements GraphReader {
final Object[] propsReady = supportsEdgeIds ? ElementHelper.upsert(propsAsArray, T.id, edgeId) : propsAsArray;
edgeOutVertex.addEdge(edgeLabel, edgeInVertex, propsReady);
+ if (supportsTx && counter.incrementAndGet() % batchSize == 0)
+ graphToWriteTo.tx().commit();
+
edgeId = null;
edgeLabel = null;
edgeOutVertex = null;
@@ -211,8 +220,7 @@ public class GraphMLReader implements GraphReader {
}
}
- if (supportsTx && counter.incrementAndGet() % batchSize == 0)
- graphToWriteTo.tx().commit();
+ if (supportsTx) graphToWriteTo.tx().commit();
} catch (XMLStreamException xse) {
// rollback whatever portion failed
if (supportsTx && counter.incrementAndGet() % batchSize == 0)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4a1aa020/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONReader.java
index fad958a..a91cc24 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONReader.java
@@ -88,6 +88,8 @@ public class GraphSONReader implements GraphReader {
if (supportsTx && counter.incrementAndGet() % batchSize == 0)
graphToWriteTo.tx().commit();
}));
+
+ if (supportsTx) graphToWriteTo.tx().commit();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4a1aa020/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
index 95dce2c..cf19c62 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
@@ -109,6 +109,8 @@ public class GryoReader implements GraphReader {
if (supportsTx && counter.incrementAndGet() % batchSize == 0)
graphToWriteTo.tx().commit();
}));
+
+ if (supportsTx) graphToWriteTo.tx().commit();
}
@Override