You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/12 21:21:14 UTC
incubator-tinkerpop git commit: minor optimizations. GraphSONReader
proving itself at scale.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 33bd19b82 -> dd0c2b201
minor optimizations. GraphSONReader proving itself at scale.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/dd0c2b20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/dd0c2b20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/dd0c2b20
Branch: refs/heads/master
Commit: dd0c2b2010e0aff6ef62b1fc5a18cf1c217bd823
Parents: 33bd19b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Mar 12 14:20:57 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Mar 12 14:21:10 2015 -0600
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoReader.java | 4 ++--
.../structure/util/detached/DetachedEdge.java | 14 ++++++++++++--
.../computer/giraph/GiraphComputeVertex.java | 19 ++++---------------
.../hadoop/structure/io/VertexWritable.java | 11 +++++------
.../structure/io/gryo/VertexStreamIterator.java | 3 +--
.../tinkergraph/structure/TinkerEdge.java | 10 +++++++++-
6 files changed, 33 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/dd0c2b20/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
index d8109a7..4088526 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
@@ -308,8 +308,8 @@ public class GryoReader implements GraphReader {
Object next = kryo.readClassAndObject(input);
while (!next.equals(EdgeTerminator.INSTANCE)) {
final DetachedEdge detachedEdge = (DetachedEdge) next;
- final Vertex vOut = graphToWriteTo.vertices(detachedEdge.vertices(Direction.OUT).next().id()).next();
- final Vertex inV = graphToWriteTo.vertices(detachedEdge.vertices(Direction.IN).next().id()).next();
+ final Vertex vOut = graphToWriteTo.vertices(detachedEdge.outVertex().id()).next();
+ final Vertex inV = graphToWriteTo.vertices(detachedEdge.inVertex().id()).next();
detachedEdge.properties().forEachRemaining(p -> edgeArgs.addAll(Arrays.asList(p.key(), p.value())));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/dd0c2b20/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/detached/DetachedEdge.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/detached/DetachedEdge.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/detached/DetachedEdge.java
index 4c8ea05..36e0d43 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/detached/DetachedEdge.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/detached/DetachedEdge.java
@@ -60,8 +60,8 @@ public class DetachedEdge extends DetachedElement<Edge> implements Edge {
protected DetachedEdge(final Edge edge, final boolean withProperties) {
super(edge);
- this.outVertex = DetachedFactory.detach(edge.vertices(Direction.OUT).next(), false);
- this.inVertex = DetachedFactory.detach(edge.vertices(Direction.IN).next(), false);
+ this.outVertex = DetachedFactory.detach(edge.outVertex(), false);
+ this.inVertex = DetachedFactory.detach(edge.inVertex(), false);
// only serialize properties if requested, the graph supports it and there are meta properties present.
// this prevents unnecessary object creation of a new HashMap of a new HashMap which will just be empty.
@@ -143,6 +143,16 @@ public class DetachedEdge extends DetachedElement<Edge> implements Edge {
}
@Override
+ public Vertex inVertex() {
+ return this.inVertex;
+ }
+
+ @Override
+ public Vertex outVertex() {
+ return this.outVertex;
+ }
+
+ @Override
public Iterator<Vertex> vertices(final Direction direction) {
switch (direction) {
case OUT:
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/dd0c2b20/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
index 670e239..da3c7ac 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
@@ -57,8 +57,6 @@ public final class GiraphComputeVertex extends Vertex<LongWritable, Text, NullWr
private static final String VERTEX_ID = Graph.Hidden.hide("giraph.gremlin.vertexId");
private TinkerVertex tinkerVertex;
private StrategyVertex wrappedVertex;
- private GryoWriter KRYO_WRITER;
- private GryoReader KRYO_READER;
public GiraphComputeVertex() {
}
@@ -95,18 +93,11 @@ public final class GiraphComputeVertex extends Vertex<LongWritable, Text, NullWr
///////////////////////////////////////////////
- private void checkReaderWriters() {
- if (null == KRYO_READER)
- KRYO_READER = GryoReader.build().create();
- if (null == KRYO_WRITER)
- KRYO_WRITER = GryoWriter.build().create();
- }
private Text deflateTinkerVertex() {
try {
- checkReaderWriters();
final ByteArrayOutputStream bos = new ByteArrayOutputStream();
- KRYO_WRITER.writeGraph(bos, this.tinkerVertex.graph());
+ GryoWriter.build().create().writeGraph(bos, this.tinkerVertex.graph());
bos.flush();
bos.close();
return new Text(bos.toByteArray());
@@ -117,10 +108,9 @@ public final class GiraphComputeVertex extends Vertex<LongWritable, Text, NullWr
private void inflateTinkerVertex() {
try {
- checkReaderWriters();
final ByteArrayInputStream bis = new ByteArrayInputStream(this.getValue().getBytes());
final TinkerGraph tinkerGraph = TinkerGraph.open();
- KRYO_READER.readGraph(bis, tinkerGraph);
+ GryoReader.build().create().readGraph(bis, tinkerGraph);
bis.close();
this.tinkerVertex = (TinkerVertex) tinkerGraph.vertices(tinkerGraph.variables().get(VERTEX_ID).get()).next();
} catch (final Exception e) {
@@ -133,9 +123,8 @@ public final class GiraphComputeVertex extends Vertex<LongWritable, Text, NullWr
return (TinkerVertex) otherVertex;
else {
try {
- checkReaderWriters();
final ByteArrayOutputStream bos = new ByteArrayOutputStream();
- KRYO_WRITER.writeVertex(bos, otherVertex, Direction.BOTH);
+ GryoWriter.build().create().writeVertex(bos, otherVertex, Direction.BOTH);
bos.flush();
bos.close();
final ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
@@ -144,7 +133,7 @@ public final class GiraphComputeVertex extends Vertex<LongWritable, Text, NullWr
final Function<DetachedVertex, org.apache.tinkerpop.gremlin.structure.Vertex> vertexMaker = detachedVertex -> DetachedVertex.addTo(tinkerGraph, detachedVertex);
final Function<DetachedEdge, Edge> edgeMaker = detachedEdge -> DetachedEdge.addTo(tinkerGraph, detachedEdge);
try (InputStream in = new ByteArrayInputStream(bos.toByteArray())) {
- tinkerVertex = (TinkerVertex) KRYO_READER.readVertex(in, Direction.BOTH, vertexMaker, edgeMaker);
+ tinkerVertex = (TinkerVertex) GryoReader.build().create().readVertex(in, Direction.BOTH, vertexMaker, edgeMaker);
}
bis.close();
return tinkerVertex;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/dd0c2b20/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
index bbf942d..f1cedaf 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
@@ -18,6 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -27,8 +29,6 @@ import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -42,8 +42,6 @@ import java.io.IOException;
public final class VertexWritable<V extends Vertex> implements Writable {
private Vertex vertex;
- private final GryoWriter KRYO_WRITER = GryoWriter.build().create();
- private final GryoReader KRYO_READER = GryoReader.build().create();
public VertexWritable(final Vertex vertex) {
this.vertex = vertex;
@@ -59,9 +57,10 @@ public final class VertexWritable<V extends Vertex> implements Writable {
@Override
public void readFields(final DataInput input) throws IOException {
+ this.vertex = null;
final ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[WritableUtils.readVInt(input)]);
final Graph gLocal = TinkerGraph.open();
- this.vertex = KRYO_READER.readVertex(inputStream, Direction.BOTH,
+ this.vertex = GryoReader.build().create().readVertex(inputStream, Direction.BOTH,
detachedVertex -> DetachedVertex.addTo(gLocal, detachedVertex),
detachedEdge -> DetachedEdge.addTo(gLocal, detachedEdge));
@@ -70,7 +69,7 @@ public final class VertexWritable<V extends Vertex> implements Writable {
@Override
public void write(final DataOutput output) throws IOException {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- KRYO_WRITER.writeVertex(outputStream, this.vertex, Direction.BOTH);
+ GryoWriter.build().create().writeVertex(outputStream, this.vertex, Direction.BOTH);
WritableUtils.writeVInt(output, outputStream.size());
output.write(outputStream.toByteArray());
outputStream.close();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/dd0c2b20/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/VertexStreamIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/VertexStreamIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/VertexStreamIterator.java
index dcd5a49..fe1cfe7 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/VertexStreamIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/VertexStreamIterator.java
@@ -44,7 +44,6 @@ public class VertexStreamIterator implements Iterator<VertexWritable> {
private static final int[] TERMINATOR = new int[]{58, 21, 138, 17, 112, 155, 153, 150};
private final InputStream inputStream;
- private final GryoReader GRYO_READER = GryoReader.build().create();
private final ByteArrayOutputStream output = new ByteArrayOutputStream();
private int currentByte;
@@ -126,7 +125,7 @@ public class VertexStreamIterator implements Iterator<VertexWritable> {
final Function<DetachedVertex, Vertex> vertexMaker = detachedVertex -> DetachedVertex.addTo(gLocal, detachedVertex);
final Function<DetachedEdge, Edge> edgeMaker = detachedEdge -> DetachedEdge.addTo(gLocal, detachedEdge);
try (InputStream in = new ByteArrayInputStream(this.output.toByteArray())) {
- return GRYO_READER.readVertex(in, Direction.BOTH, vertexMaker, edgeMaker);
+ return GryoReader.build().create().readVertex(in, Direction.BOTH, vertexMaker, edgeMaker);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/dd0c2b20/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerEdge.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerEdge.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerEdge.java
index 6dbc497..3e8fc70 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerEdge.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerEdge.java
@@ -92,7 +92,15 @@ public class TinkerEdge extends TinkerElement implements Edge {
}
- //////////////////////////////////////////////
+ @Override
+ public Vertex outVertex() {
+ return this.outVertex;
+ }
+
+ @Override
+ public Vertex inVertex() {
+ return this.inVertex;
+ }
@Override
public Iterator<Vertex> vertices(final Direction direction) {