You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/18 21:24:14 UTC
incubator-tinkerpop git commit: GryoReader and GryoWriter now support
readObject() and writeObject() respectively. GiraphComputeVertex ID is
ObjectWritable -- no longer LongWritable constrained. ObjectWritable uses
Gryo instead of Java serialization ---
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 207d458f8 -> 839cb8ae2
GryoReader and GryoWriter now support readObject() and writeObject() respectively. GiraphComputeVertex ID is ObjectWritable -- no longer LongWritable constrained. ObjectWritable uses Gryo instead of Java serialization --- in order to reduce message sizes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/839cb8ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/839cb8ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/839cb8ae
Branch: refs/heads/master
Commit: 839cb8ae2d9977e63f2fe90d7bfa32cc46b540ff
Parents: 207d458
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 18 14:24:09 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 18 14:24:09 2015 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 4 ++
.../graph/traversal/step/map/GroupStep.java | 9 ++++-
.../gremlin/structure/io/gryo/GryoPool.java | 39 ++++++++------------
.../gremlin/structure/io/gryo/GryoReader.java | 4 ++
.../gremlin/structure/io/gryo/GryoWriter.java | 4 ++
.../process/computer/giraph/EmptyOutEdges.java | 11 +++---
.../computer/giraph/GiraphComputeVertex.java | 5 +--
.../computer/giraph/GiraphGraphComputer.java | 4 +-
.../computer/giraph/GiraphMessageCombiner.java | 11 +++---
.../computer/giraph/GiraphMessenger.java | 5 +--
.../computer/giraph/GiraphWorkerContext.java | 3 ++
.../hadoop/structure/io/ObjectWritable.java | 38 ++++++++++++-------
.../hadoop/structure/io/VertexWritable.java | 12 +++---
.../gremlin/hadoop/HadoopGraphProvider.java | 15 ++++----
14 files changed, 90 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 0a7e21f..3910e7f 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -25,6 +25,10 @@ image::http://www.tinkerpop.com/docs/current/images/gremlin-hindu.png[width=225]
TinkerPop 3.0.0.M8 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Hadoop-Gremlin `ObjectWritable` now leverages Kryo for data serialization.
+* `GiraphGraphComputer` supports arbitrary objects as the vertex id -- previously, only long ids were supported.
+* Added `VertexProgramPool` to support thread safe pooling of vertex programs for graph computers that provide threaded workers.
+* Added `GryoPool` to support thread safe pooling of Gryo readers and writers.
* Added `TraversalSource` which contextualizes a traversal to a graph, DSL, execution engine, and runtime strategies.
* Added `AddVertexStep` (`addV`), `AddPropertyStep` (`property`), and changed `AddEdgeStep` to a map-step instead of a sideEffect-step.
* Added `compile` method to `GremlinExecutor` and related classes.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
index ae75c70..9bbe439 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
@@ -39,7 +39,6 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@@ -228,7 +227,13 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
public void map(final Vertex vertex, final MapEmitter<K, Collection<V>> emitter) {
vertex.<TraverserSet<Object[]>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet -> traverserSet.forEach(traverser -> {
final Object[] objects = traverser.get();
- emitter.emit((K) objects[0], objects[1] instanceof Collection ? (Collection<V>) objects[1] : Arrays.asList((V) objects[1]));
+ if (objects[1] instanceof Collection)
+ emitter.emit((K) objects[0], (Collection<V>) objects[1]);
+ else {
+ final List<V> collection = new ArrayList<>();
+ collection.add((V) objects[1]);
+ emitter.emit((K) objects[0], collection);
+ }
}));
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
index 0b09c5d..3d41070 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
@@ -19,8 +19,8 @@
package org.apache.tinkerpop.gremlin.structure.io.gryo;
import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.function.BiFunction;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Consumer;
import java.util.function.Function;
/**
@@ -30,40 +30,32 @@ public class GryoPool {
private final Queue<GryoReader> gryoReaders;
private final Queue<GryoWriter> gryoWriters;
- private static final Integer MAX_QUEUE_SIZE = 256;
- public GryoPool() {
- this.gryoReaders = new ConcurrentLinkedQueue<>();
- this.gryoWriters = new ConcurrentLinkedQueue<>();
+ public GryoPool(final int poolSize) {
+ this.gryoReaders = new LinkedBlockingQueue<>(poolSize);
+ this.gryoWriters = new LinkedBlockingQueue<>(poolSize);
+ for (int i = 0; i < poolSize; i++) {
+ this.gryoReaders.add(GryoReader.build().create());
+ this.gryoWriters.add(GryoWriter.build().create());
+ }
}
public GryoReader takeReader() {
final GryoReader reader = this.gryoReaders.poll();
- return (null == reader) ? GryoReader.build().create() : reader;
+ return null == reader ? GryoReader.build().create() : reader;
}
public GryoWriter takeWriter() {
final GryoWriter writer = this.gryoWriters.poll();
- return (null == writer) ? GryoWriter.build().create() : writer;
+ return null == writer ? GryoWriter.build().create() : writer;
}
public void offerReader(final GryoReader gryoReader) {
- if (this.gryoReaders.size() < MAX_QUEUE_SIZE)
- this.gryoReaders.offer(gryoReader);
+ this.gryoReaders.offer(gryoReader);
}
public void offerWriter(final GryoWriter gryoWriter) {
- if (this.gryoWriters.size() < MAX_QUEUE_SIZE)
- this.gryoWriters.offer(gryoWriter);
- }
-
- public <A> A doWithReaderWriter(final BiFunction<GryoReader, GryoWriter, A> readerWriterBiFunction) {
- final GryoReader gryoReader = this.takeReader();
- final GryoWriter gryoWriter = this.takeWriter();
- final A a = readerWriterBiFunction.apply(gryoReader, gryoWriter);
- this.offerReader(gryoReader);
- this.offerWriter(gryoWriter);
- return a;
+ this.gryoWriters.offer(gryoWriter);
}
public <A> A doWithReader(final Function<GryoReader, A> readerFunction) {
@@ -73,10 +65,9 @@ public class GryoPool {
return a;
}
- public <A> A doWithWriter(final Function<GryoWriter, A> writerFunction) {
+ public void doWithWriter(final Consumer<GryoWriter> writerFunction) {
final GryoWriter gryoWriter = this.takeWriter();
- final A a = writerFunction.apply(gryoWriter);
+ writerFunction.accept(gryoWriter);
this.offerWriter(gryoWriter);
- return a;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
index 5bb327d..03bf58a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
@@ -332,6 +332,10 @@ public class GryoReader implements GraphReader {
}
}
+ public <T> T readObject(final Input input) {
+ return (T) this.kryo.readClassAndObject(input);
+ }
+
public static Builder build() {
return new Builder();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
index 454571b..65d128b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
@@ -149,6 +149,10 @@ public class GryoWriter implements GraphWriter {
kryo.writeClassAndObject(output, EdgeTerminator.INSTANCE);
}
+ public void writeObject(final Output output, final Object object) {
+ this.kryo.writeClassAndObject(output, object);
+ }
+
public static Builder build() {
return new Builder();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
index b5de047..6484b03 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
@@ -22,6 +22,7 @@ import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.OutEdges;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import java.io.DataInput;
import java.io.DataOutput;
@@ -32,7 +33,7 @@ import java.util.Iterator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class EmptyOutEdges implements OutEdges<LongWritable, NullWritable> {
+public class EmptyOutEdges implements OutEdges<ObjectWritable, NullWritable> {
private static final EmptyOutEdges INSTANCE = new EmptyOutEdges();
@@ -41,7 +42,7 @@ public class EmptyOutEdges implements OutEdges<LongWritable, NullWritable> {
}
@Override
- public void initialize(final Iterable<Edge<LongWritable, NullWritable>> edges) {
+ public void initialize(final Iterable<Edge<ObjectWritable, NullWritable>> edges) {
}
@Override
@@ -53,11 +54,11 @@ public class EmptyOutEdges implements OutEdges<LongWritable, NullWritable> {
}
@Override
- public void add(final Edge<LongWritable, NullWritable> edge) {
+ public void add(final Edge<ObjectWritable, NullWritable> edge) {
}
@Override
- public void remove(final LongWritable targetVertexId) {
+ public void remove(final ObjectWritable targetVertexId) {
}
@Override
@@ -66,7 +67,7 @@ public class EmptyOutEdges implements OutEdges<LongWritable, NullWritable> {
}
@Override
- public Iterator<Edge<LongWritable, NullWritable>> iterator() {
+ public Iterator<Edge<ObjectWritable, NullWritable>> iterator() {
return Collections.emptyIterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
index 1d657b3..4ce5056 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
@@ -19,7 +19,6 @@
package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
@@ -32,7 +31,7 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class GiraphComputeVertex extends Vertex<LongWritable, VertexWritable, NullWritable, ObjectWritable> {
+public final class GiraphComputeVertex extends Vertex<ObjectWritable, VertexWritable, NullWritable, ObjectWritable> {
//TODO: Dangerous that the underlying TinkerGraph Vertex can have edges written to it.
//TODO: LongWritable as the key is not general enough -- ObjectWritable causes problems though :|
@@ -43,7 +42,7 @@ public final class GiraphComputeVertex extends Vertex<LongWritable, VertexWritab
public GiraphComputeVertex(final VertexWritable vertexWritable) {
final VertexWritable newWritable = new VertexWritable();
newWritable.set(vertexWritable.get());
- this.initialize(new LongWritable(Long.valueOf(newWritable.get().id().toString())), newWritable, EmptyOutEdges.instance());
+ this.initialize(new ObjectWritable<>(newWritable.get().id()), newWritable, EmptyOutEdges.instance());
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
index 578f16a..a84a7af 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
@@ -40,6 +39,7 @@ import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io.GiraphVert
import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MemoryMapReduce;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
@@ -86,7 +86,7 @@ public class GiraphGraphComputer extends Configured implements GraphComputer, To
this.giraphConfiguration.setVertexClass(GiraphComputeVertex.class);
this.giraphConfiguration.setWorkerContextClass(GiraphWorkerContext.class);
this.giraphConfiguration.setOutEdgesClass(EmptyOutEdges.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), LongWritable.class, LongWritable.class);
+ this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), ObjectWritable.class, ObjectWritable.class);
this.giraphConfiguration.setClass(GiraphConstants.VERTEX_VALUE_CLASS.getKey(), VertexWritable.class, VertexWritable.class);
this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
index 0f42081..ef71207 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
@@ -18,25 +18,24 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
+import org.apache.giraph.combiner.Combiner;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.giraph.combiner.Combiner;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.hadoop.io.LongWritable;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class GiraphMessageCombiner extends Combiner<LongWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
+public class GiraphMessageCombiner extends Combiner<ObjectWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
private MessageCombiner messageCombiner;
private ImmutableClassesGiraphConfiguration configuration;
@Override
- public void combine(final LongWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
+ public void combine(final ObjectWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
originalMessage.set(originalMessage.isEmpty() ?
messageToCombine.get() :
this.messageCombiner.combine(originalMessage.get(), messageToCombine.get()));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
index c0bec5d..b2855de 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
@@ -29,7 +29,6 @@ import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import org.apache.hadoop.io.LongWritable;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -57,12 +56,12 @@ public class GiraphMessenger<M> implements Messenger<M> {
final Direction direction = GiraphMessenger.getOppositeDirection(incidentTraversal);
incidentTraversal.forEachRemaining(edge ->
this.giraphComputeVertex.sendMessage(
- new LongWritable(Long.valueOf(edge.vertices(direction).next().id().toString())),
+ new ObjectWritable<>(edge.vertices(direction).next().id()),
new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge))));
} else {
final MessageScope.Global globalMessageScope = (MessageScope.Global) messageScope;
globalMessageScope.vertices().forEach(vertex ->
- this.giraphComputeVertex.sendMessage(new LongWritable(Long.valueOf(vertex.id().toString())), new ObjectWritable<>(message)));
+ this.giraphComputeVertex.sendMessage(new ObjectWritable<>(vertex.id()), new ObjectWritable<>(message)));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
index a315c5c..2017a6b 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
@@ -26,12 +26,15 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public final class GiraphWorkerContext extends WorkerContext {
+ public static final GryoPool GRYO_POOL = new GryoPool(256);
+
private VertexProgramPool vertexProgramPool;
private GiraphMemory memory;
private boolean deriveMemory;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
index 1d5cd69..3b03714 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
@@ -18,10 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-import org.apache.tinkerpop.gremlin.util.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphWorkerContext;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -55,23 +59,29 @@ public final class ObjectWritable<T> implements WritableComparable<ObjectWritabl
@Override
public void readFields(final DataInput input) throws IOException {
- try {
- this.t = (T) Serializer.deserializeObject(WritableUtils.readCompressedByteArray(input));
- } catch (final ClassNotFoundException e) {
- throw new IOException(e.getMessage(), e);
- }
- //this.t = (T) Constants.GRYO.readClassAndObject(new Input(new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input))));
+ this.t = GiraphWorkerContext.GRYO_POOL.doWithReader(gryoReader -> {
+ try {
+ return gryoReader.readObject(new Input(new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input))));
+ } catch (IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ });
}
@Override
public void write(final DataOutput output) throws IOException {
- WritableUtils.writeCompressedByteArray(output, Serializer.serializeObject(this.t));
- /*final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- final Output out = new Output(outputStream);
- Constants.GRYO.writeClassAndObject(out, this.t);
- out.flush();
- WritableUtils.writeCompressedByteArray(output, outputStream.toByteArray());
- out.close();*/
+ GiraphWorkerContext.GRYO_POOL.doWithWriter(gryoWriter -> {
+ try {
+ final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ final Output out = new Output(outputStream);
+ gryoWriter.writeObject(out, this.t);
+ out.flush();
+ WritableUtils.writeCompressedByteArray(output, outputStream.toByteArray());
+ out.close();
+ } catch (IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ });
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
index c180ec7..bd8aa07 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
@@ -20,10 +20,10 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphWorkerContext;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
@@ -41,7 +41,6 @@ import java.io.IOException;
public final class VertexWritable implements Writable {
private Vertex vertex;
- private static final GryoPool GRYO_POOL = new GryoPool();
public VertexWritable() {
@@ -64,7 +63,7 @@ public final class VertexWritable implements Writable {
public void readFields(final DataInput input) throws IOException {
try {
this.vertex = null;
- this.vertex = GRYO_POOL.doWithReader(gryoReader -> {
+ this.vertex = GiraphWorkerContext.GRYO_POOL.doWithReader(gryoReader -> {
try {
final ByteArrayInputStream inputStream = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
final Graph gLocal = TinkerGraph.open();
@@ -72,7 +71,7 @@ public final class VertexWritable implements Writable {
detachedVertex -> DetachedVertex.addTo(gLocal, detachedVertex),
detachedEdge -> DetachedEdge.addTo(gLocal, detachedEdge));
} catch (final IOException e) {
- throw new IllegalStateException(e);
+ throw new IllegalStateException(e.getMessage(), e);
}
});
} catch (final IllegalStateException e) {
@@ -86,14 +85,13 @@ public final class VertexWritable implements Writable {
@Override
public void write(final DataOutput output) throws IOException {
try {
- GRYO_POOL.doWithWriter(gryoWriter -> {
+ GiraphWorkerContext.GRYO_POOL.doWithWriter(gryoWriter -> {
try {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
gryoWriter.writeVertex(outputStream, this.vertex, Direction.BOTH);
WritableUtils.writeCompressedByteArray(output, outputStream.toByteArray());
- return null;
} catch (final IOException e) {
- throw new IllegalStateException(e);
+ throw new IllegalStateException(e.getMessage(), e);
}
});
} catch (final IllegalStateException e) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/839cb8ae/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index d127132..1cf2370 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -111,14 +111,13 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
put(GiraphConstants.MAX_WORKERS, 1);
put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
put(GiraphConstants.ZOOKEEPER_JAR, GiraphGraphComputer.class.getResource("zookeeper-3.3.3.jar").getPath());
- put("giraph.zkServerPort", "2181"); // you must have a local zookeeper running on this port
- put("giraph.nettyServerUseExecutionHandler", false); // this prevents so many integration tests running out of threads
- put("giraph.nettyClientUseExecutionHandler", false); // this prevents so many integration tests running out of threads
- put("giraph.numInputThreads", 3);
- put("giraph.numComputeThreads", 3);
- put("giraph.vertexOutputFormatThreadSafe", true);
- put("giraph.numOutputThreads", 3);
-
+ put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
+ put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
+ put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
+ //put("giraph.vertexOutputFormatThreadSafe", false);
+ //put("giraph.numOutputThreads", 3);
/// spark configuration
put("spark.master", "local[4]");