You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/06 20:54:48 UTC
[17/30] incubator-tinkerpop git commit: Learned about
ExternalizableSerializer which makes registration of Kryo serializers alot
more simple. Ran this code on the cluster -- what took 25 minutes now takes
6.8 minutes.
Learned about ExternalizableSerializer which makes registration of Kryo serializers alot more simple. Ran this code on the cluster -- what took 25 minutes now takes 6.8 minutes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1c9a31c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1c9a31c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1c9a31c4
Branch: refs/heads/master
Commit: 1c9a31c4c3d3f09c829d135363ad7ebff6590c8d
Parents: 4afe29a
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Feb 3 12:49:53 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Feb 3 12:49:53 2016 -0700
----------------------------------------------------------------------
.../gremlin/spark/structure/io/gryo/GryoSerializer.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1c9a31c4/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 66314af..3819584 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -30,7 +30,6 @@ import org.apache.spark.scheduler.CompressedMapStatus;
import org.apache.spark.scheduler.HighlyCompressedMapStatus;
import org.apache.spark.serializer.Serializer;
import org.apache.spark.serializer.SerializerInstance;
-import org.apache.spark.storage.BlockManagerId;
import org.apache.spark.util.SerializableConfiguration;
import org.apache.spark.util.collection.CompactBuffer;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
@@ -41,6 +40,7 @@ import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingP
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.apache.tinkerpop.shaded.kryo.serializers.ExternalizableSerializer;
import org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer;
import scala.Tuple2;
import scala.Tuple3;
@@ -87,12 +87,11 @@ public final class GryoSerializer extends Serializer {
.addCustom(Tuple3[].class)
.addCustom(CompactBuffer.class, new CompactBufferSerializer())
.addCustom(CompactBuffer[].class)
- .addCustom(CompressedMapStatus.class)
- .addCustom(HighlyCompressedMapStatus.class)
+ .addCustom(CompressedMapStatus.class, new ExternalizableSerializer()) // externalizable implemented so its okay
+ .addCustom(HighlyCompressedMapStatus.class, new ExternalizableSerializer()) // externalizable implemented so its okay
.addCustom(HttpBroadcast.class)
.addCustom(PythonBroadcast.class)
.addCustom(BoxedUnit.class)
- .addCustom(BlockManagerId.class)
.addCustom(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer())
.addCustom(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer())
.addCustom(WrappedArray.ofRef.class, new WrappedArraySerializer())