You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/10/26 14:18:54 UTC
[22/29] tinkerpop git commit: if the System.getProperty() has a
shim-service set,
then it is also set as a DRIVER_JAVA_OPTIONS and an EXECUTOR_JAVA_OPTIONS in
SparkGraphComputer (this is necessary for Spark 2.0). Also,
some is.testing class registrations
if the System.getProperty() has a shim-service set, then it is also set as a DRIVER_JAVA_OPTIONS and an EXECUTOR_JAVA_OPTIONS in SparkGraphComputer (this is necessary for Spark 2.0). Also, some is.testing class registrations are now also needed outside of testing -- Spark 2.0 anonomaly.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/c3d1c44e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/c3d1c44e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/c3d1c44e
Branch: refs/heads/TINKERPOP-1389
Commit: c3d1c44e326c4f4db763039ff6b1abfc6d5bac6d
Parents: 508e1fd
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Oct 24 15:21:23 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:14:21 2016 -0600
----------------------------------------------------------------------
.../spark/process/computer/SparkGraphComputer.java | 8 ++++++++
.../spark/structure/io/gryo/GryoRegistrator.java | 12 +++++-------
2 files changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c3d1c44e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 40598c0..36618a1 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -70,6 +70,7 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.File;
import java.io.IOException;
@@ -105,6 +106,13 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
super(hadoopGraph);
this.sparkConfiguration = new HadoopConfiguration();
ConfigurationUtils.copy(this.hadoopGraph.configuration(), this.sparkConfiguration);
+ if (null != System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)) {
+ final String shimService = System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE);
+ this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c3d1c44e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
index ffd731a..9495375 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -178,13 +178,11 @@ public class GryoRegistrator implements KryoRegistrator {
// duplication, but it would be a bit cumbersome to do so without disturbing
// the ordering of the existing entries in that constructor, since not all
// of the entries are for TinkerPop (and the ordering is significant).
- if (Boolean.valueOf(System.getProperty("is.testing", "false"))) {
- try {
- m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
- m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
- } catch (final ClassNotFoundException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
+ try {
+ m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
+ m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
+ } catch (final ClassNotFoundException e) {
+ throw new IllegalStateException(e.getMessage(), e);
}
m.put(WrappedArray.ofRef.class, null);
m.put(MessagePayload.class, null);