You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/04/10 15:35:00 UTC
[4/9] incubator-tinkerpop git commit: registered the Gryo classes
with SparkGraphComputer at construction time.
registered the Gryo classes with SparkGraphComputer at construction time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5b708e45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5b708e45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5b708e45
Branch: refs/heads/TINKERPOP3-581
Commit: 5b708e45f4d807f0b06911b34f2b2f036ab45641
Parents: 5b7ad6f
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 9 17:45:33 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 9 17:45:33 2015 -0600
----------------------------------------------------------------------
.../apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java | 4 ++++
.../hadoop/process/computer/spark/SparkGraphComputer.java | 5 +++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b708e45/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 17b09e5..5ab18e6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -141,6 +141,10 @@ public final class GryoMapper implements Mapper<Kryo> {
return headerReader;
}
+ public List<Class> getRegisteredClasses() {
+ return this.serializationList.stream().map(Triplet::getValue0).collect(Collectors.toList());
+ }
+
/**
* Gets the header for a Gremlin Kryo file, which is based on the version of Gremlin Kryo that is constructed
* via the builder classes.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b708e45/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 0c979a7..5eb353d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -43,6 +43,7 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.GraphComputerHelper;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +51,7 @@ import scala.Tuple2;
import java.io.File;
import java.util.HashSet;
+import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -150,6 +152,9 @@ public final class SparkGraphComputer implements GraphComputer {
// wire up a spark context
final SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX + (null == this.vertexProgram ? "No VertexProgram" : this.vertexProgram) + "[" + this.mapReducers + "]");
+ final List<Class> classes = GryoMapper.build().create().getRegisteredClasses();
+ sparkConfiguration.registerKryoClasses(classes.toArray(new Class[classes.size()]));
+
hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class)))
hadoopConfiguration.set(Constants.MAPRED_INPUT_DIR, SparkExecutor.getInputLocation(hadoopConfiguration)); // necessary for Spark and newAPIHadoopRDD