You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/05/22 00:46:56 UTC
incubator-tinkerpop git commit: GiraphGraphComputer now tests for
serializability of the traversal prior to job distribution -- much cleaner
than the previous model.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 452bc12f7 -> b57ef286f
GiraphGraphComputer now tests for serializability of the traversal prior to job distribution -- much cleaner than the previous model.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b57ef286
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b57ef286
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b57ef286
Branch: refs/heads/master
Commit: b57ef286f7e7c9fdeb03b8ebf8f4cfe2ec87e74e
Parents: 452bc12
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu May 21 16:47:07 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu May 21 16:47:07 2015 -0600
----------------------------------------------------------------------
.../hadoop/process/computer/giraph/GiraphGraphComputer.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b57ef286/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
index e2f8e4d..e51b741 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
@@ -119,6 +119,13 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
try {
// it is possible to run graph computer without a vertex program (and thus, only map reduce jobs if they exist)
if (null != this.vertexProgram) {
+ // a way to verify in Giraph whether the traversal will go over the wire or not
+ try {
+ VertexProgram.createVertexProgram(this.hadoopGraph, ConfUtil.makeApacheConfiguration(this.giraphConfiguration));
+ } catch (IllegalStateException e) {
+ if (e.getCause() instanceof NumberFormatException)
+ throw new NotSerializableException("The provided traversal is not serializable and thus, can not be distributed across the cluster");
+ }
// calculate main vertex program memory if desired (costs one mapreduce job)
if (this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_DERIVE_MEMORY, false)) {
final Set<String> memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
@@ -146,7 +153,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
this.logger.info(Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
// execute the job and wait until it completes (if it fails, throw an exception)
if (!job.run(true))
- throw new NotSerializableException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs"); // how do I get the exception that occured?
+ throw new IllegalStateException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs"); // how do I get the exception that occured?
}
// do map reduce jobs
this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT_HAS_EDGES, this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, true));