You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/12/09 14:51:34 UTC
[13/14] incubator-tinkerpop git commit: worker regulation based on
coalesce.
worker regulation based on coalesce.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/357bc442
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/357bc442
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/357bc442
Branch: refs/heads/master
Commit: 357bc4423166f7bcdba74ff306dcaaa6ca2b2feb
Parents: aef9528
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 9 06:40:09 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Dec 9 06:40:09 2015 -0700
----------------------------------------------------------------------
.../gremlin/spark/process/computer/SparkGraphComputer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/357bc442/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 9219211..56364eb 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -67,6 +67,7 @@ import java.util.stream.Stream;
public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
private final org.apache.commons.configuration.Configuration sparkConfiguration;
+ private boolean workersSet = false;
public SparkGraphComputer(final HadoopGraph hadoopGraph) {
super(hadoopGraph);
@@ -80,6 +81,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
if (this.sparkConfiguration.getString(SparkLauncher.SPARK_MASTER).startsWith("local")) {
this.sparkConfiguration.setProperty(SparkLauncher.SPARK_MASTER, "local[" + this.workers + "]");
}
+ this.workersSet = true;
return this;
}
@@ -153,7 +155,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
graphRDD = hadoopConfiguration.getClass(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, InputFormatRDD.class, InputRDD.class)
.newInstance()
.readGraphRDD(apacheConfiguration, sparkContext);
- if (graphRDD.partitions().size() > this.workers) // ensures that the graphRDD does not have more partitions than workers
+ if (this.workersSet && graphRDD.partitions().size() > this.workers) // ensures that the graphRDD does not have more partitions than workers
graphRDD = graphRDD.coalesce(this.workers);
graphRDD = graphRDD.cache();
} catch (final InstantiationException | IllegalAccessException e) {