You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/04/02 19:29:42 UTC
[2/3] incubator-tinkerpop git commit: excluded the Spark dependency
on the LGPL FindBugs library.
excluded the Spark dependency on the LGPL FindBugs library.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6c18af82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6c18af82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6c18af82
Branch: refs/heads/master
Commit: 6c18af829675eae11eb273b3b73898bb3efc4bd9
Parents: 6a8fa28
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 2 09:20:40 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 2 11:29:37 2015 -0600
----------------------------------------------------------------------
hadoop-gremlin/pom.xml | 5 +++++
.../gremlin/hadoop/process/computer/spark/SparkExecutor.java | 6 +++---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6c18af82/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index a7e039c..03f77ba 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -168,6 +168,11 @@ limitations under the License.
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
+ <!-- lgpl conflics -->
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>findbugs</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- consistent dependencies -->
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6c18af82/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
index 65a75c1..6afaa96 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
@@ -79,7 +79,7 @@ public final class SparkExecutor {
workerVertexProgram.workerIterationStart(memory); // start the worker
return () -> IteratorUtils.map(partitionIterator, vertexViewAndMessages -> {
final Vertex vertex = vertexViewAndMessages._2()._1().get();
- final boolean hasViewAndMessages = vertexViewAndMessages._2()._2().isPresent();
+ final boolean hasViewAndMessages = vertexViewAndMessages._2()._2().isPresent(); // if this is the first iteration, then there are no views or messages
final List<DetachedVertexProperty<Object>> previousView = hasViewAndMessages ? vertexViewAndMessages._2()._2().get()._1() : Collections.emptyList();
final List<M> incomingMessages = hasViewAndMessages ? vertexViewAndMessages._2()._2().get()._2() : Collections.emptyList();
previousView.forEach(property -> DetachedVertexProperty.addTo(vertex, property)); // attach the view to the vertex
@@ -120,8 +120,8 @@ public final class SparkExecutor {
// isolate the views and then join the incoming messages
final JavaPairRDD<Object, Tuple2<List<DetachedVertexProperty<Object>>, List<M>>> viewAndIncomingMessagesRDD = viewAndOutgoingMessagesRDD
.mapValues(Tuple2::_1)
- .fullOuterJoin(incomingMessagesRDD)
- .mapValues(tuple -> new Tuple2<>(tuple._1().or(Collections.emptyList()), tuple._2().or(Collections.emptyList())));
+ .leftOuterJoin(incomingMessagesRDD) // there will always be views (even if empty), but there will not always be incoming messages
+ .mapValues(tuple -> new Tuple2<>(tuple._1(), tuple._2().or(Collections.emptyList())));
viewAndIncomingMessagesRDD.foreachPartition(partitionIterator -> {
}); // need to complete a task so its BSP and the memory for this iteration is updated