You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/12/08 14:47:19 UTC
[08/13] incubator-tinkerpop git commit: minor tweaks. Payload null
check (can happen with persistent RDDs...) and clear out Spark cache for
stale unpersited RDDs.
minor tweaks. Payload null check (can happen with persistent RDDs...) and clear out Spark cache for stale unpersited RDDs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/8d7b8533
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/8d7b8533
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/8d7b8533
Branch: refs/heads/master
Commit: 8d7b8533cc3a236b6423561d52beb3d8cb2f71a8
Parents: 74d1fde
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Dec 7 16:50:19 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Dec 7 16:50:19 2015 -0700
----------------------------------------------------------------------
.../spark/process/computer/payload/ViewIncomingPayload.java | 2 ++
.../org/apache/tinkerpop/gremlin/spark/structure/Spark.java | 7 +++++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8d7b8533/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
index a2c9205..2dc9796 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
@@ -83,6 +83,8 @@ public final class ViewIncomingPayload<M> implements Payload {
}
public void mergePayload(final Payload payload, final MessageCombiner<M> messageCombiner) {
+ if (null == payload)
+ return;
if (payload instanceof ViewPayload)
this.view = ((ViewPayload) payload).getView();
else if (payload instanceof MessagePayload)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8d7b8533/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java
index a889c6d..62b0ab3 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java
@@ -27,9 +27,12 @@ import org.apache.tinkerpop.gremlin.hadoop.Constants;
import scala.collection.JavaConversions;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -58,11 +61,15 @@ public class Spark {
}
public static void refresh() {
+ final Set<String> keepNames = new HashSet<>();
for (final RDD<?> rdd : JavaConversions.asJavaIterable(CONTEXT.persistentRdds().values())) {
if (null != rdd.name()) {
+ keepNames.add(rdd.name());
NAME_TO_RDD.put(rdd.name(), rdd);
}
}
+ // remove all stale names in the NAME_TO_RDD map
+ NAME_TO_RDD.keySet().stream().filter(key -> !keepNames.contains(key)).collect(Collectors.toList()).forEach(NAME_TO_RDD::remove);
}
public static RDD<?> getRDD(final String name) {