You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/05 00:26:44 UTC
incubator-tinkerpop git commit: introduced the concept of a Payload
where there are VertexPayloads and MessagesPayloads. This greatly reduces the
binary footprint of a message in SparkGraphComputer.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master e32fb749e -> 1126869e4
introduced the concept of a Payload where there are VertexPayloads and MessagesPayloads. This greatly reduces the binary footprint of a message in SparkGraphComputer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1126869e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1126869e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1126869e
Branch: refs/heads/master
Commit: 1126869e4ba8476536b4972299da09ed4cea38cd
Parents: e32fb74
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 4 16:26:42 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 4 16:26:42 2015 -0700
----------------------------------------------------------------------
.../computer/spark/SparkGraphComputer.java | 6 +-
.../computer/spark/SparkMessagePayload.java | 49 +++++++
.../process/computer/spark/SparkMessenger.java | 132 -------------------
.../process/computer/spark/SparkPayload.java | 51 +++++++
.../computer/spark/SparkVertexPayload.java | 108 +++++++++++++++
.../computer/spark/util/SparkHelper.java | 46 +++----
6 files changed, 235 insertions(+), 157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 2d13a2e..4ee61cb 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -136,15 +136,15 @@ public final class SparkGraphComputer implements GraphComputer {
// add the project jars to the cluster
SparkGraphComputer.loadJars(sparkContext, hadoopConfiguration);
// create a message-passing friendly rdd from the hadoop input format
- JavaPairRDD<Object, SparkMessenger<Object>> graphRDD = sparkContext.newAPIHadoopRDD(hadoopConfiguration,
+ JavaPairRDD<Object, SparkPayload<Object>> graphRDD = sparkContext.newAPIHadoopRDD(hadoopConfiguration,
(Class<InputFormat<NullWritable, VertexWritable>>) hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class),
NullWritable.class,
VertexWritable.class)
- .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), SparkMessenger.forGraphVertex(new SparkVertex((TinkerVertex) tuple._2().get()))));
+ .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), new SparkVertexPayload<>(new SparkVertex((TinkerVertex) tuple._2().get()))));
// set up the vertex program and wire up configurations
memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
- this.vertexProgram.setup(memory); // TODO: setup variables are not being broadcasted on first call
+ this.vertexProgram.setup(memory); // TODO: setup variables are not being broadcasted on first call (force using broadcast or dummy reduce that has a memory reference?)
final SApacheConfiguration vertexProgramConfiguration = new SApacheConfiguration();
this.vertexProgram.storeState(vertexProgramConfiguration);
ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java
new file mode 100644
index 0000000..43e34ab
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkMessagePayload<M> implements Serializable, SparkPayload<M> {
+
+ public final List<M> messages = new ArrayList<>();
+
+ private SparkMessagePayload() {
+
+ }
+
+ public SparkMessagePayload(final M message) {
+ this.messages.add(message);
+ }
+
+ @Override
+ public List<M> getMessages() {
+ return this.messages;
+ }
+
+ @Override
+ public final boolean isVertex() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
deleted file mode 100644
index ea6187a..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.process.Traversal;
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.graph.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.graph.traversal.step.sideEffect.StartStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
-import scala.Tuple2;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Stream;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class SparkMessenger<M> implements Serializable, Messenger<M> {
-
- private Vertex vertex;
- private List<M> incoming;
- private List<Tuple2<Object, M>> outgoing;
-
- private SparkMessenger() {
-
- }
-
- public static final <M> SparkMessenger<M> forGraphVertex(final Vertex graphVertex) {
- final SparkMessenger<M> messenger = new SparkMessenger<>();
- messenger.vertex = graphVertex;
- messenger.incoming = new ArrayList<>();
- messenger.outgoing = new ArrayList<>();
- return messenger;
- }
-
- public static final <M> SparkMessenger<M> forMessageVertex(final Object vertexId, final M message) {
- final SparkMessenger<M> messenger = new SparkMessenger<>();
- messenger.vertex = new DetachedVertex(vertexId, Vertex.DEFAULT_LABEL, Collections.emptyMap());
- messenger.incoming = new ArrayList<>();
- messenger.incoming.add(message);
- return messenger;
- }
-
- public void clearIncomingMessages() {
- this.incoming.clear();
- }
-
- public void clearOutgoingMessages() {
- if (null != this.outgoing)
- this.outgoing.clear();
- }
-
- public Vertex getVertex() {
- return this.vertex;
- }
-
- public void mergeInMessenger(final SparkMessenger<M> otherMessenger) {
- this.vertex = otherMessenger.vertex;
- this.outgoing = otherMessenger.outgoing;
- }
-
- public void addIncomingMessages(final SparkMessenger<M> otherMessenger, final Optional<MessageCombiner<M>> messageCombinerOptional) {
- if (messageCombinerOptional.isPresent()) {
- final MessageCombiner<M> messageCombiner = messageCombinerOptional.get();
- final M combinedMessage = Stream.concat(this.incoming.stream(), otherMessenger.incoming.stream()).reduce(messageCombiner::combine).get();
- this.incoming.clear();
- this.incoming.add(combinedMessage);
- } else {
- this.incoming.addAll(otherMessenger.incoming);
- }
- }
-
- public List<Tuple2<Object, M>> getOutgoingMessages() {
- return this.outgoing;
- }
-
- @Override
- public Iterable<M> receiveMessages(final MessageScope messageScope) {
- return this.incoming;
- }
-
- @Override
- public void sendMessage(final MessageScope messageScope, final M message) {
- if (messageScope instanceof MessageScope.Local) {
- final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
- final Traversal.Admin<Vertex, Edge> incidentTraversal = SparkMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.vertex);
- final Direction direction = SparkMessenger.getOppositeDirection(incidentTraversal);
- incidentTraversal.forEachRemaining(edge -> this.outgoing.add(new Tuple2<>(edge.iterators().vertexIterator(direction).next().id(), message)));
- } else {
- ((MessageScope.Global) messageScope).vertices().forEach(v -> this.outgoing.add(new Tuple2<>(v.id(), message)));
- }
- }
-
- ///////////
-
- private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
- incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
- return (T) incidentTraversal;
- }
-
- private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
- final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
- return step.getDirection().opposite();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java
new file mode 100644
index 0000000..69980bb
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface SparkPayload<M> {
+
+ public default void addMessages(final List<M> otherMessages, final Optional<MessageCombiner<M>> messageCombinerOptional) {
+ if (messageCombinerOptional.isPresent()) {
+ final MessageCombiner<M> messageCombiner = messageCombinerOptional.get();
+ final M combinedMessage = Stream.concat(this.getMessages().stream(), otherMessages.stream()).reduce(messageCombiner::combine).get();
+ this.getMessages().clear();
+ this.getMessages().add(combinedMessage);
+ } else {
+ this.getMessages().addAll(otherMessages);
+ }
+ }
+
+ public List<M> getMessages();
+
+ public boolean isVertex();
+
+ public default SparkVertexPayload<M> asVertexPayload() {
+ return (SparkVertexPayload<M>) this;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java
new file mode 100644
index 0000000..9cfdebe
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import org.apache.tinkerpop.gremlin.process.Traversal;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.graph.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.graph.traversal.step.sideEffect.StartStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import scala.Tuple2;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkVertexPayload<M> implements SparkPayload<M>, Messenger<M>, Serializable {
+
+ private Vertex vertex;
+ private List<M> incoming;
+ private List<Tuple2<Object, M>> outgoing;
+
+ private SparkVertexPayload() {
+
+ }
+
+ public SparkVertexPayload(final Vertex vertex) {
+ this.vertex = vertex;
+ this.incoming = new ArrayList<>();
+ this.outgoing = new ArrayList<>();
+ }
+
+ @Override
+ public final boolean isVertex() {
+ return true;
+ }
+
+ @Override
+ public SparkVertexPayload<M> asVertexPayload() {
+ return this;
+ }
+
+ @Override
+ public List<M> getMessages() {
+ return this.incoming;
+ }
+
+ public Vertex getVertex() {
+ return this.vertex;
+ }
+
+ public List<Tuple2<Object, M>> getOutgoingMessages() {
+ return this.outgoing;
+ }
+
+ ///////////
+
+ @Override
+ public Iterable<M> receiveMessages(final MessageScope messageScope) {
+ return this.incoming;
+ }
+
+ @Override
+ public void sendMessage(final MessageScope messageScope, final M message) {
+ if (messageScope instanceof MessageScope.Local) {
+ final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
+ final Traversal.Admin<Vertex, Edge> incidentTraversal = SparkVertexPayload.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.vertex);
+ final Direction direction = SparkVertexPayload.getOppositeDirection(incidentTraversal);
+ incidentTraversal.forEachRemaining(edge -> this.outgoing.add(new Tuple2<>(edge.iterators().vertexIterator(direction).next().id(), message)));
+ } else {
+ ((MessageScope.Global) messageScope).vertices().forEach(v -> this.outgoing.add(new Tuple2<>(v.id(), message)));
+ }
+ }
+
+ ///////////
+
+ private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
+ incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
+ return (T) incidentTraversal;
+ }
+
+ private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
+ final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
+ return step.getDirection().opposite();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
index d7094e2..095207a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
@@ -29,7 +29,8 @@ import org.apache.spark.api.java.function.Function2;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMapEmitter;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMemory;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMessenger;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMessagePayload;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkPayload;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkReduceEmitter;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
@@ -39,7 +40,6 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import scala.Tuple2;
@@ -54,49 +54,51 @@ public final class SparkHelper {
private SparkHelper() {
}
- public static <M> JavaPairRDD<Object, SparkMessenger<M>> executeStep(final JavaPairRDD<Object, SparkMessenger<M>> graphRDD, final SparkMemory memory, final Configuration apacheConfiguration) {
- JavaPairRDD<Object, SparkMessenger<M>> current = graphRDD;
+ public static <M> JavaPairRDD<Object, SparkPayload<M>> executeStep(final JavaPairRDD<Object, SparkPayload<M>> graphRDD, final SparkMemory memory, final Configuration apacheConfiguration) {
+ JavaPairRDD<Object, SparkPayload<M>> current = graphRDD;
// execute vertex program
current = current.mapPartitionsToPair(partitionIterator -> { // each partition(Spark)/worker(TP3) has a local copy of the vertex program to reduce object creation
final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(apacheConfiguration);
workerVertexProgram.workerIterationStart(memory);
- return () -> IteratorUtils.<Tuple2<Object, SparkMessenger<M>>, Tuple2<Object, SparkMessenger<M>>>map(partitionIterator, keyValue -> {
- workerVertexProgram.execute(keyValue._2().getVertex(), keyValue._2(), memory);
+ return () -> IteratorUtils.<Tuple2<Object, SparkPayload<M>>, Tuple2<Object, SparkPayload<M>>>map(partitionIterator, keyValue -> {
+ workerVertexProgram.execute(keyValue._2().asVertexPayload().getVertex(), keyValue._2().asVertexPayload(), memory);
if (!partitionIterator.hasNext()) workerVertexProgram.workerIterationEnd(memory); // is this safe?
return keyValue;
});
});
// emit messages by appending them to the graph vertices as message "vertices"
- current = current.<Object, SparkMessenger<M>>flatMapToPair(keyValue -> () -> {
- keyValue._2().clearIncomingMessages(); // the graph vertex should not have any incoming messages (should be cleared from the previous stage)
- return IteratorUtils.<Tuple2<Object, SparkMessenger<M>>>concat(
+ current = current.<Object, SparkPayload<M>>flatMapToPair(keyValue -> () -> {
+ keyValue._2().asVertexPayload().getMessages().clear(); // the graph vertex should not have any incoming messages (should be cleared from the previous stage)
+ return IteratorUtils.<Tuple2<Object, SparkPayload<M>>>concat(
IteratorUtils.of(keyValue),
- IteratorUtils.map(keyValue._2().getOutgoingMessages().iterator(), // this is the graph vertex
- entry -> new Tuple2<>(entry._1(), SparkMessenger.forMessageVertex(entry._2(), entry._2())))); // this is a message "vertex";
+ IteratorUtils.map(keyValue._2().asVertexPayload().getOutgoingMessages().iterator(), // this is a vertex
+ entry -> new Tuple2<>(entry._1(), new SparkMessagePayload<>(entry._2())))); // this is a message;
});
// "message pass" via reduction joining the "message vertices" with the graph vertices
- // addIncomingMessages is provided the vertex program message combiner for partition and global level combining
- current = current.reduceByKey(new Function2<SparkMessenger<M>, SparkMessenger<M>, SparkMessenger<M>>() {
+ // addMessages is provided the vertex program message combiner for partition and global level combining
+ current = current.reduceByKey(new Function2<SparkPayload<M>, SparkPayload<M>, SparkPayload<M>>() {
private Optional<MessageCombiner<M>> messageCombinerOptional = null; // a hack to simulate partition(Spark)/worker(TP3) local variables
@Override
- public SparkMessenger<M> call(final SparkMessenger<M> messengerA, final SparkMessenger<M> messengerB) throws Exception {
+ public SparkPayload<M> call(final SparkPayload<M> payloadA, final SparkPayload<M> payloadB) throws Exception {
if (null == this.messageCombinerOptional)
this.messageCombinerOptional = VertexProgram.<VertexProgram<M>>createVertexProgram(apacheConfiguration).getMessageCombiner();
- if (messengerA.getVertex() instanceof DetachedVertex && !(messengerB.getVertex() instanceof DetachedVertex)) // fold the message vertices into the graph vertices
- messengerA.mergeInMessenger(messengerB);
- messengerA.addIncomingMessages(messengerB, this.messageCombinerOptional);
-
- return messengerA; // always reduce on the first argument
+ if (payloadA.isVertex()) {
+ payloadA.addMessages(payloadB.getMessages(), this.messageCombinerOptional);
+ return payloadA;
+ } else {
+ payloadB.addMessages(payloadA.getMessages(), this.messageCombinerOptional);
+ return payloadB;
+ }
}
});
// clear all previous outgoing messages (why can't we do this prior to the shuffle?)
current = current.mapValues(messenger -> {
- messenger.clearOutgoingMessages();
+ messenger.asVertexPayload().getOutgoingMessages().clear();
return messenger;
});
@@ -140,11 +142,11 @@ public final class SparkHelper {
}
}
- public static <M> void saveVertexProgramRDD(final JavaPairRDD<Object, SparkMessenger<M>> graphRDD, final org.apache.hadoop.conf.Configuration hadoopConfiguration) {
+ public static <M> void saveVertexProgramRDD(final JavaPairRDD<Object, SparkPayload<M>> graphRDD, final org.apache.hadoop.conf.Configuration hadoopConfiguration) {
final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
if (null != outputLocation) {
// map back to a <nullwritable,vertexwritable> stream for output
- graphRDD.mapToPair(tuple -> new Tuple2<>(NullWritable.get(), new VertexWritable<>(tuple._2().getVertex())))
+ graphRDD.mapToPair(tuple -> new Tuple2<>(NullWritable.get(), new VertexWritable<>(tuple._2().asVertexPayload().getVertex())))
.saveAsNewAPIHadoopFile(outputLocation + "/" + Constants.SYSTEM_G,
NullWritable.class,
VertexWritable.class,