You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/05 00:26:44 UTC

incubator-tinkerpop git commit: introduced the concept of a Payload where there are VertexPayloads and MessagesPayloads. This greatly reduces the binary footprint of a message in SparkGraphComputer.

Repository: incubator-tinkerpop
Updated Branches:
  refs/heads/master e32fb749e -> 1126869e4


introduced the concept of a Payload where there are VertexPayloads and MessagesPayloads. This greatly reduces the binary footprint of a message in SparkGraphComputer.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1126869e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1126869e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1126869e

Branch: refs/heads/master
Commit: 1126869e4ba8476536b4972299da09ed4cea38cd
Parents: e32fb74
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 4 16:26:42 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 4 16:26:42 2015 -0700

----------------------------------------------------------------------
 .../computer/spark/SparkGraphComputer.java      |   6 +-
 .../computer/spark/SparkMessagePayload.java     |  49 +++++++
 .../process/computer/spark/SparkMessenger.java  | 132 -------------------
 .../process/computer/spark/SparkPayload.java    |  51 +++++++
 .../computer/spark/SparkVertexPayload.java      | 108 +++++++++++++++
 .../computer/spark/util/SparkHelper.java        |  46 +++----
 6 files changed, 235 insertions(+), 157 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 2d13a2e..4ee61cb 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -136,15 +136,15 @@ public final class SparkGraphComputer implements GraphComputer {
                             // add the project jars to the cluster
                             SparkGraphComputer.loadJars(sparkContext, hadoopConfiguration);
                             // create a message-passing friendly rdd from the hadoop input format
-                            JavaPairRDD<Object, SparkMessenger<Object>> graphRDD = sparkContext.newAPIHadoopRDD(hadoopConfiguration,
+                            JavaPairRDD<Object, SparkPayload<Object>> graphRDD = sparkContext.newAPIHadoopRDD(hadoopConfiguration,
                                     (Class<InputFormat<NullWritable, VertexWritable>>) hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class),
                                     NullWritable.class,
                                     VertexWritable.class)
-                                    .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), SparkMessenger.forGraphVertex(new SparkVertex((TinkerVertex) tuple._2().get()))));
+                                    .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), new SparkVertexPayload<>(new SparkVertex((TinkerVertex) tuple._2().get()))));
 
                             // set up the vertex program and wire up configurations
                             memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
-                            this.vertexProgram.setup(memory); // TODO: setup variables are not being broadcasted on first call
+                            this.vertexProgram.setup(memory); // TODO: setup variables are not being broadcasted on first call (force using broadcast or dummy reduce that has a memory reference?)
                             final SApacheConfiguration vertexProgramConfiguration = new SApacheConfiguration();
                             this.vertexProgram.storeState(vertexProgramConfiguration);
                             ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java
new file mode 100644
index 0000000..43e34ab
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessagePayload.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkMessagePayload<M> implements Serializable, SparkPayload<M> {
+
+    public final List<M> messages = new ArrayList<>();
+
+    private SparkMessagePayload() {
+
+    }
+
+    public SparkMessagePayload(final M message) {
+        this.messages.add(message);
+    }
+
+    @Override
+    public List<M> getMessages() {
+        return this.messages;
+    }
+
+    @Override
+    public final boolean isVertex() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
deleted file mode 100644
index ea6187a..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.process.Traversal;
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.graph.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.graph.traversal.step.sideEffect.StartStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
-import scala.Tuple2;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Stream;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class SparkMessenger<M> implements Serializable, Messenger<M> {
-
-    private Vertex vertex;
-    private List<M> incoming;
-    private List<Tuple2<Object, M>> outgoing;
-
-    private SparkMessenger() {
-
-    }
-
-    public static final <M> SparkMessenger<M> forGraphVertex(final Vertex graphVertex) {
-        final SparkMessenger<M> messenger = new SparkMessenger<>();
-        messenger.vertex = graphVertex;
-        messenger.incoming = new ArrayList<>();
-        messenger.outgoing = new ArrayList<>();
-        return messenger;
-    }
-
-    public static final <M> SparkMessenger<M> forMessageVertex(final Object vertexId, final M message) {
-        final SparkMessenger<M> messenger = new SparkMessenger<>();
-        messenger.vertex = new DetachedVertex(vertexId, Vertex.DEFAULT_LABEL, Collections.emptyMap());
-        messenger.incoming = new ArrayList<>();
-        messenger.incoming.add(message);
-        return messenger;
-    }
-
-    public void clearIncomingMessages() {
-        this.incoming.clear();
-    }
-
-    public void clearOutgoingMessages() {
-        if (null != this.outgoing)
-            this.outgoing.clear();
-    }
-
-    public Vertex getVertex() {
-        return this.vertex;
-    }
-
-    public void mergeInMessenger(final SparkMessenger<M> otherMessenger) {
-        this.vertex = otherMessenger.vertex;
-        this.outgoing = otherMessenger.outgoing;
-    }
-
-    public void addIncomingMessages(final SparkMessenger<M> otherMessenger, final Optional<MessageCombiner<M>> messageCombinerOptional) {
-        if (messageCombinerOptional.isPresent()) {
-            final MessageCombiner<M> messageCombiner = messageCombinerOptional.get();
-            final M combinedMessage = Stream.concat(this.incoming.stream(), otherMessenger.incoming.stream()).reduce(messageCombiner::combine).get();
-            this.incoming.clear();
-            this.incoming.add(combinedMessage);
-        } else {
-            this.incoming.addAll(otherMessenger.incoming);
-        }
-    }
-
-    public List<Tuple2<Object, M>> getOutgoingMessages() {
-        return this.outgoing;
-    }
-
-    @Override
-    public Iterable<M> receiveMessages(final MessageScope messageScope) {
-        return this.incoming;
-    }
-
-    @Override
-    public void sendMessage(final MessageScope messageScope, final M message) {
-        if (messageScope instanceof MessageScope.Local) {
-            final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
-            final Traversal.Admin<Vertex, Edge> incidentTraversal = SparkMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.vertex);
-            final Direction direction = SparkMessenger.getOppositeDirection(incidentTraversal);
-            incidentTraversal.forEachRemaining(edge -> this.outgoing.add(new Tuple2<>(edge.iterators().vertexIterator(direction).next().id(), message)));
-        } else {
-            ((MessageScope.Global) messageScope).vertices().forEach(v -> this.outgoing.add(new Tuple2<>(v.id(), message)));
-        }
-    }
-
-    ///////////
-
-    private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
-        incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
-        return (T) incidentTraversal;
-    }
-
-    private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
-        final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
-        return step.getDirection().opposite();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java
new file mode 100644
index 0000000..69980bb
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkPayload.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface SparkPayload<M> {
+
+    public default void addMessages(final List<M> otherMessages, final Optional<MessageCombiner<M>> messageCombinerOptional) {
+        if (messageCombinerOptional.isPresent()) {
+            final MessageCombiner<M> messageCombiner = messageCombinerOptional.get();
+            final M combinedMessage = Stream.concat(this.getMessages().stream(), otherMessages.stream()).reduce(messageCombiner::combine).get();
+            this.getMessages().clear();
+            this.getMessages().add(combinedMessage);
+        } else {
+            this.getMessages().addAll(otherMessages);
+        }
+    }
+
+    public List<M> getMessages();
+
+    public boolean isVertex();
+
+    public default SparkVertexPayload<M> asVertexPayload() {
+        return (SparkVertexPayload<M>) this;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java
new file mode 100644
index 0000000..9cfdebe
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertexPayload.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import org.apache.tinkerpop.gremlin.process.Traversal;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.graph.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.graph.traversal.step.sideEffect.StartStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import scala.Tuple2;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkVertexPayload<M> implements SparkPayload<M>, Messenger<M>, Serializable {
+
+    private Vertex vertex;
+    private List<M> incoming;
+    private List<Tuple2<Object, M>> outgoing;
+
+    private SparkVertexPayload() {
+
+    }
+
+    public SparkVertexPayload(final Vertex vertex) {
+        this.vertex = vertex;
+        this.incoming = new ArrayList<>();
+        this.outgoing = new ArrayList<>();
+    }
+
+    @Override
+    public final boolean isVertex() {
+        return true;
+    }
+
+    @Override
+    public SparkVertexPayload<M> asVertexPayload() {
+        return this;
+    }
+
+    @Override
+    public List<M> getMessages() {
+        return this.incoming;
+    }
+
+    public Vertex getVertex() {
+        return this.vertex;
+    }
+
+    public List<Tuple2<Object, M>> getOutgoingMessages() {
+        return this.outgoing;
+    }
+
+    ///////////
+
+    @Override
+    public Iterable<M> receiveMessages(final MessageScope messageScope) {
+        return this.incoming;
+    }
+
+    @Override
+    public void sendMessage(final MessageScope messageScope, final M message) {
+        if (messageScope instanceof MessageScope.Local) {
+            final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
+            final Traversal.Admin<Vertex, Edge> incidentTraversal = SparkVertexPayload.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.vertex);
+            final Direction direction = SparkVertexPayload.getOppositeDirection(incidentTraversal);
+            incidentTraversal.forEachRemaining(edge -> this.outgoing.add(new Tuple2<>(edge.iterators().vertexIterator(direction).next().id(), message)));
+        } else {
+            ((MessageScope.Global) messageScope).vertices().forEach(v -> this.outgoing.add(new Tuple2<>(v.id(), message)));
+        }
+    }
+
+    ///////////
+
+    private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
+        incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
+        return (T) incidentTraversal;
+    }
+
+    private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
+        final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
+        return step.getDirection().opposite();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1126869e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
index d7094e2..095207a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/util/SparkHelper.java
@@ -29,7 +29,8 @@ import org.apache.spark.api.java.function.Function2;
 import org.apache.tinkerpop.gremlin.hadoop.Constants;
 import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMapEmitter;
 import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMemory;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMessenger;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkMessagePayload;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkPayload;
 import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkReduceEmitter;
 import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
 import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
@@ -39,7 +40,6 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
 import org.apache.tinkerpop.gremlin.process.computer.Memory;
 import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
 import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 import scala.Tuple2;
 
@@ -54,49 +54,51 @@ public final class SparkHelper {
     private SparkHelper() {
     }
 
-    public static <M> JavaPairRDD<Object, SparkMessenger<M>> executeStep(final JavaPairRDD<Object, SparkMessenger<M>> graphRDD, final SparkMemory memory, final Configuration apacheConfiguration) {
-        JavaPairRDD<Object, SparkMessenger<M>> current = graphRDD;
+    public static <M> JavaPairRDD<Object, SparkPayload<M>> executeStep(final JavaPairRDD<Object, SparkPayload<M>> graphRDD, final SparkMemory memory, final Configuration apacheConfiguration) {
+        JavaPairRDD<Object, SparkPayload<M>> current = graphRDD;
         // execute vertex program
         current = current.mapPartitionsToPair(partitionIterator -> {     // each partition(Spark)/worker(TP3) has a local copy of the vertex program to reduce object creation
             final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(apacheConfiguration);
             workerVertexProgram.workerIterationStart(memory);
-            return () -> IteratorUtils.<Tuple2<Object, SparkMessenger<M>>, Tuple2<Object, SparkMessenger<M>>>map(partitionIterator, keyValue -> {
-                workerVertexProgram.execute(keyValue._2().getVertex(), keyValue._2(), memory);
+            return () -> IteratorUtils.<Tuple2<Object, SparkPayload<M>>, Tuple2<Object, SparkPayload<M>>>map(partitionIterator, keyValue -> {
+                workerVertexProgram.execute(keyValue._2().asVertexPayload().getVertex(), keyValue._2().asVertexPayload(), memory);
                 if (!partitionIterator.hasNext()) workerVertexProgram.workerIterationEnd(memory);  // is this safe?
                 return keyValue;
             });
         });
 
         // emit messages by appending them to the graph vertices as message "vertices"
-        current = current.<Object, SparkMessenger<M>>flatMapToPair(keyValue -> () -> {
-            keyValue._2().clearIncomingMessages(); // the graph vertex should not have any incoming messages (should be cleared from the previous stage)
-            return IteratorUtils.<Tuple2<Object, SparkMessenger<M>>>concat(
+        current = current.<Object, SparkPayload<M>>flatMapToPair(keyValue -> () -> {
+            keyValue._2().asVertexPayload().getMessages().clear(); // the graph vertex should not have any incoming messages (should be cleared from the previous stage)
+            return IteratorUtils.<Tuple2<Object, SparkPayload<M>>>concat(
                     IteratorUtils.of(keyValue),
-                    IteratorUtils.map(keyValue._2().getOutgoingMessages().iterator(),                                      // this is the graph vertex
-                            entry -> new Tuple2<>(entry._1(), SparkMessenger.forMessageVertex(entry._2(), entry._2()))));  // this is a message "vertex";
+                    IteratorUtils.map(keyValue._2().asVertexPayload().getOutgoingMessages().iterator(),            // this is a vertex
+                            entry -> new Tuple2<>(entry._1(), new SparkMessagePayload<>(entry._2()))));            // this is a message;
         });
 
         // "message pass" via reduction joining the "message vertices" with the graph vertices
-        // addIncomingMessages is provided the vertex program message combiner for partition and global level combining
-        current = current.reduceByKey(new Function2<SparkMessenger<M>, SparkMessenger<M>, SparkMessenger<M>>() {
+        // addMessages is provided the vertex program message combiner for partition and global level combining
+        current = current.reduceByKey(new Function2<SparkPayload<M>, SparkPayload<M>, SparkPayload<M>>() {
             private Optional<MessageCombiner<M>> messageCombinerOptional = null; // a hack to simulate partition(Spark)/worker(TP3) local variables
 
             @Override
-            public SparkMessenger<M> call(final SparkMessenger<M> messengerA, final SparkMessenger<M> messengerB) throws Exception {
+            public SparkPayload<M> call(final SparkPayload<M> payloadA, final SparkPayload<M> payloadB) throws Exception {
                 if (null == this.messageCombinerOptional)
                     this.messageCombinerOptional = VertexProgram.<VertexProgram<M>>createVertexProgram(apacheConfiguration).getMessageCombiner();
 
-                if (messengerA.getVertex() instanceof DetachedVertex && !(messengerB.getVertex() instanceof DetachedVertex)) // fold the message vertices into the graph vertices
-                    messengerA.mergeInMessenger(messengerB);
-                messengerA.addIncomingMessages(messengerB, this.messageCombinerOptional);
-
-                return messengerA;  // always reduce on the first argument
+                if (payloadA.isVertex()) {
+                    payloadA.addMessages(payloadB.getMessages(), this.messageCombinerOptional);
+                    return payloadA;
+                } else {
+                    payloadB.addMessages(payloadA.getMessages(), this.messageCombinerOptional);
+                    return payloadB;
+                }
             }
         });
 
         // clear all previous outgoing messages (why can't we do this prior to the shuffle?)
         current = current.mapValues(messenger -> {
-            messenger.clearOutgoingMessages();
+            messenger.asVertexPayload().getOutgoingMessages().clear();
             return messenger;
         });
 
@@ -140,11 +142,11 @@ public final class SparkHelper {
         }
     }
 
-    public static <M> void saveVertexProgramRDD(final JavaPairRDD<Object, SparkMessenger<M>> graphRDD, final org.apache.hadoop.conf.Configuration hadoopConfiguration) {
+    public static <M> void saveVertexProgramRDD(final JavaPairRDD<Object, SparkPayload<M>> graphRDD, final org.apache.hadoop.conf.Configuration hadoopConfiguration) {
         final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
         if (null != outputLocation) {
             // map back to a <nullwritable,vertexwritable> stream for output
-            graphRDD.mapToPair(tuple -> new Tuple2<>(NullWritable.get(), new VertexWritable<>(tuple._2().getVertex())))
+            graphRDD.mapToPair(tuple -> new Tuple2<>(NullWritable.get(), new VertexWritable<>(tuple._2().asVertexPayload().getVertex())))
                     .saveAsNewAPIHadoopFile(outputLocation + "/" + Constants.SYSTEM_G,
                             NullWritable.class,
                             VertexWritable.class,