You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/05/06 16:08:55 UTC
[01/18] incubator-tinkerpop git commit: Created the first
GraphComputer Provider TraversalStrategy -- SparkPartitionAwareStratgegy.
This strategy analyzes the traversal to determine if partitioning should be
skipped -- e.g. if no message pass is happenin
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 55172b447 -> e560970f4
Created the first GraphComputer Provider TraversalStrategy -- SparkPartitionAwareStratgegy. This strategy analyzes the traversal to determine if partitioning should be skipped -- e.g. if no message pass is happening. I think this strategy will start to set the stage for how future GraphComputer provider-specific strategies will work.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/77480a2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/77480a2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/77480a2c
Branch: refs/heads/master
Commit: 77480a2c7875e284b063ecca434bf2165116689d
Parents: 76d5be2
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 3 14:14:29 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 3 14:14:29 2016 -0600
----------------------------------------------------------------------
.../gremlin/process/computer/Computer.java | 75 +++++++++---
.../traversal/TraversalVertexProgram.java | 11 +-
.../process/traversal/TraversalSource.java | 4 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 1 +
.../optimization/Neo4jGraphStepStrategy.java | 2 +-
.../spark/process/computer/SparkExecutor.java | 68 ++++++-----
.../process/computer/SparkGraphComputer.java | 22 +++-
.../SparkPartitionAwareStrategy.java | 86 ++++++++++++++
.../SparkPartitionAwareStrategyTest.java | 115 +++++++++++++++++++
9 files changed, 331 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/Computer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/Computer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/Computer.java
index 4f416dc..ce31d5d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/Computer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/Computer.java
@@ -32,7 +32,7 @@ import java.util.function.Function;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class Computer implements Function<Graph, GraphComputer>, Serializable {
+public final class Computer implements Function<Graph, GraphComputer>, Serializable, Cloneable {
private Class<? extends GraphComputer> graphComputerClass = GraphComputer.class;
private Map<String, Object> configuration = new HashMap<>();
@@ -55,41 +55,42 @@ public final class Computer implements Function<Graph, GraphComputer>, Serializa
}
public Computer configure(final String key, final Object value) {
- this.configuration.put(key, value);
- return this;
+ final Computer clone = this.clone();
+ clone.configuration.put(key, value);
+ return clone;
}
public Computer workers(final int workers) {
- this.workers = workers;
- return this;
+ final Computer clone = this.clone();
+ clone.workers = workers;
+ return clone;
}
public Computer persist(final GraphComputer.Persist persist) {
- this.persist = persist;
- return this;
+ final Computer clone = this.clone();
+ clone.persist = persist;
+ return clone;
}
public Computer result(final GraphComputer.ResultGraph resultGraph) {
- this.resultGraph = resultGraph;
- return this;
+ final Computer clone = this.clone();
+ clone.resultGraph = resultGraph;
+ return clone;
}
public Computer vertices(final Traversal<Vertex, Vertex> vertexFilter) {
- this.vertices = vertexFilter;
- return this;
+ final Computer clone = this.clone();
+ clone.vertices = vertexFilter;
+ return clone;
}
public Computer edges(final Traversal<Vertex, Edge> edgeFilter) {
- this.edges = edgeFilter;
- return this;
+ final Computer clone = this.clone();
+ clone.edges = edgeFilter;
+ return clone;
}
- public Class<? extends GraphComputer> getGraphComputerClass() {
- return this.graphComputerClass;
- }
-
- @Override
public GraphComputer apply(final Graph graph) {
GraphComputer computer = this.graphComputerClass.equals(GraphComputer.class) ? graph.compute() : graph.compute(this.graphComputerClass);
for (final Map.Entry<String, Object> entry : this.configuration.entrySet()) {
@@ -112,4 +113,42 @@ public final class Computer implements Function<Graph, GraphComputer>, Serializa
public String toString() {
return this.graphComputerClass.getSimpleName().toLowerCase();
}
+
+ @Override
+ public Computer clone() {
+ try {
+ final Computer clone = (Computer) super.clone();
+ clone.configuration = new HashMap<>(this.configuration);
+ if (null != this.vertices)
+ clone.vertices = this.vertices.asAdmin().clone();
+ if (null != this.edges)
+ clone.edges = this.edges.asAdmin().clone();
+ return clone;
+ } catch (final CloneNotSupportedException e) {
+ throw new IllegalStateException(e.getMessage());
+ }
+ }
+
+ /////////////////
+ /////////////////
+
+ public Class<? extends GraphComputer> getGraphComputerClass() {
+ return this.graphComputerClass;
+ }
+
+ public Map<String,Object> getConfiguration() {
+ return this.configuration;
+ }
+
+ public Traversal<Vertex,Vertex> getVertices() {
+ return this.vertices;
+ }
+
+ public Traversal<Vertex,Edge> getEdges() {
+ return this.edges;
+ }
+
+ public int getWorkers() {
+ return this.workers;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index 8e4a75e..166625a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -31,6 +31,7 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ComputerResultStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Path;
@@ -59,7 +60,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileSid
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
@@ -133,6 +133,15 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
return VertexProgram.<TraversalVertexProgram>createVertexProgram(graph, configuration).traversal.get();
}
+ /**
+ * Get the {@link Traversal} associated with the current instance of the traversal vertex program.
+ *
+ * @return the traversal of the instantiated program
+ */
+ public Traversal.Admin<?, ?> getTraversal() {
+ return this.traversal.get();
+ }
+
@Override
public void loadState(final Graph graph, final Configuration configuration) {
if (!configuration.containsKey(TRAVERSAL))
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
index 98163c7..3b60940 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
@@ -98,11 +98,11 @@ public interface TraversalSource extends Cloneable {
* @return a new traversal source with updated strategies
*/
public default TraversalSource withComputer(final Computer computer) {
- final List<TraversalStrategy<?>> graphComputerStrategies = TraversalStrategies.GlobalCache.getStrategies(computer.getGraphComputerClass()).toList();
+ final List<TraversalStrategy<?>> graphComputerStrategies = TraversalStrategies.GlobalCache.getStrategies(computer.apply(this.getGraph()).getClass()).toList();
final TraversalStrategy[] traversalStrategies = new TraversalStrategy[graphComputerStrategies.size() + 1];
traversalStrategies[0] = new VertexProgramStrategy(computer);
for (int i = 0; i < graphComputerStrategies.size(); i++) {
- traversalStrategies[i+1] = graphComputerStrategies.get(i);
+ traversalStrategies[i + 1] = graphComputerStrategies.get(i);
}
return this.withStrategies(traversalStrategies);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index 8d5bb21..872b3fb 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -57,6 +57,7 @@ public final class Constants {
public static final String GREMLIN_SPARK_PERSIST_CONTEXT = "gremlin.spark.persistContext";
public static final String GREMLIN_SPARK_GRAPH_STORAGE_LEVEL = "gremlin.spark.graphStorageLevel";
public static final String GREMLIN_SPARK_PERSIST_STORAGE_LEVEL = "gremlin.spark.persistStorageLevel";
+ public static final String GREMLIN_SPARK_SKIP_PARTITIONER = "gremlin.spark.skipPartitioner";
public static final String SPARK_SERIALIZER = "spark.serializer";
public static String getGraphLocation(final String location) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/neo4j-gremlin/src/main/java/org/apache/tinkerpop/gremlin/neo4j/process/traversal/strategy/optimization/Neo4jGraphStepStrategy.java
----------------------------------------------------------------------
diff --git a/neo4j-gremlin/src/main/java/org/apache/tinkerpop/gremlin/neo4j/process/traversal/strategy/optimization/Neo4jGraphStepStrategy.java b/neo4j-gremlin/src/main/java/org/apache/tinkerpop/gremlin/neo4j/process/traversal/strategy/optimization/Neo4jGraphStepStrategy.java
index 1d2e24a..eed7b15 100644
--- a/neo4j-gremlin/src/main/java/org/apache/tinkerpop/gremlin/neo4j/process/traversal/strategy/optimization/Neo4jGraphStepStrategy.java
+++ b/neo4j-gremlin/src/main/java/org/apache/tinkerpop/gremlin/neo4j/process/traversal/strategy/optimization/Neo4jGraphStepStrategy.java
@@ -31,7 +31,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
* @author Pieter Martin
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class Neo4jGraphStepStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> {
+public final class Neo4jGraphStepStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
private static final Neo4jGraphStepStrategy INSTANCE = new Neo4jGraphStepStrategy();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index 3e0f09a..aadb70d 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -21,6 +21,8 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import com.google.common.base.Optional;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
@@ -78,7 +80,8 @@ public final class SparkExecutor {
final SparkMemory memory,
final Configuration apacheConfiguration) {
- if (null != viewIncomingRDD) // the graphRDD and the viewRDD must have the same partitioner
+ final boolean partitionedGraphRDD = graphRDD.partitioner().isPresent();
+ if (partitionedGraphRDD && null != viewIncomingRDD) // the graphRDD and the viewRDD must have the same partitioner
assert graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get());
final JavaPairRDD<Object, ViewOutgoingPayload<M>> viewOutgoingRDD = (((null == viewIncomingRDD) ?
graphRDD.mapValues(vertexWritable -> new Tuple2<>(vertexWritable, Optional.<ViewIncomingPayload<M>>absent())) : // first iteration will not have any views or messages
@@ -119,34 +122,46 @@ public final class SparkExecutor {
});
}, true)); // true means that the partition is preserved
// the graphRDD and the viewRDD must have the same partitioner
- assert graphRDD.partitioner().get().equals(viewOutgoingRDD.partitioner().get());
+ if (partitionedGraphRDD)
+ assert graphRDD.partitioner().get().equals(viewOutgoingRDD.partitioner().get());
// "message pass" by reducing on the vertex object id of the view and message payloads
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().orElse(null);
- final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD = viewOutgoingRDD
- .flatMapToPair(tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
+
+ /////////////////////////////////////////////////////////////
+ /////////////////////////////////////////////////////////////
+ final PairFlatMapFunction<Tuple2<Object, ViewOutgoingPayload<M>>, Object, Payload> messageFunction =
+ tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())), // emit the view payload
- IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))))) // emit the outgoing message payloads one by one
- .reduceByKey(graphRDD.partitioner().get(), (a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
- if (a instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
- return a;
- } else if (b instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
- return b;
- } else {
- final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
- c.mergePayload(a, messageCombiner);
- c.mergePayload(b, messageCombiner);
- return c;
- }
- })
- .filter(payload -> !(payload._2() instanceof MessagePayload)) // this happens if there is a message to a vertex that does not exist
- .filter(payload -> !((payload._2() instanceof ViewIncomingPayload) && !((ViewIncomingPayload<M>) payload._2()).hasView())) // this happens if there are many messages to a vertex that does not exist
- .mapValues(payload -> payload instanceof ViewIncomingPayload ?
- (ViewIncomingPayload<M>) payload : // this happens if there is a vertex with incoming messages
- new ViewIncomingPayload<>((ViewPayload) payload)); // this happens if there is a vertex with no incoming messages
+ IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))));
+ final Function2<Payload, Payload, Payload> reducerFunction = (a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
+ if (a instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
+ return a;
+ } else if (b instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
+ return b;
+ } else {
+ final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
+ c.mergePayload(a, messageCombiner);
+ c.mergePayload(b, messageCombiner);
+ return c;
+ }
+ };
+ /////////////////////////////////////////////////////////////
+ /////////////////////////////////////////////////////////////
+
+ final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD =
+ (partitionedGraphRDD ?
+ viewOutgoingRDD.flatMapToPair(messageFunction).reduceByKey(graphRDD.partitioner().get(), reducerFunction) :
+ viewOutgoingRDD.flatMapToPair(messageFunction).reduceByKey(reducerFunction))
+ .filter(payload -> !(payload._2() instanceof MessagePayload)) // this happens if there is a message to a vertex that does not exist
+ .filter(payload -> !((payload._2() instanceof ViewIncomingPayload) && !((ViewIncomingPayload<M>) payload._2()).hasView())) // this happens if there are many messages to a vertex that does not exist
+ .mapValues(payload -> payload instanceof ViewIncomingPayload ?
+ (ViewIncomingPayload<M>) payload : // this happens if there is a vertex with incoming messages
+ new ViewIncomingPayload<>((ViewPayload) payload)); // this happens if there is a vertex with no incoming messages
// the graphRDD and the viewRDD must have the same partitioner
- assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
+ if (partitionedGraphRDD)
+ assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
newViewIncomingRDD
.foreachPartition(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
@@ -156,7 +171,8 @@ public final class SparkExecutor {
public static <M> JavaPairRDD<Object, VertexWritable> prepareFinalGraphRDD(final JavaPairRDD<Object, VertexWritable> graphRDD, final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD, final Set<VertexComputeKey> vertexComputeKeys) {
// the graphRDD and the viewRDD must have the same partitioner
- assert (graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get()));
+ if (graphRDD.partitioner().isPresent())
+ assert (graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get()));
// attach the final computed view to the cached graph
return graphRDD.leftOuterJoin(viewIncomingRDD)
.mapValues(tuple -> {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 928a880..cd01779 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -51,7 +51,9 @@ import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.SparkPartitionAwareStrategy;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputFormatRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputOutputHelper;
@@ -80,6 +82,10 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
private final org.apache.commons.configuration.Configuration sparkConfiguration;
private boolean workersSet = false;
+ static {
+ TraversalStrategies.GlobalCache.registerStrategies(SparkGraphComputer.class, TraversalStrategies.GlobalCache.getStrategies(GraphComputer.class).clone().addStrategies(SparkPartitionAwareStrategy.instance()));
+ }
+
public SparkGraphComputer(final HadoopGraph hadoopGraph) {
super(hadoopGraph);
this.sparkConfiguration = new HadoopConfiguration();
@@ -124,6 +130,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
final boolean inputFromSpark = PersistedInputRDD.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_READER, Object.class));
final boolean outputToHDFS = FileOutputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, Object.class));
final boolean outputToSpark = PersistedOutputRDD.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, Object.class));
+ final boolean skipPartitioner = apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false);
String inputLocation = null;
if (inputFromSpark)
inputLocation = Constants.getSearchGraphLocation(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION), sparkContextStorage).orElse(null);
@@ -202,12 +209,17 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
if (loadedGraphRDD.partitioner().isPresent())
this.logger.debug("Using the existing partitioner associated with the loaded graphRDD: " + loadedGraphRDD.partitioner().get());
else {
- final Partitioner partitioner = new HashPartitioner(this.workersSet ? this.workers : loadedGraphRDD.partitions().size());
- this.logger.debug("Partitioning the loaded graphRDD: " + partitioner);
- loadedGraphRDD = loadedGraphRDD.partitionBy(partitioner);
- partitioned = true;
+ if (!skipPartitioner) {
+ final Partitioner partitioner = new HashPartitioner(this.workersSet ? this.workers : loadedGraphRDD.partitions().size());
+ this.logger.debug("Partitioning the loaded graphRDD: " + partitioner);
+ loadedGraphRDD = loadedGraphRDD.partitionBy(partitioner);
+ partitioned = true;
+ assert loadedGraphRDD.partitioner().isPresent();
+ } else {
+ assert skipPartitioner == !loadedGraphRDD.partitioner().isPresent(); // no easy way to test this with a test case
+ this.logger.debug("Partitioning has been skipped for the loaded graphRDD via " + Constants.GREMLIN_SPARK_SKIP_PARTITIONER);
+ }
}
- assert loadedGraphRDD.partitioner().isPresent();
// if the loaded graphRDD was already partitioned previous, then this coalesce/repartition will not take place
if (this.workersSet) {
if (loadedGraphRDD.partitions().size() > this.workers) // ensures that the loaded graphRDD does not have more partitions than workers
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
new file mode 100644
index 0000000..02aa1ae
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaFlatMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkPartitionAwareStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
+
+ private static final SparkPartitionAwareStrategy INSTANCE = new SparkPartitionAwareStrategy();
+
+ private static final Set<Class<? extends Step>> MESSAGE_PASS_CLASSES = new HashSet<>(Arrays.asList(
+ EdgeVertexStep.class,
+ LambdaMapStep.class, // maybe?
+ LambdaFlatMapStep.class // maybe?
+ // VertexStep is special as you need to see if the return class is Edge or Vertex (logic below)
+ ));
+
+ private SparkPartitionAwareStrategy() {
+ }
+
+ @Override
+ public void apply(final Traversal.Admin<?, ?> traversal) {
+ final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
+ for (final TraversalVertexProgramStep step : steps) {
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(traversal.getGraph().orElse(EmptyGraph.instance())).getTraversal();
+ boolean messagePasses = MESSAGE_PASS_CLASSES.stream()
+ .flatMap(clazz -> TraversalHelper.<Step<?, ?>>getStepsOfAssignableClassRecursively((Class) clazz, computerTraversal).stream())
+ .filter(s -> TraversalHelper.isGlobalChild(((Step) s).getTraversal().asAdmin()))
+ .findAny()
+ .isPresent();
+ if (!messagePasses) {
+ for (final VertexStep vertexStep : TraversalHelper.getStepsOfAssignableClassRecursively(VertexStep.class, computerTraversal)) {
+ if (TraversalHelper.isGlobalChild(vertexStep.getTraversal()) &&
+ (vertexStep.returnsVertex() || !vertexStep.getDirection().equals(Direction.OUT))) { // in edges require message pass in OLAP
+ messagePasses = true;
+ break;
+ }
+ }
+ }
+ if (!messagePasses) // if no message passing, don't partition (save time)
+ step.setComputer(step.getComputer().configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true));
+ }
+ }
+
+ public static SparkPartitionAwareStrategy instance() {
+ return INSTANCE;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77480a2c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java
new file mode 100644
index 0000000..4324202
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.Computer;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkPartitionAwareStrategyTest extends AbstractSparkTest {
+
+ @Test
+ public void shouldPartitionTheInputRDDAccordingly() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkPartitionAwareStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkPartitionAwareStrategy.class.getSimpleName()));
+ //
+ assertEquals(6l, g.V().count().next().longValue());
+ assertEquals(2l, g.V().out().out().count().next().longValue());
+ }
+
+
+ @Test
+ public void shouldSetConfigurationsCorrectly() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer(Computer.compute().persist(GraphComputer.Persist.EDGES).result(GraphComputer.ResultGraph.NEW));
+
+ assertTrue(skipPartitioner(g.V().limit(10)));
+ assertTrue(skipPartitioner(g.V().values("age").groupCount()));
+ assertTrue(skipPartitioner(g.V().groupCount().by(__.out().count())));
+ assertTrue(skipPartitioner(g.V().outE()));
+ assertTrue(skipPartitioner(g.V().count()));
+ assertTrue(skipPartitioner(g.V().out().count()));
+ assertTrue(skipPartitioner(g.V().local(__.inE()).count()));
+ assertTrue(skipPartitioner(g.V().outE().inV().count()));
+ ////
+ assertFalse(skipPartitioner(g.V().outE().inV()));
+ assertFalse(skipPartitioner(g.V().both()));
+ assertFalse(skipPartitioner(g.V().both().count()));
+ assertFalse(skipPartitioner(g.V().out().id()));
+ assertFalse(skipPartitioner(g.V().out().out().count()));
+ assertFalse(skipPartitioner(g.V().in().count()));
+ assertFalse(skipPartitioner(g.V().inE().count()));
+
+
+ }
+
+ private static boolean skipPartitioner(final Traversal<?, ?> traversal) {
+ traversal.asAdmin().applyStrategies();
+ return (Boolean) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
+ .getComputer()
+ .getConfiguration()
+ .getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false);
+
+ }
+
+}
[15/18] incubator-tinkerpop git commit: forgot to get rid of this
class which was created in this branch -- not needed.
Posted by ok...@apache.org.
forgot to get rid of this class which was created in this branch -- not needed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1e6aa16b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1e6aa16b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1e6aa16b
Branch: refs/heads/master
Commit: 1e6aa16b9dfbd8a07c0fab8bb0ad7ee88a3cd630
Parents: 854b259
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu May 5 10:15:02 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu May 5 10:15:02 2016 -0600
----------------------------------------------------------------------
.../traversal/strategy/TraversalMatcher.java | 47 --------------------
1 file changed, 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1e6aa16b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java
deleted file mode 100644
index cd37031..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.process.computer.traversal.strategy;
-
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
-
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class TraversalMatcher {
-
- private TraversalMatcher() {
- }
-
- public static boolean is_g_V_count(final Traversal.Admin<?, ?> traversal) {
- final List<Step> steps = traversal.getSteps();
- return steps.size() == 2 &&
- steps.get(0).getLabels().isEmpty() &&
- steps.get(1).getLabels().isEmpty() &&
- (steps.get(1) instanceof CountGlobalStep) &&
- traversal.getSteps().get(0) instanceof GraphStep &&
- ((GraphStep) traversal.getSteps().get(0)).returnsVertex() &&
- ((GraphStep) traversal.getSteps().get(0)).getIds().length == 0;
- }
-}
[03/18] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1120'
into TINKERPOP-1288
Posted by ok...@apache.org.
Merge branch 'TINKERPOP-1120' into TINKERPOP-1288
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/40b3931f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/40b3931f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/40b3931f
Branch: refs/heads/master
Commit: 40b3931f843dcd3fe22da4ec91b1b252e41ffe7e
Parents: e8bba35 8fd9502
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 3 14:48:31 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 3 14:48:31 2016 -0600
----------------------------------------------------------------------
.../traversal/TraversalVertexProgram.java | 26 +++++++++++---------
.../computer/traversal/TraverserExecutor.java | 6 +++--
.../process/computer/GraphComputerTest.java | 4 +--
.../spark/process/computer/SparkExecutor.java | 2 +-
.../computer/payload/ViewIncomingPayload.java | 9 ++++++-
.../computer/payload/ViewOutgoingPayload.java | 7 +++---
.../process/computer/payload/ViewPayload.java | 2 +-
7 files changed, 34 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/40b3931f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
[07/18] incubator-tinkerpop git commit: Generalized
SparkVertexCountInterceptor to SparkCountInterceptor and it can recognize and
natively process various g.V.name.count(), g.V.out.count,
etc. type of traversals. Added a few more test cases to AdjacentTo
Posted by ok...@apache.org.
Generalized SparkVertexCountInterceptor to SparkCountInterceptor and it can recognize and natively process various g.V.name.count(), g.V.out.count, etc. type of traversals. Added a few more test cases to AdjacentToIncidentStrategy. Came up with a nice test infrastructure for SparkInterceptorStrategyTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f0acfd75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f0acfd75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f0acfd75
Branch: refs/heads/master
Commit: f0acfd7512330f82d14e0c17d90f46487008b6e1
Parents: d5e7c78
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 08:40:34 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 08:40:34 2016 -0600
----------------------------------------------------------------------
.../AdjacentToIncidentStrategyTest.java | 4 +
.../optimization/SparkInterceptorStrategy.java | 7 +-
.../interceptor/SparkCountInterceptor.java | 113 +++++++++++++++++++
.../SparkVertexCountInterceptor.java | 53 ---------
.../SparkInterceptorStrategyTest.java | 69 +++++++----
5 files changed, 164 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f0acfd75/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/optimization/AdjacentToIncidentStrategyTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/optimization/AdjacentToIncidentStrategyTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/optimization/AdjacentToIncidentStrategyTest.java
index f2668fd..c791cd5 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/optimization/AdjacentToIncidentStrategyTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/optimization/AdjacentToIncidentStrategyTest.java
@@ -60,6 +60,10 @@ public class AdjacentToIncidentStrategyTest {
public static Iterable<Object[]> generateTestParameters() {
return Arrays.asList(new Traversal[][]{
+ {__.outE().count(), __.outE().count()},
+ {__.bothE("knows").count(), __.bothE("knows").count()},
+ {__.properties().count(), __.properties().count()},
+ {__.properties("name").count(), __.properties("name").count()},
{__.out().count(), __.outE().count()},
{__.in().count(), __.inE().count()},
{__.both().count(), __.bothE().count()},
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f0acfd75/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index a22cef7..b3d250a 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -21,12 +21,11 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.TraversalMatcher;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkVertexCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkCountInterceptor;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
@@ -47,8 +46,8 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance());
final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
for (final TraversalVertexProgramStep step : steps) {
- if (TraversalMatcher.is_g_V_count(step.generateProgram(graph).getTraversal()))
- step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkVertexCountInterceptor.class.getCanonicalName()));
+ if (SparkCountInterceptor.isLegal(step.generateProgram(graph).getTraversal()))
+ step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkCountInterceptor.class.getCanonicalName()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f0acfd75/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
new file mode 100644
index 0000000..7ca4b61
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkCountInterceptor implements SparkVertexProgramInterceptor<TraversalVertexProgram> {
+
+ public SparkCountInterceptor() {
+
+ }
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
+ vertexProgram.setup(memory);
+ final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal();
+ JavaPairRDD<Object, VertexWritable> nextRDD = inputRDD;
+ long count = -1;
+ for (final Step step : traversal.getSteps()) {
+ if (step instanceof GraphStep) {
+ final Object[] ids = ((GraphStep) step).getIds();
+ if (ids.length > 0)
+ nextRDD = nextRDD.filter(tuple -> ElementHelper.idExists(tuple._2().get().id(), ids));
+ } else if (step instanceof HasStep) {
+ final List<HasContainer> hasContainers = ((HasStep<?>) step).getHasContainers();
+ if (hasContainers.size() > 0)
+ nextRDD = nextRDD.filter(tuple -> HasContainer.testAll(tuple._2().get(), hasContainers));
+ } else if (step instanceof PropertiesStep) {
+ final String[] keys = ((PropertiesStep) step).getPropertyKeys();
+ count = nextRDD.mapValues(vertex -> IteratorUtils.count(vertex.get().properties(keys))).fold(new Tuple2<>("sum", 0l), (a, b) -> new Tuple2<>("sum", a._2() + b._2()))._2();
+ } else if (step instanceof VertexStep) {
+ final String[] labels = ((VertexStep) step).getEdgeLabels();
+ final Direction direction = ((VertexStep) step).getDirection();
+ count = nextRDD.mapValues(vertex -> IteratorUtils.count(vertex.get().edges(direction, labels))).fold(new Tuple2<>("sum", 0l), (a, b) -> new Tuple2<>("sum", a._2() + b._2()))._2();
+ } else if (step instanceof CountGlobalStep) {
+ if (count == -1)
+ count = nextRDD.count();
+ }
+ }
+ assert count != -1;
+ final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(count, (Step) traversal.getEndStep(), 1l));
+ memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
+ memory.incrIteration();
+ return inputRDD;
+ }
+
+ public static boolean isLegal(final Traversal.Admin<?, ?> traversal) {
+ if (!TraversalHelper.getLabels(traversal).isEmpty())
+ return false;
+ final List<Step> steps = traversal.getSteps();
+ if (!steps.get(0).getClass().equals(GraphStep.class) || ((GraphStep) steps.get(0)).returnsEdge())
+ return false;
+ if (!steps.get(steps.size() - 1).getClass().equals(CountGlobalStep.class))
+ return false;
+ int vertexPropertiesStepCount = 0;
+ for (int i = 1; i < steps.size() - 1; i++) {
+ final Step<?, ?> step = steps.get(i);
+ final Class<? extends Step> stepClass = step.getClass();
+ if (!stepClass.equals(HasStep.class) && !stepClass.equals(PropertiesStep.class) && !stepClass.equals(VertexStep.class))
+ return false;
+ if ((stepClass.equals(VertexStep.class) || stepClass.equals(PropertiesStep.class)) && (++vertexPropertiesStepCount > 1 || !step.getNextStep().getClass().equals(CountGlobalStep.class)))
+ return false;
+ if (stepClass.equals(HasStep.class) && !step.getPreviousStep().getClass().equals(GraphStep.class) && !step.getPreviousStep().getClass().equals(HasStep.class))
+ return false;
+ }
+ return true;
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f0acfd75/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java
deleted file mode 100644
index 8feb628..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkVertexCountInterceptor implements SparkVertexProgramInterceptor<TraversalVertexProgram> {
-
- public SparkVertexCountInterceptor() {
-
- }
-
- @Override
- public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
- vertexProgram.setup(memory);
- final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal();
- final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
- haltedTraversers.add(traversal.getTraverserGenerator().generate(inputRDD.count(), (Step) traversal.getEndStep(), 1l));
- memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
- memory.incrIteration();
- return inputRDD;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f0acfd75/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index 1e73372..c1cf8cf 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -24,15 +24,18 @@ import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.VertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkVertexCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkCountInterceptor;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -55,40 +58,56 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
///
Graph graph = GraphFactory.open(configuration);
GraphTraversalSource g = graph.traversal().withComputer();
assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
- //
- assertEquals(6l, g.V().count().next().longValue());
- assertEquals(2l, g.V().out().out().count().next().longValue());
+ /// SparkCountInterceptor matches
+ test(SparkCountInterceptor.class, 6l, g.V().count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("software").count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).values("name").count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name").count());
+ test(SparkCountInterceptor.class, 4l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name", "age").count());
+ test(SparkCountInterceptor.class, 3l, g.V().hasLabel("person").has("age", P.gt(30)).out().count());
+ test(SparkCountInterceptor.class, 0l, g.V().hasLabel("person").has("age", P.gt(30)).out("knows").count());
+ test(SparkCountInterceptor.class, 3l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age", P.gt(30)).out("created").count());
+ test(SparkCountInterceptor.class, 3l, g.V(1).out().count());
+ test(SparkCountInterceptor.class, 2l, g.V(1).out("knows").count());
+ test(SparkCountInterceptor.class, 3l, g.V(1).out("knows", "created").count());
+ test(SparkCountInterceptor.class, 5l, g.V(1, 4).out("knows", "created").count());
+ test(SparkCountInterceptor.class, 1l, g.V(2).in("knows").count());
+ test(SparkCountInterceptor.class, 0l, g.V(6).has("name", "peter").in().count());
+ /// No interceptor matches
+ test(2l, g.V().out().out().count());
+ test(6l, g.E().count());
+ test(6l, g.V().as("a").values("name").as("b").count());
+ test(6l, g.V().as("a").count());
+ test(1l, g.V().has("name", "marko").as("a").values("name").as("b").count());
+ test(2l, g.V().out().out().count());
+ test(6l, g.V().out().values("name").count());
+ test(2l, g.V().out("knows").values("name").count());
+ test(3l, g.V().in().has("name", "marko").count());
+ test(4l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age").out("created").count()); // TODO: filter(values()) should be okay.
}
- @Test
- public void shouldInterceptExceptedTraversals() throws Exception {
- final Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
- ///
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
- //
- assertEquals(SparkVertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().count()));
- assertEquals(SparkVertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().identity().identity().count()));
- assertNull(getInterceptor(g.V().out().count()));
- assertNull(getInterceptor(g.V().as("a").count()));
- }
-
- private static String getInterceptor(final Traversal<?, ?> traversal) {
- traversal.asAdmin().applyStrategies();
- return (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
+ private static <R> void test(Class<? extends VertexProgramInterceptor> expectedInterceptor, R expectedResult, final Traversal<?, R> traversal) throws Exception {
+ final Traversal.Admin<?, ?> clone = traversal.asAdmin().clone();
+ clone.applyStrategies();
+ final String interceptor = (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, clone).get()
.getComputer()
.getConfiguration()
.getOrDefault(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, null);
+ if (null == expectedInterceptor)
+ assertNull(interceptor);
+ else
+ assertEquals(expectedInterceptor, Class.forName(interceptor));
+ assertEquals(expectedResult, traversal.next());
+ }
+ private static <R> void test(R expectedResult, final Traversal<?, R> traversal) throws Exception {
+ test(null, expectedResult, traversal);
}
}
[11/18] incubator-tinkerpop git commit: merged master into this
branch. had to fix up SparkGraphComputer conflicts cause of ThreadInterrupt
work from @spmallette. Added some Scope-based recurssive methods to
TraversalHelper. SparkCountItercepter is now s
Posted by ok...@apache.org.
merged master into this branch. had to fix up SparkGraphComputer conflicts cause of ThreadInterrupt work from @spmallette. Added some Scope-based recurssive methods to TraversalHelper. SparkCountItercepter is now smart about sideEffects. Added more tests to ensure proper interception.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/01fda967
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/01fda967
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/01fda967
Branch: refs/heads/master
Commit: 01fda96707335f2e33b073cef980c7844ca6d80c
Parents: 71f66c5 5fafb0b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 13:48:39 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 13:48:39 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +
docker/scripts/build.sh | 1 +
.../upgrade/release-3.2.x-incubating.asciidoc | 47 +++++++-
.../benchmark/util/AbstractBenchmarkBase.java | 2 +-
.../traversal/step/map/VertexProgramStep.java | 20 +++-
.../gremlin/process/remote/RemoteGraph.java | 8 ++
.../traversal/step/util/AbstractStep.java | 3 +
.../process/traversal/util/TraversalHelper.java | 73 +++++++++---
.../util/TraversalInterruptedException.java | 27 +++++
.../traversal/util/DefaultTraversalTest.java | 40 +++++++
gremlin-driver/pom.xml | 1 -
.../process/AbstractGremlinProcessTest.java | 8 +-
.../gremlin/process/ProcessComputerSuite.java | 4 +
.../gremlin/process/ProcessStandardSuite.java | 2 +
.../TraversalInterruptionComputerTest.java | 97 ++++++++++++++++
.../traversal/TraversalInterruptionTest.java | 114 +++++++++++++++++++
.../gremlin/hadoop/structure/HadoopGraph.java | 9 ++
pom.xml | 5 +
.../process/computer/SparkGraphComputer.java | 24 +++-
.../interceptor/SparkCountInterceptor.java | 8 +-
.../SparkInterceptorStrategyTest.java | 4 +
tinkergraph-gremlin/pom.xml | 4 +
.../process/computer/TinkerGraphComputer.java | 30 ++++-
.../process/computer/TinkerWorkerPool.java | 12 +-
24 files changed, 513 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/01fda967/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
----------------------------------------------------------------------
diff --cc gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
index 98b337c,fe76381..09ea337
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
@@@ -19,10 -19,8 +19,11 @@@
package org.apache.tinkerpop.gremlin.process.traversal.util;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
++import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.lambda.ElementValueTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.lambda.TokenTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder;
import org.apache.tinkerpop.gremlin.process.traversal.step.Scoping;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
@@@ -228,16 -222,16 +229,24 @@@ public final class TraversalHelper
}
public static <S> List<S> getStepsOfAssignableClassRecursively(final Class<S> stepClass, final Traversal.Admin<?, ?> traversal) {
++ return getStepsOfAssignableClassRecursively(null, stepClass, traversal);
++ }
++
++ public static <S> List<S> getStepsOfAssignableClassRecursively(final Scope scope, final Class<S> stepClass, final Traversal.Admin<?, ?> traversal) {
final List<S> list = new ArrayList<>();
for (final Step<?, ?> step : traversal.getSteps()) {
if (stepClass.isAssignableFrom(step.getClass()))
list.add((S) step);
if (step instanceof TraversalParent) {
-- for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) {
-- list.addAll(TraversalHelper.getStepsOfAssignableClassRecursively(stepClass, localChild));
++ if (null == scope || Scope.local.equals(scope)) {
++ for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) {
++ list.addAll(TraversalHelper.getStepsOfAssignableClassRecursively(stepClass, localChild));
++ }
}
-- for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) {
-- list.addAll(TraversalHelper.getStepsOfAssignableClassRecursively(stepClass, globalChild));
++ if (null == scope || Scope.global.equals(scope)) {
++ for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) {
++ list.addAll(TraversalHelper.getStepsOfAssignableClassRecursively(stepClass, globalChild));
++ }
}
}
}
@@@ -287,7 -281,7 +296,7 @@@
/**
* Determine if the traversal has a step of an assignable class in the current {@link Traversal} and its
-- * child traversals.
++ * local and global child traversals.
*
* @param stepClass the step class to look for
* @param traversal the traversal in which to look for the given step class
@@@ -295,16 -289,16 +304,34 @@@
* given <code>stepClass</code>, otherwise <code>false</code>.
*/
public static boolean hasStepOfAssignableClassRecursively(final Class stepClass, final Traversal.Admin<?, ?> traversal) {
++ return hasStepOfAssignableClassRecursively(null, stepClass, traversal);
++ }
++
++ /**
++ * Determine if the traversal has a step of an assignable class in the current {@link Traversal} and its
++ * {@link Scope} child traversals.
++ *
++ * @param scope the child traversal scope to check
++ * @param stepClass the step class to look for
++ * @param traversal the traversal in which to look for the given step class
++ * @return <code>true</code> if any step in the given traversal (and its child traversals) is an instance of the
++ * given <code>stepClass</code>, otherwise <code>false</code>.
++ */
++ public static boolean hasStepOfAssignableClassRecursively(final Scope scope, final Class stepClass, final Traversal.Admin<?, ?> traversal) {
for (final Step<?, ?> step : traversal.getSteps()) {
if (stepClass.isAssignableFrom(step.getClass())) {
return true;
}
if (step instanceof TraversalParent) {
-- for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) {
-- if (hasStepOfAssignableClassRecursively(stepClass, localChild)) return true;
++ if (null == scope || Scope.local.equals(scope)) {
++ for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) {
++ if (hasStepOfAssignableClassRecursively(stepClass, localChild)) return true;
++ }
}
-- for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) {
-- if (hasStepOfAssignableClassRecursively(stepClass, globalChild)) return true;
++ if (null == scope || Scope.global.equals(scope)) {
++ for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) {
++ if (hasStepOfAssignableClassRecursively(stepClass, globalChild)) return true;
++ }
}
}
}
@@@ -313,7 -307,7 +340,7 @@@
/**
* Determine if the traversal has any of the supplied steps of an assignable class in the current {@link Traversal}
-- * and its child traversals.
++ * and its global or local child traversals.
*
* @param stepClasses the step classes to look for
* @param traversal the traversal in which to look for the given step classes
@@@ -321,6 -315,6 +348,20 @@@
* provided in <code>stepClasses</code>, otherwise <code>false</code>.
*/
public static boolean hasStepOfAssignableClassRecursively(final Collection<Class> stepClasses, final Traversal.Admin<?, ?> traversal) {
++ return hasStepOfAssignableClassRecursively(null, stepClasses, traversal);
++ }
++
++ /**
++ * Determine if the traversal has any of the supplied steps of an assignable class in the current {@link Traversal}
++ * and its {@link Scope} child traversals.
++ *
++ * @param scope whether to check global or local children (null for both).
++ * @param stepClasses the step classes to look for
++ * @param traversal the traversal in which to look for the given step classes
++ * @return <code>true</code> if any step in the given traversal (and its child traversals) is an instance of a class
++ * provided in <code>stepClasses</code>, otherwise <code>false</code>.
++ */
++ public static boolean hasStepOfAssignableClassRecursively(final Scope scope, final Collection<Class> stepClasses, final Traversal.Admin<?, ?> traversal) {
if (stepClasses.size() == 1)
return hasStepOfAssignableClassRecursively(stepClasses.iterator().next(), traversal);
for (final Step<?, ?> step : traversal.getSteps()) {
@@@ -328,11 -322,11 +369,15 @@@
return true;
}
if (step instanceof TraversalParent) {
-- for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) {
-- if (hasStepOfAssignableClassRecursively(stepClasses, localChild)) return true;
++ if (null == scope || Scope.local.equals(scope)) {
++ for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) {
++ if (hasStepOfAssignableClassRecursively(stepClasses, localChild)) return true;
++ }
}
-- for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) {
-- if (hasStepOfAssignableClassRecursively(stepClasses, globalChild)) return true;
++ if (null == scope || Scope.global.equals(scope)) {
++ for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) {
++ if (hasStepOfAssignableClassRecursively(stepClasses, globalChild)) return true;
++ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/01fda967/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index b7ec133,bc8bc50..82b3369
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@@ -51,11 -52,8 +52,12 @@@ import org.apache.tinkerpop.gremlin.pro
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
+ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedException;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkInterceptorStrategy;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkPartitionAwareStrategy;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputFormatRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputOutputHelper;
@@@ -71,9 -69,11 +73,12 @@@ import org.apache.tinkerpop.gremlin.str
import java.io.File;
import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+ import java.util.concurrent.ThreadFactory;
import java.util.stream.Stream;
/**
@@@ -81,15 -81,17 +86,22 @@@
*/
public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
-
+ private final org.apache.commons.configuration.Configuration sparkConfiguration;
+ private boolean workersSet = false;
+ private final ThreadFactory threadFactoryBoss = new BasicThreadFactory.Builder().namingPattern(SparkGraphComputer.class.getSimpleName() + "-boss").build();
+
+ /**
+ * An {@code ExecutorService} that schedules up background work. Since a {@link GraphComputer} is only used once
+ * for a {@link VertexProgram} a single threaded executor is sufficient.
+ */
+ private final ExecutorService computerService = Executors.newSingleThreadExecutor(threadFactoryBoss);
- private final org.apache.commons.configuration.Configuration sparkConfiguration;
- private boolean workersSet = false;
+ static {
+ TraversalStrategies.GlobalCache.registerStrategies(SparkGraphComputer.class,
+ TraversalStrategies.GlobalCache.getStrategies(GraphComputer.class).clone().addStrategies(
+ SparkPartitionAwareStrategy.instance(),
+ SparkInterceptorStrategy.instance()));
+ }
public SparkGraphComputer(final HadoopGraph hadoopGraph) {
super(hadoopGraph);
@@@ -120,8 -122,8 +132,8 @@@
}
private Future<ComputerResult> submitWithExecutor(Executor exec) {
-- // create the completable future �
- return CompletableFuture.<ComputerResult>supplyAsync(() -> {
++ // create the completable future
+ return computerService.submit(() -> {
final long startTime = System.currentTimeMillis();
// apache and hadoop configurations that are used throughout the graph computer computation
final org.apache.commons.configuration.Configuration apacheConfiguration = new HadoopConfiguration(this.sparkConfiguration);
@@@ -240,43 -236,34 +252,47 @@@
// process the vertex program //
////////////////////////////////
if (null != this.vertexProgram) {
- // set up the vertex program and wire up configurations
- JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
- this.vertexProgram.setup(memory);
- memory.broadcastMemory(sparkContext);
- final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
- this.vertexProgram.storeState(vertexProgramConfiguration);
- ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
- // execute the vertex program
- while (true) {
- if (Thread.interrupted()) {
- sparkContext.cancelAllJobs();
- throw new TraversalInterruptedException();
+ /////////////////
+ // if there is a registered VertexProgramInterceptor, use it to bypass the GraphComputer semantics
+ if (apacheConfiguration.containsKey(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR)) {
+ try {
+ final SparkVertexProgramInterceptor<VertexProgram> interceptor =
+ (SparkVertexProgramInterceptor) Class.forName(apacheConfiguration.getString(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR)).newInstance();
+ computedGraphRDD = interceptor.apply(this.vertexProgram, loadedGraphRDD, memory);
+ } catch (final ClassNotFoundException | IllegalAccessException | InstantiationException e) {
+ throw new IllegalStateException(e.getMessage());
}
- memory.setInExecute(true);
- viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(loadedGraphRDD, viewIncomingRDD, memory, vertexProgramConfiguration);
- memory.setInExecute(false);
- if (this.vertexProgram.terminate(memory))
- break;
- else {
- memory.incrIteration();
- memory.broadcastMemory(sparkContext);
+ } else { // standard GraphComputer semantics
+ // set up the vertex program and wire up configurations
+ this.vertexProgram.setup(memory);
+ JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
+ memory.broadcastMemory(sparkContext);
+ final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
+ this.vertexProgram.storeState(vertexProgramConfiguration);
+ ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
+ ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
+ // execute the vertex program
+ while (true) {
++ if (Thread.interrupted()) {
++ sparkContext.cancelAllJobs();
++ throw new TraversalInterruptedException();
++ }
+ memory.setInExecute(true);
+ viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(loadedGraphRDD, viewIncomingRDD, memory, vertexProgramConfiguration);
+ memory.setInExecute(false);
+ if (this.vertexProgram.terminate(memory))
+ break;
+ else {
+ memory.incrIteration();
+ memory.broadcastMemory(sparkContext);
+ }
}
+ // write the computed graph to the respective output (rdd or output format)
+ computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
}
+ /////////////////
memory.complete(); // drop all transient memory keys
- // write the computed graph to the respective output (rdd or output format)
- computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
if (null != outputRDD && !this.persist.equals(Persist.NOTHING)) {
outputRDD.writeGraphRDD(apacheConfiguration, computedGraphRDD);
}
@@@ -405,4 -391,4 +421,4 @@@
final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
new SparkGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
}
--}
++}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/01fda967/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
index e737108,0000000..f85d6db
mode 100644,000000..100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
@@@ -1,97 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
++import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
++import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
++import org.apache.tinkerpop.gremlin.process.traversal.step.util.CollectingBarrierStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
++import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkCountInterceptor implements SparkVertexProgramInterceptor<TraversalVertexProgram> {
+
+ public SparkCountInterceptor() {
+
+ }
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
+ vertexProgram.setup(memory);
+ final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal().getPure().clone();
+ final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds(); // any V(1,2,3)-style ids to filter on
+ final CountGlobalStep countGlobalStep = (CountGlobalStep) traversal.getEndStep(); // needed for the final traverser generation
+ traversal.removeStep(0); // remove GraphStep
+ traversal.removeStep(traversal.getSteps().size() - 1); // remove CountGlobalStep
+ traversal.applyStrategies(); // compile
+ boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
+
+ ((MemoryTraversalSideEffects) traversal.getSideEffects()).setMemory(memory, true); // any intermediate sideEffect steps are backed by SparkMemory
+ memory.setInExecute(true);
+ final long count = inputRDD
+ .filter(tuple -> ElementHelper.idExists(tuple._2().get().id(), graphStepIds))
+ .flatMapValues(vertexWritable -> {
+ if (identityTraversal) // g.V.count()-style (identity)
+ return () -> (Iterator) IteratorUtils.of(vertexWritable);
+ else { // add the vertex to head of the traversal
+ return () -> { // and iterate it for its results
- final Traversal.Admin<Vertex, ?> clone = traversal.clone();
++ final Traversal.Admin<Vertex, ?> clone = traversal.clone(); // need a unique clone for each vertex to isolate the computation
+ clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
+ return clone;
+ };
+ }
+ }).count();
+ memory.setInExecute(false);
+
+ // generate the HALTED_TRAVERSERS for the memory
+ final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(count, countGlobalStep, 1l));
+ memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
+ memory.incrIteration(); // any local star graph reduction take a single iteration
+ return inputRDD;
+ }
+
+ public static boolean isLegal(final Traversal.Admin<?, ?> traversal) {
+ final List<Step> steps = traversal.getSteps();
+ if (!steps.get(0).getClass().equals(GraphStep.class) || ((GraphStep) steps.get(0)).returnsEdge())
+ return false;
+ if (!steps.get(steps.size() - 1).getClass().equals(CountGlobalStep.class))
+ return false;
++ if (TraversalHelper.hasStepOfAssignableClassRecursively(Scope.global, Arrays.asList(CollectingBarrierStep.class, DedupGlobalStep.class), traversal))
++ return false;
+ return TraversalHelper.isLocalStarGraph(traversal);
+
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/01fda967/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index c89aed8,0000000..4529fd3
mode 100644,000000..100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@@ -1,141 -1,0 +1,145 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.VertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
++import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkInterceptorStrategyTest extends AbstractSparkTest {
+
+ @Test
+ public void shouldHandleSideEffectsCorrectly() throws Exception {
+ final Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
+ /// groupCount(m)-test
+ Traversal.Admin<Vertex, Long> traversal = g.V().groupCount("m").by(T.label).count().asAdmin();
+ test(SparkCountInterceptor.class, 6l, traversal);
+ assertEquals(1, traversal.getSideEffects().keys().size());
+ assertTrue(traversal.getSideEffects().exists("m"));
+ assertTrue(traversal.getSideEffects().keys().contains("m"));
+ final Map<String, Long> map = traversal.getSideEffects().get("m");
+ assertEquals(2, map.size());
+ assertEquals(2, map.get("software").intValue());
+ assertEquals(4, map.get("person").intValue());
+ }
+
+ @Test
+ public void shouldSuccessfullyEvaluateInterceptedTraversals() throws Exception {
+ final Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
+ /// SparkCountInterceptor matches
+ test(SparkCountInterceptor.class, 6l, g.V().count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("software").count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).values("name").count());
+ test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name").count());
+ test(SparkCountInterceptor.class, 4l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name", "age").count());
+ test(SparkCountInterceptor.class, 3l, g.V().hasLabel("person").has("age", P.gt(30)).out().count());
+ test(SparkCountInterceptor.class, 0l, g.V().hasLabel("person").has("age", P.gt(30)).out("knows").count());
+ test(SparkCountInterceptor.class, 3l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age", P.gt(30)).out("created").count());
+ test(SparkCountInterceptor.class, 3l, g.V(1).out().count());
+ test(SparkCountInterceptor.class, 2l, g.V(1).out("knows").count());
+ test(SparkCountInterceptor.class, 3l, g.V(1).out("knows", "created").count());
+ test(SparkCountInterceptor.class, 5l, g.V(1, 4).out("knows", "created").count());
+ test(SparkCountInterceptor.class, 1l, g.V(2).in("knows").count());
+ test(SparkCountInterceptor.class, 0l, g.V(6).has("name", "peter").in().count());
+ test(SparkCountInterceptor.class, 6l, g.V().as("a").values("name").as("b").count());
+ test(SparkCountInterceptor.class, 6l, g.V().as("a").count());
+ test(SparkCountInterceptor.class, 1l, g.V().has("name", "marko").as("a").values("name").as("b").count());
+ test(SparkCountInterceptor.class, 4l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age").out("created").count());
+ /// No interceptor matches
+ test(2l, g.V().out().out().count());
+ test(6l, g.E().count());
+ test(2l, g.V().out().out().count());
+ test(6l, g.V().out().values("name").count());
+ test(2l, g.V().out("knows").values("name").count());
+ test(3l, g.V().in().has("name", "marko").count());
++ test(6l, g.V().repeat(__.dedup()).times(2).count());
++ test(6l, g.V().dedup().count());
++ test(4l, g.V().hasLabel("person").order().by("age").count());
+ }
+
+ private static <R> void test(Class<? extends VertexProgramInterceptor> expectedInterceptor, R expectedResult, final Traversal<?, R> traversal) throws Exception {
+ final Traversal.Admin<?, ?> clone = traversal.asAdmin().clone();
+ clone.applyStrategies();
+ final String interceptor = (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, clone).get()
+ .getComputer()
+ .getConfiguration()
+ .getOrDefault(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, null);
+ if (null == expectedInterceptor)
+ assertNull(interceptor);
+ else
+ assertEquals(expectedInterceptor, Class.forName(interceptor));
+ assertEquals(expectedResult, traversal.next());
+ }
+
+ private static <R> void test(R expectedResult, final Traversal<?, R> traversal) throws Exception {
+ test(null, expectedResult, traversal);
+ }
+}
[02/18] incubator-tinkerpop git commit: syncing with 1120.
Posted by ok...@apache.org.
syncing with 1120.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e8bba359
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e8bba359
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e8bba359
Branch: refs/heads/master
Commit: e8bba359be7a773a138b023929c0a2c02a9d6de8
Parents: 77480a2
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 3 14:48:27 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 3 14:48:27 2016 -0600
----------------------------------------------------------------------
.../spark/process/computer/SparkExecutor.java | 138 ++++++++++---------
1 file changed, 70 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e8bba359/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index aadb70d..3e6a816 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -21,8 +21,6 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import com.google.common.base.Optional;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.function.Function2;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
@@ -62,7 +60,7 @@ public final class SparkExecutor {
// DATA LOADING //
//////////////////
- public static JavaPairRDD<Object, VertexWritable> applyGraphFilter(JavaPairRDD<Object, VertexWritable> graphRDD, final GraphFilter graphFilter) {
+ public static JavaPairRDD<Object, VertexWritable> applyGraphFilter(final JavaPairRDD<Object, VertexWritable> graphRDD, final GraphFilter graphFilter) {
return graphRDD.mapPartitionsToPair(partitionIterator -> {
final GraphFilter gFilter = graphFilter.clone();
return () -> IteratorUtils.filter(partitionIterator, tuple -> (tuple._2().get().applyGraphFilter(gFilter)).isPresent());
@@ -80,17 +78,16 @@ public final class SparkExecutor {
final SparkMemory memory,
final Configuration apacheConfiguration) {
- final boolean partitionedGraphRDD = graphRDD.partitioner().isPresent();
- if (partitionedGraphRDD && null != viewIncomingRDD) // the graphRDD and the viewRDD must have the same partitioner
+ if (null != viewIncomingRDD) // the graphRDD and the viewRDD must have the same partitioner
assert graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get());
- final JavaPairRDD<Object, ViewOutgoingPayload<M>> viewOutgoingRDD = (((null == viewIncomingRDD) ?
+ final JavaPairRDD<Object, ViewOutgoingPayload<M>> viewOutgoingRDD = ((null == viewIncomingRDD) ?
graphRDD.mapValues(vertexWritable -> new Tuple2<>(vertexWritable, Optional.<ViewIncomingPayload<M>>absent())) : // first iteration will not have any views or messages
graphRDD.leftOuterJoin(viewIncomingRDD)) // every other iteration may have views and messages
// for each partition of vertices emit a view and their outgoing messages
.mapPartitionsToPair(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
- final String[] elementComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
+ final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
final SparkMessenger<M> messenger = new SparkMessenger<>();
workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
return () -> IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
@@ -98,70 +95,67 @@ public final class SparkExecutor {
final boolean hasViewAndMessages = vertexViewIncoming._2()._2().isPresent(); // if this is the first iteration, then there are no views or messages
final List<DetachedVertexProperty<Object>> previousView = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getView() : memory.isInitialIteration() ? new ArrayList<>() : Collections.emptyList();
// revive compute properties if they already exist
- if (memory.isInitialIteration() && elementComputeKeysArray.length > 0) {
- vertex.properties(elementComputeKeysArray).forEachRemaining(vertexProperty -> previousView.add(DetachedFactory.detach(vertexProperty, true)));
- }
+ if (memory.isInitialIteration() && vertexComputeKeysArray.length > 0)
+ vertex.properties(vertexComputeKeysArray).forEachRemaining(vertexProperty -> previousView.add(DetachedFactory.detach(vertexProperty, true)));
// drop any computed properties that are cached in memory
- if (elementComputeKeysArray.length > 0)
- vertex.dropVertexProperties(elementComputeKeysArray);
+ vertex.dropVertexProperties(vertexComputeKeysArray);
final List<M> incomingMessages = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getIncomingMessages() : Collections.emptyList();
- previousView.forEach(property -> property.attach(Attachable.Method.create(vertex))); // attach the view to the vertex
- // previousView.clear(); // no longer needed so kill it from memory
- ///
+ IteratorUtils.removeOnNext(previousView.iterator()).forEachRemaining(property -> property.attach(Attachable.Method.create(vertex))); // attach the view to the vertex
+ assert previousView.isEmpty();
+ // do the vertex's vertex program iteration
messenger.setVertexAndIncomingMessages(vertex, incomingMessages); // set the messenger with the incoming messages
workerVertexProgram.execute(ComputerGraph.vertexProgram(vertex, workerVertexProgram), messenger, memory); // execute the vertex program on this vertex for this iteration
- // incomingMessages.clear(); // no longer needed so kill it from memory
- ///
- final List<DetachedVertexProperty<Object>> nextView = elementComputeKeysArray.length == 0 ? // not all vertex programs have compute keys
+ // assert incomingMessages.isEmpty(); // maybe the program didn't read all the messages
+ incomingMessages.clear();
+ // detached the compute property view from the vertex
+ final List<DetachedVertexProperty<Object>> nextView = vertexComputeKeysArray.length == 0 ? // not all vertex programs have compute keys
Collections.emptyList() :
- IteratorUtils.list(IteratorUtils.map(vertex.properties(elementComputeKeysArray), property -> DetachedFactory.detach(property, true)));
- final List<Tuple2<Object, M>> outgoingMessages = messenger.getOutgoingMessages(); // get the outgoing messages
+ IteratorUtils.list(IteratorUtils.map(vertex.properties(vertexComputeKeysArray), vertexProperty -> DetachedFactory.detach(vertexProperty, true)));
+ // drop compute property view as it has now been detached from the vertex
+ vertex.dropVertexProperties(vertexComputeKeysArray);
+ final List<Tuple2<Object, M>> outgoingMessages = messenger.getOutgoingMessages(); // get the outgoing messages being sent by this vertex
if (!partitionIterator.hasNext())
workerVertexProgram.workerIterationEnd(memory.asImmutable()); // if no more vertices in the partition, end the worker's iteration
- return new Tuple2<>(vertex.id(), new ViewOutgoingPayload<>(nextView, outgoingMessages));
+ return (nextView.isEmpty() && outgoingMessages.isEmpty()) ?
+ null : // if there is no view nor outgoing messages, emit nothing
+ new Tuple2<>(vertex.id(), new ViewOutgoingPayload<>(nextView, outgoingMessages)); // else, emit the vertex id, its view, and its outgoing messages
});
- }, true)); // true means that the partition is preserved
+ }, true) // true means that the partition is preserved
+ .filter(tuple -> null != tuple); // if there are no messages or views, then the tuple is null (memory optimization)
// the graphRDD and the viewRDD must have the same partitioner
- if (partitionedGraphRDD)
- assert graphRDD.partitioner().get().equals(viewOutgoingRDD.partitioner().get());
+ assert graphRDD.partitioner().get().equals(viewOutgoingRDD.partitioner().get());
// "message pass" by reducing on the vertex object id of the view and message payloads
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().orElse(null);
-
- /////////////////////////////////////////////////////////////
- /////////////////////////////////////////////////////////////
- final PairFlatMapFunction<Tuple2<Object, ViewOutgoingPayload<M>>, Object, Payload> messageFunction =
- tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
- IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())), // emit the view payload
- IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))));
- final Function2<Payload, Payload, Payload> reducerFunction = (a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
- if (a instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
- return a;
- } else if (b instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
- return b;
- } else {
- final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
- c.mergePayload(a, messageCombiner);
- c.mergePayload(b, messageCombiner);
- return c;
- }
- };
- /////////////////////////////////////////////////////////////
- /////////////////////////////////////////////////////////////
-
- final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD =
- (partitionedGraphRDD ?
- viewOutgoingRDD.flatMapToPair(messageFunction).reduceByKey(graphRDD.partitioner().get(), reducerFunction) :
- viewOutgoingRDD.flatMapToPair(messageFunction).reduceByKey(reducerFunction))
- .filter(payload -> !(payload._2() instanceof MessagePayload)) // this happens if there is a message to a vertex that does not exist
- .filter(payload -> !((payload._2() instanceof ViewIncomingPayload) && !((ViewIncomingPayload<M>) payload._2()).hasView())) // this happens if there are many messages to a vertex that does not exist
- .mapValues(payload -> payload instanceof ViewIncomingPayload ?
- (ViewIncomingPayload<M>) payload : // this happens if there is a vertex with incoming messages
- new ViewIncomingPayload<>((ViewPayload) payload)); // this happens if there is a vertex with no incoming messages
+ final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD = viewOutgoingRDD
+ .flatMapToPair(tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
+ // emit the view payload
+ IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())),
+ // emit the outgoing message payloads one by one
+ IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2())))))
+ .reduceByKey(graphRDD.partitioner().get(), (a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
+ if (a instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
+ return a;
+ } else if (b instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
+ return b;
+ } else {
+ final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
+ c.mergePayload(a, messageCombiner);
+ c.mergePayload(b, messageCombiner);
+ return c;
+ }
+ })
+ .mapValues(payload -> { // handle various corner cases of when views don't exist, messages don't exist, or neither exists.
+ if (payload instanceof ViewIncomingPayload) // this happens if there is a vertex view with incoming messages
+ return (ViewIncomingPayload<M>) payload;
+ else if (payload instanceof ViewPayload) // this happens if there is a vertex view with no incoming messages
+ return new ViewIncomingPayload<>((ViewPayload) payload);
+ else // this happens when there is a single message to a vertex that has no view or outgoing messages
+ return new ViewIncomingPayload<>((MessagePayload<M>) payload);
+ });
// the graphRDD and the viewRDD must have the same partitioner
- if (partitionedGraphRDD)
- assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
+ assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
newViewIncomingRDD
.foreachPartition(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
@@ -169,17 +163,20 @@ public final class SparkExecutor {
return newViewIncomingRDD;
}
- public static <M> JavaPairRDD<Object, VertexWritable> prepareFinalGraphRDD(final JavaPairRDD<Object, VertexWritable> graphRDD, final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD, final Set<VertexComputeKey> vertexComputeKeys) {
+ public static <M> JavaPairRDD<Object, VertexWritable> prepareFinalGraphRDD(
+ final JavaPairRDD<Object, VertexWritable> graphRDD,
+ final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD,
+ final Set<VertexComputeKey> vertexComputeKeys) {
// the graphRDD and the viewRDD must have the same partitioner
- if (graphRDD.partitioner().isPresent())
- assert (graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get()));
- // attach the final computed view to the cached graph
+ assert (graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get()));
+ final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(vertexComputeKeys); // the compute keys as an array
return graphRDD.leftOuterJoin(viewIncomingRDD)
.mapValues(tuple -> {
final StarGraph.StarVertex vertex = tuple._1().get();
+ vertex.dropVertexProperties(vertexComputeKeysArray); // drop all existing compute keys
+ // attach the final computed view to the cached graph
final List<DetachedVertexProperty<Object>> view = tuple._2().isPresent() ? tuple._2().get().getView() : Collections.emptyList();
for (final DetachedVertexProperty<Object> property : view) {
- vertex.dropVertexProperties(property.key());
if (!VertexProgramHelper.isTransientVertexComputeKey(property.key(), vertexComputeKeys))
property.attach(Attachable.Method.create(vertex));
}
@@ -191,7 +188,9 @@ public final class SparkExecutor {
// MAP REDUCE //
////////////////
- public static <K, V> JavaPairRDD<K, V> executeMap(final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration apacheConfiguration) {
+ public static <K, V> JavaPairRDD<K, V> executeMap(
+ final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce,
+ final Configuration apacheConfiguration) {
JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
return () -> new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration), partitionIterator);
@@ -201,14 +200,17 @@ public final class SparkExecutor {
return mapRDD;
}
- public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeCombine(final JavaPairRDD<K, V> mapRDD, final Configuration apacheConfiguration) {
+ public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeCombine(final JavaPairRDD<K, V> mapRDD,
+ final Configuration apacheConfiguration) {
return mapRDD.mapPartitionsToPair(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
return () -> new CombineIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration), partitionIterator);
});
}
- public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce(final JavaPairRDD<K, V> mapOrCombineRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration apacheConfiguration) {
+ public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce(
+ final JavaPairRDD<K, V> mapOrCombineRDD, final MapReduce<K, V, OK, OV, ?> mapReduce,
+ final Configuration apacheConfiguration) {
JavaPairRDD<OK, OV> reduceRDD = mapOrCombineRDD.groupByKey().mapPartitionsToPair(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
return () -> new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration), partitionIterator);
@@ -217,4 +219,4 @@ public final class SparkExecutor {
reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get(), true, 1);
return reduceRDD;
}
-}
+}
\ No newline at end of file
[06/18] incubator-tinkerpop git commit: lots of cleanup.
NativeInterceptor is now called VertexProgramInterceptor. Its generalized to
be used by any GraphComputer implementation. There exists
SparkVertexProgramInterceptor which extends VertexProgramInter
Posted by ok...@apache.org.
lots of cleanup. NativeInterceptor is now called VertexProgramInterceptor. Its generalized to be used by any GraphComputer implementation. There exists SparkVertexProgramInterceptor which extends VertexProgramInterceptor. Cleaned up naming conventions in the test cases. Found a bug in TraversalSource that was causing RemoteGraph computer tests to fail. SparkGraphComputer is cleaner with how interception happens -- really clear delineation which will be used for all GraphComptuers that use the Interceptor-model. Running integration tests over night.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d5e7c786
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d5e7c786
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d5e7c786
Branch: refs/heads/master
Commit: d5e7c78628d89c464418f1329a797fc192880e8b
Parents: 7c103c8
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 3 20:55:03 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 3 20:55:03 2016 -0600
----------------------------------------------------------------------
.../traversal/strategy/TraversalMatcher.java | 47 ++++++++
.../strategy/VertexProgramInterceptor.java | 31 +++++
.../process/traversal/TraversalSource.java | 8 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 2 +-
.../process/computer/NativeInterceptor.java | 32 ------
.../process/computer/SparkGraphComputer.java | 19 +--
.../optimization/SparkInterceptorStrategy.java | 60 ----------
.../SparkPartitionAwareStrategy.java | 86 --------------
.../interceptors/VertexCountInterceptor.java | 52 ---------
.../strategy/SparkVertexProgramInterceptor.java | 32 ++++++
.../optimization/SparkInterceptorStrategy.java | 60 ++++++++++
.../SparkPartitionAwareStrategy.java | 86 ++++++++++++++
.../SparkVertexCountInterceptor.java | 53 +++++++++
.../SparkInterceptorStrategyTest.java | 96 ----------------
.../SparkPartitionAwareStrategyTest.java | 115 -------------------
.../SparkInterceptorStrategyTest.java | 94 +++++++++++++++
.../SparkPartitionAwareStrategyTest.java | 113 ++++++++++++++++++
17 files changed, 536 insertions(+), 450 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java
new file mode 100644
index 0000000..cd37031
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/TraversalMatcher.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.computer.traversal.strategy;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
+
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class TraversalMatcher {
+
+ private TraversalMatcher() {
+ }
+
+ public static boolean is_g_V_count(final Traversal.Admin<?, ?> traversal) {
+ final List<Step> steps = traversal.getSteps();
+ return steps.size() == 2 &&
+ steps.get(0).getLabels().isEmpty() &&
+ steps.get(1).getLabels().isEmpty() &&
+ (steps.get(1) instanceof CountGlobalStep) &&
+ traversal.getSteps().get(0) instanceof GraphStep &&
+ ((GraphStep) traversal.getSteps().get(0)).returnsVertex() &&
+ ((GraphStep) traversal.getSteps().get(0)).getIds().length == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/VertexProgramInterceptor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/VertexProgramInterceptor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/VertexProgramInterceptor.java
new file mode 100644
index 0000000..f174034
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/VertexProgramInterceptor.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.computer.traversal.strategy;
+
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface VertexProgramInterceptor<V extends VertexProgram, G, M extends Memory> {
+
+ public G apply(final V vertexProgram, final G graph, final M memory);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
index 3b60940..cf0b51f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalSource.java
@@ -98,7 +98,13 @@ public interface TraversalSource extends Cloneable {
* @return a new traversal source with updated strategies
*/
public default TraversalSource withComputer(final Computer computer) {
- final List<TraversalStrategy<?>> graphComputerStrategies = TraversalStrategies.GlobalCache.getStrategies(computer.apply(this.getGraph()).getClass()).toList();
+ Class<? extends GraphComputer> graphComputerClass;
+ try {
+ graphComputerClass = computer.apply(this.getGraph()).getClass();
+ } catch (final Exception e) {
+ graphComputerClass = computer.getGraphComputerClass();
+ }
+ final List<TraversalStrategy<?>> graphComputerStrategies = TraversalStrategies.GlobalCache.getStrategies(graphComputerClass).toList();
final TraversalStrategy[] traversalStrategies = new TraversalStrategy[graphComputerStrategies.size() + 1];
traversalStrategies[0] = new VertexProgramStrategy(computer);
for (int i = 0; i < graphComputerStrategies.size(); i++) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index ab45903..fb4f15a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -39,6 +39,7 @@ public final class Constants {
public static final String GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES = "gremlin.hadoop.graphWriter.hasEdges";
public static final String GREMLIN_HADOOP_GRAPH_FILTER = "gremlin.hadoop.graphFilter";
public static final String GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER = "gremlin.hadoop.defaultGraphComputer";
+ public static final String GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR = "gremlin.hadoop.vertexProgramInterceptor";
public static final String GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE = "gremlin.hadoop.jarsInDistributedCache";
public static final String HIDDEN_G = Graph.Hidden.hide("g");
@@ -58,7 +59,6 @@ public final class Constants {
public static final String GREMLIN_SPARK_GRAPH_STORAGE_LEVEL = "gremlin.spark.graphStorageLevel";
public static final String GREMLIN_SPARK_PERSIST_STORAGE_LEVEL = "gremlin.spark.persistStorageLevel";
public static final String GREMLIN_SPARK_SKIP_PARTITIONER = "gremlin.spark.skipPartitioner";
- public static final String GREMLIN_SPARK_NATIVE_INTERCEPTOR = "gremlin.spark.nativeInterceptor";
public static final String SPARK_SERIALIZER = "spark.serializer";
public static String getGraphLocation(final String location) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java
deleted file mode 100644
index 8448a1b..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public interface NativeInterceptor<V extends VertexProgram> {
-
- public JavaPairRDD<Object, VertexWritable> apply(final V vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 49de96a..65e1c96 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -53,8 +53,9 @@ import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.SparkInterceptorStrategy;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.SparkPartitionAwareStrategy;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkInterceptorStrategy;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkPartitionAwareStrategy;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputFormatRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputOutputHelper;
@@ -241,16 +242,19 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
////////////////////////////////
if (null != this.vertexProgram) {
memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
- this.vertexProgram.setup(memory);
- if (apacheConfiguration.containsKey(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR)) {
+ /////////////////
+ // if there is a registered VertexProgramInterceptor, use it to by pass the GraphComputer semantics
+ if (apacheConfiguration.containsKey(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR)) {
try {
- final NativeInterceptor<VertexProgram> interceptor = (NativeInterceptor) Class.forName(apacheConfiguration.getString(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR)).newInstance();
+ final SparkVertexProgramInterceptor<VertexProgram> interceptor =
+ (SparkVertexProgramInterceptor) Class.forName(apacheConfiguration.getString(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR)).newInstance();
computedGraphRDD = interceptor.apply(this.vertexProgram, loadedGraphRDD, memory);
} catch (final ClassNotFoundException | IllegalAccessException | InstantiationException e) {
throw new IllegalStateException(e.getMessage());
}
- } else {
+ } else { // standard GraphComputer semantics
// set up the vertex program and wire up configurations
+ this.vertexProgram.setup(memory);
JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
memory.broadcastMemory(sparkContext);
final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
@@ -272,6 +276,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
// write the computed graph to the respective output (rdd or output format)
computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
}
+ /////////////////
memory.complete(); // drop all transient memory keys
if (null != outputRDD && !this.persist.equals(Persist.NOTHING)) {
outputRDD.writeGraphRDD(apacheConfiguration, computedGraphRDD);
@@ -318,7 +323,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
// unpersist the loaded graph if it will not be used again (no PersistedInputRDD)
// if the graphRDD was loaded from Spark, but then partitioned, its a different RDD
- if ((!inputFromSpark || partitioned || filtered) && computedGraphCreated)
+ if ((!skipPartitioner && (!inputFromSpark || partitioned || filtered)) && computedGraphCreated)
loadedGraphRDD.unpersist();
// unpersist the computed graph if it will not be used again (no PersistedOutputRDD)
if (!outputToSpark || this.persist.equals(GraphComputer.Persist.NOTHING))
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java
deleted file mode 100644
index 2ad1ad6..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.interceptors.VertexCountInterceptor;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
-
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
-
- private static final SparkInterceptorStrategy INSTANCE = new SparkInterceptorStrategy();
-
- private SparkInterceptorStrategy() {
- }
-
- @Override
- public void apply(final Traversal.Admin<?, ?> traversal) {
- final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance());
- final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
- for (final TraversalVertexProgramStep step : steps) {
- if (step.generateProgram(graph).getTraversal().equals(graph.traversal().V().count())) {
- step.setComputer(step.getComputer().configure(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR, VertexCountInterceptor.class.getCanonicalName()));
- }
- }
- }
-
- public static SparkInterceptorStrategy instance() {
- return INSTANCE;
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
deleted file mode 100644
index 5c28d41..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaFlatMapStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaMapStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkPartitionAwareStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
-
- private static final SparkPartitionAwareStrategy INSTANCE = new SparkPartitionAwareStrategy();
-
- private static final Set<Class<? extends Step>> MESSAGE_PASS_CLASSES = new HashSet<>(Arrays.asList(
- EdgeVertexStep.class,
- LambdaMapStep.class, // maybe?
- LambdaFlatMapStep.class // maybe?
- // VertexStep is special as you need to see if the return class is Edge or Vertex (logic below)
- ));
-
- private SparkPartitionAwareStrategy() {
- }
-
- @Override
- public void apply(final Traversal.Admin<?, ?> traversal) {
- final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
- for (final TraversalVertexProgramStep step : steps) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(traversal.getGraph().orElse(EmptyGraph.instance())).getTraversal();
- boolean messagePasses = MESSAGE_PASS_CLASSES.stream()
- .flatMap(clazz -> TraversalHelper.<Step<?, ?>>getStepsOfAssignableClassRecursively((Class) clazz, computerTraversal).stream())
- .filter(s -> TraversalHelper.isGlobalChild(((Step) s).getTraversal().asAdmin()))
- .findAny()
- .isPresent();
- if (!messagePasses) {
- for (final VertexStep vertexStep : TraversalHelper.getStepsOfAssignableClassRecursively(VertexStep.class, computerTraversal)) {
- if (TraversalHelper.isGlobalChild(vertexStep.getTraversal()) &&
- (vertexStep.returnsVertex() || !vertexStep.getDirection().equals(Direction.OUT))) { // in edges require message pass in OLAP
- messagePasses = true;
- break;
- }
- }
- }
- if (!messagePasses) // if no message passing, don't partition (save time)
- step.setComputer(step.getComputer().configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true));
- }
- }
-
- public static SparkPartitionAwareStrategy instance() {
- return INSTANCE;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java
deleted file mode 100644
index b685677..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.interceptors;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
-import org.apache.tinkerpop.gremlin.spark.process.computer.NativeInterceptor;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class VertexCountInterceptor implements NativeInterceptor<TraversalVertexProgram> {
-
- public VertexCountInterceptor() {
-
- }
-
- @Override
- public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
- final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal();
- final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
- haltedTraversers.add(traversal.getTraverserGenerator().generate(inputRDD.count(), (Step) traversal.getEndStep(), 1l));
- memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
- memory.incrIteration();
- return inputRDD;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/SparkVertexProgramInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/SparkVertexProgramInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/SparkVertexProgramInterceptor.java
new file mode 100644
index 0000000..0e34e45
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/SparkVertexProgramInterceptor.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.VertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface SparkVertexProgramInterceptor<V extends VertexProgram> extends VertexProgramInterceptor<V, JavaPairRDD<Object, VertexWritable>, SparkMemory> {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
new file mode 100644
index 0000000..a22cef7
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.TraversalMatcher;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkVertexCountInterceptor;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
+
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
+
+ private static final SparkInterceptorStrategy INSTANCE = new SparkInterceptorStrategy();
+
+ private SparkInterceptorStrategy() {
+ }
+
+ @Override
+ public void apply(final Traversal.Admin<?, ?> traversal) {
+ final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance());
+ final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
+ for (final TraversalVertexProgramStep step : steps) {
+ if (TraversalMatcher.is_g_V_count(step.generateProgram(graph).getTraversal()))
+ step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkVertexCountInterceptor.class.getCanonicalName()));
+ }
+ }
+
+ public static SparkInterceptorStrategy instance() {
+ return INSTANCE;
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
new file mode 100644
index 0000000..de3e737
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaFlatMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkPartitionAwareStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
+
+ private static final SparkPartitionAwareStrategy INSTANCE = new SparkPartitionAwareStrategy();
+
+ private static final Set<Class<? extends Step>> MESSAGEPASS_CLASSES = new HashSet<>(Arrays.asList(
+ EdgeVertexStep.class,
+ LambdaMapStep.class, // maybe?
+ LambdaFlatMapStep.class // maybe?
+ // VertexStep is special as you need to see if the return class is Edge or Vertex (logic below)
+ ));
+
+ private SparkPartitionAwareStrategy() {
+ }
+
+ @Override
+ public void apply(final Traversal.Admin<?, ?> traversal) {
+ final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
+ for (final TraversalVertexProgramStep step : steps) {
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(traversal.getGraph().orElse(EmptyGraph.instance())).getTraversal();
+ boolean messagePasses = MESSAGEPASS_CLASSES.stream()
+ .flatMap(clazz -> TraversalHelper.<Step<?, ?>>getStepsOfAssignableClassRecursively((Class) clazz, computerTraversal).stream())
+ .filter(s -> TraversalHelper.isGlobalChild(((Step) s).getTraversal().asAdmin()))
+ .findAny()
+ .isPresent();
+ if (!messagePasses) {
+ for (final VertexStep vertexStep : TraversalHelper.getStepsOfAssignableClassRecursively(VertexStep.class, computerTraversal)) {
+ if (TraversalHelper.isGlobalChild(vertexStep.getTraversal()) &&
+ (vertexStep.returnsVertex() || !vertexStep.getDirection().equals(Direction.OUT))) { // in edges require message pass in OLAP
+ messagePasses = true;
+ break;
+ }
+ }
+ }
+ if (!messagePasses) // if no message passing, don't partition (save time)
+ step.setComputer(step.getComputer().configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true));
+ }
+ }
+
+ public static SparkPartitionAwareStrategy instance() {
+ return INSTANCE;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java
new file mode 100644
index 0000000..8feb628
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkVertexCountInterceptor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkVertexCountInterceptor implements SparkVertexProgramInterceptor<TraversalVertexProgram> {
+
+ public SparkVertexCountInterceptor() {
+
+ }
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
+ vertexProgram.setup(memory);
+ final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal();
+ final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(inputRDD.count(), (Step) traversal.getEndStep(), 1l));
+ memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
+ memory.incrIteration();
+ return inputRDD;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java
deleted file mode 100644
index bb009eb..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.interceptors.VertexCountInterceptor;
-import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
-import org.junit.Test;
-
-import java.util.UUID;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class SparkInterceptorStrategyTest extends AbstractSparkTest {
-
- @Test
- public void shouldUseVertexCountInterceptor() throws Exception {
- final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
- Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
- ///
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
- assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
- assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
- //
- assertEquals(6l, g.V().count().next().longValue());
- assertEquals(2l, g.V().out().out().count().next().longValue());
- }
-
- @Test
- public void shouldSetConfigurationsCorrectly() throws Exception {
- final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
- Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
-
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
-
- assertEquals(VertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().count()));
- assertEquals(VertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().identity().identity().count()));
- assertNull(getInterceptor(g.V().out().count()));
- assertNull(getInterceptor(g.V().as("a").count()));
- }
-
- private static String getInterceptor(final Traversal<?, ?> traversal) {
- traversal.asAdmin().applyStrategies();
- return (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
- .getComputer()
- .getConfiguration()
- .getOrDefault(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR, null);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java
deleted file mode 100644
index 4324202..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategyTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.process.computer.Computer;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
-import org.junit.Test;
-
-import java.util.UUID;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class SparkPartitionAwareStrategyTest extends AbstractSparkTest {
-
- @Test
- public void shouldPartitionTheInputRDDAccordingly() throws Exception {
- final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
- Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
- configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
- ///
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
- assertTrue(g.getStrategies().toList().contains(SparkPartitionAwareStrategy.instance()));
- assertTrue(g.V().count().explain().toString().contains(SparkPartitionAwareStrategy.class.getSimpleName()));
- //
- assertEquals(6l, g.V().count().next().longValue());
- assertEquals(2l, g.V().out().out().count().next().longValue());
- }
-
-
- @Test
- public void shouldSetConfigurationsCorrectly() throws Exception {
- final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
- Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
-
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer(Computer.compute().persist(GraphComputer.Persist.EDGES).result(GraphComputer.ResultGraph.NEW));
-
- assertTrue(skipPartitioner(g.V().limit(10)));
- assertTrue(skipPartitioner(g.V().values("age").groupCount()));
- assertTrue(skipPartitioner(g.V().groupCount().by(__.out().count())));
- assertTrue(skipPartitioner(g.V().outE()));
- assertTrue(skipPartitioner(g.V().count()));
- assertTrue(skipPartitioner(g.V().out().count()));
- assertTrue(skipPartitioner(g.V().local(__.inE()).count()));
- assertTrue(skipPartitioner(g.V().outE().inV().count()));
- ////
- assertFalse(skipPartitioner(g.V().outE().inV()));
- assertFalse(skipPartitioner(g.V().both()));
- assertFalse(skipPartitioner(g.V().both().count()));
- assertFalse(skipPartitioner(g.V().out().id()));
- assertFalse(skipPartitioner(g.V().out().out().count()));
- assertFalse(skipPartitioner(g.V().in().count()));
- assertFalse(skipPartitioner(g.V().inE().count()));
-
-
- }
-
- private static boolean skipPartitioner(final Traversal<?, ?> traversal) {
- traversal.asAdmin().applyStrategies();
- return (Boolean) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
- .getComputer()
- .getConfiguration()
- .getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
new file mode 100644
index 0000000..1e73372
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkVertexCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkInterceptorStrategyTest extends AbstractSparkTest {
+
+ @Test
+ public void shouldSuccessfullyEvaluateInterceptedTraversals() throws Exception {
+ final Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
+ //
+ assertEquals(6l, g.V().count().next().longValue());
+ assertEquals(2l, g.V().out().out().count().next().longValue());
+ }
+
+ @Test
+ public void shouldInterceptExceptedTraversals() throws Exception {
+ final Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ //
+ assertEquals(SparkVertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().count()));
+ assertEquals(SparkVertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().identity().identity().count()));
+ assertNull(getInterceptor(g.V().out().count()));
+ assertNull(getInterceptor(g.V().as("a").count()));
+ }
+
+ private static String getInterceptor(final Traversal<?, ?> traversal) {
+ traversal.asAdmin().applyStrategies();
+ return (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
+ .getComputer()
+ .getConfiguration()
+ .getOrDefault(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, null);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d5e7c786/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java
new file mode 100644
index 0000000..045f424
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkPartitionAwareStrategyTest extends AbstractSparkTest {
+
+ @Test
+ public void shouldSuccessfullyEvaluateSkipPartitionedTraversals() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkPartitionAwareStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkPartitionAwareStrategy.class.getSimpleName()));
+ //
+ assertEquals(6l, g.V().count().next().longValue());
+ assertEquals(2l, g.V().out().out().count().next().longValue());
+ }
+
+
+ @Test
+ public void shouldSkipPartitionExceptedTraversals() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+
+ assertTrue(skipPartitioner(g.V().limit(10)));
+ assertTrue(skipPartitioner(g.V().values("age").groupCount()));
+ assertTrue(skipPartitioner(g.V().groupCount().by(__.out().count())));
+ assertTrue(skipPartitioner(g.V().outE()));
+ assertTrue(skipPartitioner(g.V().count()));
+ assertTrue(skipPartitioner(g.V().out().count()));
+ assertTrue(skipPartitioner(g.V().local(__.inE()).count()));
+ assertTrue(skipPartitioner(g.V().outE().inV().count()));
+ ////
+ assertFalse(skipPartitioner(g.V().outE().inV()));
+ assertFalse(skipPartitioner(g.V().both()));
+ assertFalse(skipPartitioner(g.V().both().count()));
+ assertFalse(skipPartitioner(g.V().out().id()));
+ assertFalse(skipPartitioner(g.V().out().out().count()));
+ assertFalse(skipPartitioner(g.V().in().count()));
+ assertFalse(skipPartitioner(g.V().inE().count()));
+
+
+ }
+
+ private static boolean skipPartitioner(final Traversal<?, ?> traversal) {
+ traversal.asAdmin().applyStrategies();
+ return (Boolean) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
+ .getComputer()
+ .getConfiguration()
+ .getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false);
+
+ }
+
+}
[13/18] incubator-tinkerpop git commit: got GroupCount working with
SparkStarBarrierInterceptor. Going to test on the Blades.
Posted by ok...@apache.org.
got GroupCount working with SparkStarBarrierInterceptor. Going to test on the Blades.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6e122e42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6e122e42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6e122e42
Branch: refs/heads/master
Commit: 6e122e42b9c49648ce42a295b94d780c3eb11aae
Parents: 6a256f8
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 16:56:04 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 16:56:04 2016 -0600
----------------------------------------------------------------------
.../interceptor/SparkStarBarrierInterceptor.java | 18 ++++++++++++++----
.../SparkInterceptorStrategyTest.java | 9 +++++++++
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6e122e42/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
index 3649c8f..76272be 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
@@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupCountStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MaxGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MeanGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MinGlobalStep;
@@ -42,13 +43,14 @@ import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequire
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import java.util.Iterator;
+import java.util.Map;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -75,7 +77,7 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
((MemoryTraversalSideEffects) traversal.getSideEffects()).setMemory(memory, true); // any intermediate sideEffect steps are backed by SparkMemory
memory.setInExecute(true);
final JavaRDD<Traverser.Admin<Object>> nextRDD = inputRDD.values()
- .filter(vertexWritable -> ElementHelper.idExists(vertexWritable.get().id(), graphStepIds))
+ .filter(vertexWritable -> ElementHelper.idExists(vertexWritable.get().id(), graphStepIds)) // ensure vertex ids are in V(x)
.flatMap(vertexWritable -> {
if (identityTraversal) // g.V.count()-style (identity)
return () -> IteratorUtils.of(traversal.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
@@ -107,6 +109,13 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
result = nextRDD
.map(traverser -> (Number) traverser.get())
.fold(Integer.MIN_VALUE, NumberHelper::max);
+ else if (endStep instanceof GroupCountStep)
+ result = nextRDD
+ .mapPartitions(partition -> {
+ final Traversal.Admin<?, Map<?, Long>> clone = (Traversal.Admin) endStepTraversal.clone();
+ return () -> IteratorUtils.map(partition, traverser -> TraversalUtil.apply((Traverser.Admin) traverser, clone));
+ })
+ .fold(((GroupCountStep<?, ?>) endStep).getSeedSupplier().get(), (a, b) -> GroupCountStep.GroupCountBiOperator.instance().apply((Map) a, (Map) b));
else
throw new IllegalArgumentException("The end step is an unsupported barrier: " + endStep);
memory.setInExecute(false);
@@ -129,8 +138,9 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
!endStep.getClass().equals(SumGlobalStep.class) &&
!endStep.getClass().equals(MeanGlobalStep.class) &&
!endStep.getClass().equals(MaxGlobalStep.class) &&
- !endStep.getClass().equals(MinGlobalStep.class))
- // TODO: group(), groupCount(), fold(), and tree()
+ !endStep.getClass().equals(MinGlobalStep.class) &&
+ !endStep.getClass().equals(GroupCountStep.class))
+ // TODO: group(), fold(), and tree()
return false;
if (TraversalHelper.getStepsOfAssignableClassRecursively(Scope.global, Barrier.class, traversal).size() != 1)
return false;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6e122e42/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index f18a944..b48a2bb 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -41,6 +41,8 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -117,6 +119,11 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
test(SparkStarBarrierInterceptor.class, 67l, g.V().has("age").has("age", P.gt(30)).values("age").sum());
test(SparkStarBarrierInterceptor.class, 27, g.V().hasLabel("person").values("age").min());
test(SparkStarBarrierInterceptor.class, 35, g.V().hasLabel("person").values("age").max());
+ test(SparkStarBarrierInterceptor.class, new HashMap<String, Long>() {{
+ put("software", 2l);
+ put("person", 4l);
+ }}, g.V().<String>groupCount().by(T.label));
+ test(SparkStarBarrierInterceptor.class, Collections.singletonMap("person", 2l), g.V().has("person", "age", P.lt(30)).<String>groupCount().by(T.label));
/// No interceptor matches
test(2l, g.V().out().out().count());
test(6l, g.E().count());
@@ -128,6 +135,8 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
test(6l, g.V().dedup().count());
test(4l, g.V().hasLabel("person").order().by("age").count());
test(1l, g.V().count().count());
+ test(2l, g.V().limit(2).count());
+ test(3l, g.V().tail(3).count());
}
private static <R> void test(Class<? extends VertexProgramInterceptor> expectedInterceptor, R expectedResult, final Traversal<?, R> traversal) throws Exception {
[10/18] incubator-tinkerpop git commit: Got sideEffects working with
SparkCountInterceptor. g.V.groupCount(m).count() works as expected. Classy.
Found a bug in TraversalHelper.isLocalStarGraph(), fixed.
Posted by ok...@apache.org.
Got sideEffects working with SparkCountInterceptor. g.V.groupCount(m).count() works as expected. Classy. Found a bug in TraversalHelper.isLocalStarGraph(), fixed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/71f66c51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/71f66c51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/71f66c51
Branch: refs/heads/master
Commit: 71f66c5114ae5e90aad8dc5f5cf77b200f6e1f30
Parents: 6f9e5fa
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 12:51:35 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 12:51:35 2016 -0600
----------------------------------------------------------------------
.../process/traversal/util/TraversalHelper.java | 6 +++++
.../spark/process/computer/SparkMemory.java | 2 +-
.../interceptor/SparkCountInterceptor.java | 18 +++++++++----
.../SparkInterceptorStrategyTest.java | 28 ++++++++++++++++++++
4 files changed, 48 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/71f66c51/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
index fe76381..98b337c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
@@ -21,6 +21,8 @@ package org.apache.tinkerpop.gremlin.process.traversal.util;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.lambda.ElementValueTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.lambda.TokenTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder;
import org.apache.tinkerpop.gremlin.process.traversal.step.Scoping;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
@@ -85,6 +87,10 @@ public final class TraversalHelper {
}
private static boolean isLocalStarGraph(final Traversal.Admin<?, ?> traversal, char state) {
+ if (state == 'u' &&
+ (traversal instanceof ElementValueTraversal ||
+ (traversal instanceof TokenTraversal && !((TokenTraversal) traversal).getToken().equals(T.id))))
+ return false;
for (final Step step : traversal.getSteps()) {
if (step instanceof RepeatStep &&
((RepeatStep<?>) step).getGlobalChildren().stream()
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/71f66c51/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
index 95bb18d..bf8590e 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
@@ -148,7 +148,7 @@ public final class SparkMemory implements Memory.Admin, Serializable {
this.memoryComputeKeys.values().stream().filter(MemoryComputeKey::isTransient).forEach(memoryComputeKey -> this.sparkMemory.remove(memoryComputeKey.getKey()));
}
- protected void setInExecute(final boolean inExecute) {
+ public void setInExecute(final boolean inExecute) {
this.inExecute = inExecute;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/71f66c51/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
index 6f91c42..e737108 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
@@ -51,23 +52,30 @@ public final class SparkCountInterceptor implements SparkVertexProgramIntercepto
public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
vertexProgram.setup(memory);
final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal().getPure().clone();
- final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds();
- final CountGlobalStep countGlobalStep = (CountGlobalStep) traversal.getEndStep();
+ final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds(); // any V(1,2,3)-style ids to filter on
+ final CountGlobalStep countGlobalStep = (CountGlobalStep) traversal.getEndStep(); // needed for the final traverser generation
traversal.removeStep(0); // remove GraphStep
traversal.removeStep(traversal.getSteps().size() - 1); // remove CountGlobalStep
traversal.applyStrategies(); // compile
boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
+
+ ((MemoryTraversalSideEffects) traversal.getSideEffects()).setMemory(memory, true); // any intermediate sideEffect steps are backed by SparkMemory
+ memory.setInExecute(true);
final long count = inputRDD
.filter(tuple -> ElementHelper.idExists(tuple._2().get().id(), graphStepIds))
.flatMapValues(vertexWritable -> {
if (identityTraversal) // g.V.count()-style (identity)
return () -> (Iterator) IteratorUtils.of(vertexWritable);
else { // add the vertex to head of the traversal
- final Traversal.Admin<Vertex, ?> clone = traversal.clone();
- clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
- return () -> clone;
+ return () -> { // and iterate it for its results
+ final Traversal.Admin<Vertex, ?> clone = traversal.clone();
+ clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
+ return clone;
+ };
}
}).count();
+ memory.setInExecute(false);
+
// generate the HALTED_TRAVERSERS for the memory
final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
haltedTraversers.add(traversal.getTraverserGenerator().generate(count, countGlobalStep, 1l));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/71f66c51/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index b2a4820..c89aed8 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -36,9 +36,11 @@ import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.op
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
+import java.util.Map;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
@@ -51,6 +53,32 @@ import static org.junit.Assert.assertTrue;
public class SparkInterceptorStrategyTest extends AbstractSparkTest {
@Test
+ public void shouldHandleSideEffectsCorrectly() throws Exception {
+ final Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
+ /// groupCount(m)-test
+ Traversal.Admin<Vertex, Long> traversal = g.V().groupCount("m").by(T.label).count().asAdmin();
+ test(SparkCountInterceptor.class, 6l, traversal);
+ assertEquals(1, traversal.getSideEffects().keys().size());
+ assertTrue(traversal.getSideEffects().exists("m"));
+ assertTrue(traversal.getSideEffects().keys().contains("m"));
+ final Map<String, Long> map = traversal.getSideEffects().get("m");
+ assertEquals(2, map.size());
+ assertEquals(2, map.get("software").intValue());
+ assertEquals(4, map.get("person").intValue());
+ }
+
+ @Test
public void shouldSuccessfullyEvaluateInterceptedTraversals() throws Exception {
final Configuration configuration = getBaseConfiguration();
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
[05/18] incubator-tinkerpop git commit: DefaultTraversal.hashCode()
was bad. Fixed. Added NativeInterceptor which right now is tied to
SparkGraphComputer,
but should be generalizable to all GraphComputer providers. In short,
instead of executing the Vert
Posted by ok...@apache.org.
DefaultTraversal.hashCode() was bad. Fixed. Added NativeInterceptor which right now is tied to SparkGraphComputer, but should be generalizable to all GraphComputer providers. In short, instead of executing the VertexProgram, an interceptor can take the Graph/Memory and pop out a Graph/Memory. Thus, skipping the VertexProgram execution. Right now I have VertexCountInterceptor which does inputRDD.count(). BAM. Works. Added more test cases and cleanup. Going to probably just roost on this branch for a bit until I get the concept clear and simple. Pushing so I can benchmark on the Blades.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/7c103c8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/7c103c8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/7c103c8b
Branch: refs/heads/master
Commit: 7c103c8b0bf218c5eb6ec83ccfe5d416fd671e3d
Parents: 6e92c4e
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 3 17:10:29 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 3 17:10:29 2016 -0600
----------------------------------------------------------------------
.../traversal/util/DefaultTraversal.java | 4 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 1 +
.../process/computer/NativeInterceptor.java | 32 +++++++
.../process/computer/SparkGraphComputer.java | 55 ++++++-----
.../optimization/SparkInterceptorStrategy.java | 60 ++++++++++++
.../SparkPartitionAwareStrategy.java | 2 +-
.../interceptors/VertexCountInterceptor.java | 52 +++++++++++
.../SparkInterceptorStrategyTest.java | 96 ++++++++++++++++++++
8 files changed, 278 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
index e7d3257..78db94a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
@@ -297,12 +297,12 @@ public class DefaultTraversal<S, E> implements Traversal.Admin<S, E> {
@Override
public boolean equals(final Object other) {
- return other != null && other.getClass().equals(this.getClass()) && this.equals(((Traversal.Admin) other));
+ return other != null && other instanceof Traversal.Admin && this.equals(((Traversal.Admin) other));
}
@Override
public int hashCode() {
- int result = this.getClass().hashCode();
+ int result = 1;
for (final Step step : this.asAdmin().getSteps()) {
result ^= step.hashCode();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index 872b3fb..ab45903 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -58,6 +58,7 @@ public final class Constants {
public static final String GREMLIN_SPARK_GRAPH_STORAGE_LEVEL = "gremlin.spark.graphStorageLevel";
public static final String GREMLIN_SPARK_PERSIST_STORAGE_LEVEL = "gremlin.spark.persistStorageLevel";
public static final String GREMLIN_SPARK_SKIP_PARTITIONER = "gremlin.spark.skipPartitioner";
+ public static final String GREMLIN_SPARK_NATIVE_INTERCEPTOR = "gremlin.spark.nativeInterceptor";
public static final String SPARK_SERIALIZER = "spark.serializer";
public static String getGraphLocation(final String location) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java
new file mode 100644
index 0000000..8448a1b
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/NativeInterceptor.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface NativeInterceptor<V extends VertexProgram> {
+
+ public JavaPairRDD<Object, VertexWritable> apply(final V vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index cd01779..49de96a 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -53,6 +53,7 @@ import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.SparkInterceptorStrategy;
import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.SparkPartitionAwareStrategy;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputFormatRDD;
@@ -83,7 +84,10 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
private boolean workersSet = false;
static {
- TraversalStrategies.GlobalCache.registerStrategies(SparkGraphComputer.class, TraversalStrategies.GlobalCache.getStrategies(GraphComputer.class).clone().addStrategies(SparkPartitionAwareStrategy.instance()));
+ TraversalStrategies.GlobalCache.registerStrategies(SparkGraphComputer.class,
+ TraversalStrategies.GlobalCache.getStrategies(GraphComputer.class).clone().addStrategies(
+ SparkPartitionAwareStrategy.instance(),
+ SparkInterceptorStrategy.instance()));
}
public SparkGraphComputer(final HadoopGraph hadoopGraph) {
@@ -228,7 +232,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
loadedGraphRDD = loadedGraphRDD.repartition(this.workers);
}
// persist the vertex program loaded graph as specified by configuration or else use default cache() which is MEMORY_ONLY
- if (!inputFromSpark || partitioned || filtered)
+ if (!skipPartitioner && (!inputFromSpark || partitioned || filtered))
loadedGraphRDD = loadedGraphRDD.persist(StorageLevel.fromString(hadoopConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_STORAGE_LEVEL, "MEMORY_ONLY")));
@@ -236,30 +240,39 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
// process the vertex program //
////////////////////////////////
if (null != this.vertexProgram) {
- // set up the vertex program and wire up configurations
- JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
this.vertexProgram.setup(memory);
- memory.broadcastMemory(sparkContext);
- final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
- this.vertexProgram.storeState(vertexProgramConfiguration);
- ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
- // execute the vertex program
- while (true) {
- memory.setInExecute(true);
- viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(loadedGraphRDD, viewIncomingRDD, memory, vertexProgramConfiguration);
- memory.setInExecute(false);
- if (this.vertexProgram.terminate(memory))
- break;
- else {
- memory.incrIteration();
- memory.broadcastMemory(sparkContext);
+ if (apacheConfiguration.containsKey(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR)) {
+ try {
+ final NativeInterceptor<VertexProgram> interceptor = (NativeInterceptor) Class.forName(apacheConfiguration.getString(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR)).newInstance();
+ computedGraphRDD = interceptor.apply(this.vertexProgram, loadedGraphRDD, memory);
+ } catch (final ClassNotFoundException | IllegalAccessException | InstantiationException e) {
+ throw new IllegalStateException(e.getMessage());
}
+ } else {
+ // set up the vertex program and wire up configurations
+ JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
+ memory.broadcastMemory(sparkContext);
+ final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
+ this.vertexProgram.storeState(vertexProgramConfiguration);
+ ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
+ ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
+ // execute the vertex program
+ while (true) {
+ memory.setInExecute(true);
+ viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(loadedGraphRDD, viewIncomingRDD, memory, vertexProgramConfiguration);
+ memory.setInExecute(false);
+ if (this.vertexProgram.terminate(memory))
+ break;
+ else {
+ memory.incrIteration();
+ memory.broadcastMemory(sparkContext);
+ }
+ }
+ // write the computed graph to the respective output (rdd or output format)
+ computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
}
memory.complete(); // drop all transient memory keys
- // write the computed graph to the respective output (rdd or output format)
- computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
if (null != outputRDD && !this.persist.equals(Persist.NOTHING)) {
outputRDD.writeGraphRDD(apacheConfiguration, computedGraphRDD);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java
new file mode 100644
index 0000000..2ad1ad6
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategy.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.interceptors.VertexCountInterceptor;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
+
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
+
+ private static final SparkInterceptorStrategy INSTANCE = new SparkInterceptorStrategy();
+
+ private SparkInterceptorStrategy() {
+ }
+
+ @Override
+ public void apply(final Traversal.Admin<?, ?> traversal) {
+ final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance());
+ final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
+ for (final TraversalVertexProgramStep step : steps) {
+ if (step.generateProgram(graph).getTraversal().equals(graph.traversal().V().count())) {
+ step.setComputer(step.getComputer().configure(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR, VertexCountInterceptor.class.getCanonicalName()));
+ }
+ }
+ }
+
+ public static SparkInterceptorStrategy instance() {
+ return INSTANCE;
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
index 02aa1ae..5c28d41 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkPartitionAwareStrategy.java
@@ -41,7 +41,7 @@ import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class SparkPartitionAwareStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
+public final class SparkPartitionAwareStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
private static final SparkPartitionAwareStrategy INSTANCE = new SparkPartitionAwareStrategy();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java
new file mode 100644
index 0000000..b685677
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/interceptors/VertexCountInterceptor.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.interceptors;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.spark.process.computer.NativeInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class VertexCountInterceptor implements NativeInterceptor<TraversalVertexProgram> {
+
+ public VertexCountInterceptor() {
+
+ }
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
+ final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal();
+ final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(inputRDD.count(), (Step) traversal.getEndStep(), 1l));
+ memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
+ memory.incrIteration();
+ return inputRDD;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7c103c8b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java
new file mode 100644
index 0000000..bb009eb
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/optimization/SparkInterceptorStrategyTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.optimization.interceptors.VertexCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkInterceptorStrategyTest extends AbstractSparkTest {
+
+ @Test
+ public void shouldUseVertexCountInterceptor() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ ///
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+ assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
+ //
+ assertEquals(6l, g.V().count().next().longValue());
+ assertEquals(2l, g.V().out().out().count().next().longValue());
+ }
+
+ @Test
+ public void shouldSetConfigurationsCorrectly() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer();
+
+ assertEquals(VertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().count()));
+ assertEquals(VertexCountInterceptor.class.getCanonicalName(), getInterceptor(g.V().identity().identity().count()));
+ assertNull(getInterceptor(g.V().out().count()));
+ assertNull(getInterceptor(g.V().as("a").count()));
+ }
+
+ private static String getInterceptor(final Traversal<?, ?> traversal) {
+ traversal.asAdmin().applyStrategies();
+ return (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
+ .getComputer()
+ .getConfiguration()
+ .getOrDefault(Constants.GREMLIN_SPARK_NATIVE_INTERCEPTOR, null);
+
+ }
+}
[17/18] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1288'
Posted by ok...@apache.org.
Merge branch 'TINKERPOP-1288'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/197f59c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/197f59c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/197f59c0
Branch: refs/heads/master
Commit: 197f59c080139dec2cd760bbb3942a76be79887c
Parents: 55172b4 3a8995b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri May 6 10:07:53 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri May 6 10:07:53 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphGraphComputer.java | 3 +
.../gremlin/process/computer/Computer.java | 75 ++++++--
.../traversal/TraversalVertexProgram.java | 10 +-
.../strategy/VertexProgramInterceptor.java | 31 ++++
.../process/traversal/TraversalSource.java | 10 +-
.../step/sideEffect/SackValueStep.java | 1 -
.../traversal/util/DefaultTraversal.java | 3 +-
.../process/traversal/util/TraversalHelper.java | 130 ++++++++++----
.../AdjacentToIncidentStrategyTest.java | 4 +
.../process/util/TraversalHelperTest.java | 36 ++++
.../traversal/step/branch/OptionalTest.java | 2 -
.../tinkerpop/gremlin/hadoop/Constants.java | 3 +
.../optimization/Neo4jGraphStepStrategy.java | 2 +-
.../spark/process/computer/SparkExecutor.java | 79 ++++----
.../process/computer/SparkGraphComputer.java | 118 +++++++-----
.../spark/process/computer/SparkMemory.java | 2 +-
.../strategy/SparkVertexProgramInterceptor.java | 32 ++++
.../optimization/SparkInterceptorStrategy.java | 63 +++++++
.../SparkSingleIterationStrategy.java | 88 +++++++++
.../SparkStarBarrierInterceptor.java | 178 +++++++++++++++++++
.../SparkInterceptorStrategyTest.java | 175 ++++++++++++++++++
.../SparkSingleIterationStrategyTest.java | 104 +++++++++++
.../io/PersistedInputOutputRDDTest.java | 46 ++++-
23 files changed, 1053 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
[04/18] incubator-tinkerpop git commit: merged in TINKERPOP-1120 as
that has all the new optimizations. going to test on the blades.
Posted by ok...@apache.org.
merged in TINKERPOP-1120 as that has all the new optimizations. going to test on the blades.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6e92c4e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6e92c4e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6e92c4e1
Branch: refs/heads/master
Commit: 6e92c4e115599a77381a34fdd94e19ce48866f37
Parents: 40b3931
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 3 14:53:14 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 3 14:53:14 2016 -0600
----------------------------------------------------------------------
.../spark/process/computer/SparkExecutor.java | 79 ++++++++++++--------
1 file changed, 46 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6e92c4e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index c216510..c2b85dd 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -21,6 +21,8 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import com.google.common.base.Optional;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
@@ -78,7 +80,9 @@ public final class SparkExecutor {
final SparkMemory memory,
final Configuration apacheConfiguration) {
- if (null != viewIncomingRDD) // the graphRDD and the viewRDD must have the same partitioner
+ boolean partitionedGraphRDD = graphRDD.partitioner().isPresent();
+
+ if (partitionedGraphRDD && null != viewIncomingRDD) // the graphRDD and the viewRDD must have the same partitioner
assert graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get());
final JavaPairRDD<Object, ViewOutgoingPayload<M>> viewOutgoingRDD = ((null == viewIncomingRDD) ?
graphRDD.mapValues(vertexWritable -> new Tuple2<>(vertexWritable, Optional.<ViewIncomingPayload<M>>absent())) : // first iteration will not have any views or messages
@@ -123,39 +127,47 @@ public final class SparkExecutor {
}, true) // true means that the partition is preserved
.filter(tuple -> null != tuple); // if there are no messages or views, then the tuple is null (memory optimization)
// the graphRDD and the viewRDD must have the same partitioner
- assert graphRDD.partitioner().get().equals(viewOutgoingRDD.partitioner().get());
- // "message pass" by reducing on the vertex object id of the view and message payloads
+ if (partitionedGraphRDD)
+ assert graphRDD.partitioner().get().equals(viewOutgoingRDD.partitioner().get());
+ /////////////////////////////////////////////////////////////
+ /////////////////////////////////////////////////////////////
+ final PairFlatMapFunction<Tuple2<Object, ViewOutgoingPayload<M>>, Object, Payload> messageFunction =
+ tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
+ IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())), // emit the view payload
+ IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))));
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().orElse(null);
- final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD = viewOutgoingRDD
- .flatMapToPair(tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
- // emit the view payload
- IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())),
- // emit the outgoing message payloads one by one
- IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2())))))
- .reduceByKey(graphRDD.partitioner().get(), (a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
- if (a instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
- return a;
- } else if (b instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
- return b;
- } else {
- final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
- c.mergePayload(a, messageCombiner);
- c.mergePayload(b, messageCombiner);
- return c;
- }
- })
- .mapValues(payload -> { // handle various corner cases of when views don't exist, messages don't exist, or neither exists.
- if (payload instanceof ViewIncomingPayload) // this happens if there is a vertex view with incoming messages
- return (ViewIncomingPayload<M>) payload;
- else if (payload instanceof ViewPayload) // this happens if there is a vertex view with no incoming messages
- return new ViewIncomingPayload<>((ViewPayload) payload);
- else // this happens when there is a single message to a vertex that has no view or outgoing messages
- return new ViewIncomingPayload<>((MessagePayload<M>) payload);
- });
+ final Function2<Payload, Payload, Payload> reducerFunction = (a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
+ if (a instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
+ return a;
+ } else if (b instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
+ return b;
+ } else {
+ final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
+ c.mergePayload(a, messageCombiner);
+ c.mergePayload(b, messageCombiner);
+ return c;
+ }
+ };
+ /////////////////////////////////////////////////////////////
+ /////////////////////////////////////////////////////////////
+ // "message pass" by reducing on the vertex object id of the view and message payloads
+ final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD =
+ (partitionedGraphRDD ?
+ viewOutgoingRDD.flatMapToPair(messageFunction).reduceByKey(graphRDD.partitioner().get(), reducerFunction) :
+ viewOutgoingRDD.flatMapToPair(messageFunction).reduceByKey(reducerFunction))
+ .mapValues(payload -> { // handle various corner cases of when views don't exist, messages don't exist, or neither exists.
+ if (payload instanceof ViewIncomingPayload) // this happens if there is a vertex view with incoming messages
+ return (ViewIncomingPayload<M>) payload;
+ else if (payload instanceof ViewPayload) // this happens if there is a vertex view with no incoming messages
+ return new ViewIncomingPayload<>((ViewPayload) payload);
+ else // this happens when there is a single message to a vertex that has no view or outgoing messages
+ return new ViewIncomingPayload<>((MessagePayload<M>) payload);
+ });
// the graphRDD and the viewRDD must have the same partitioner
- assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
+ if (partitionedGraphRDD)
+ assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
newViewIncomingRDD
.foreachPartition(partitionIterator -> {
HadoopPools.initialize(apacheConfiguration);
@@ -168,7 +180,8 @@ public final class SparkExecutor {
final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD,
final Set<VertexComputeKey> vertexComputeKeys) {
// the graphRDD and the viewRDD must have the same partitioner
- assert (graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get()));
+ if (graphRDD.partitioner().isPresent())
+ assert (graphRDD.partitioner().get().equals(viewIncomingRDD.partitioner().get()));
final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(vertexComputeKeys); // the compute keys as an array
return graphRDD.leftOuterJoin(viewIncomingRDD)
.mapValues(tuple -> {
[08/18] incubator-tinkerpop git commit: added test cases around
persisted of rdds. if the interceptor does not 'touch' the loadedGraphRDD
then its still the loadedGraphRDD and thus,
persistence rules for loadedGraphRDD should apply there.
Posted by ok...@apache.org.
added test cases around persisted of rdds. if the interceptor does not 'touch' the loadedGraphRDD then its still the loadedGraphRDD and thus, persistence rules for loadedGraphRDD should apply there.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2717b281
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2717b281
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2717b281
Branch: refs/heads/master
Commit: 2717b281ed085a365d59edc718db9a33a9266343
Parents: f0acfd7
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 09:44:03 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 09:44:03 2016 -0600
----------------------------------------------------------------------
.../process/computer/SparkGraphComputer.java | 10 ++---
.../io/PersistedInputOutputRDDTest.java | 46 ++++++++++++++++++--
2 files changed, 48 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2717b281/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 65e1c96..b7ec133 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -156,7 +156,6 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
final OutputRDD outputRDD;
final boolean filtered;
try {
-
inputRDD = InputRDD.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_READER, Object.class)) ?
hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_READER, InputRDD.class, InputRDD.class).newInstance() :
InputFormatRDD.class.newInstance();
@@ -243,7 +242,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
if (null != this.vertexProgram) {
memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
/////////////////
- // if there is a registered VertexProgramInterceptor, use it to by pass the GraphComputer semantics
+ // if there is a registered VertexProgramInterceptor, use it to bypass the GraphComputer semantics
if (apacheConfiguration.containsKey(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR)) {
try {
final SparkVertexProgramInterceptor<VertexProgram> interceptor =
@@ -322,11 +321,12 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
}
// unpersist the loaded graph if it will not be used again (no PersistedInputRDD)
- // if the graphRDD was loaded from Spark, but then partitioned, its a different RDD
- if ((!skipPartitioner && (!inputFromSpark || partitioned || filtered)) && computedGraphCreated)
+ // if the graphRDD was loaded from Spark, but then partitioned or filtered, its a different RDD
+ if ((!inputFromSpark || partitioned || filtered) && computedGraphCreated)
loadedGraphRDD.unpersist();
// unpersist the computed graph if it will not be used again (no PersistedOutputRDD)
- if (!outputToSpark || this.persist.equals(GraphComputer.Persist.NOTHING))
+ // if the computed graph is the loadedGraphRDD because it was not mutated and not-unpersisted, then don't unpersist the computedGraphRDD/loadedGraphRDD
+ if (!outputToSpark || (this.persist.equals(GraphComputer.Persist.NOTHING) && loadedGraphRDD != computedGraphRDD))
computedGraphRDD.unpersist();
// delete any file system or rdd data if persist nothing
if (null != outputLocation && this.persist.equals(GraphComputer.Persist.NOTHING)) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2717b281/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
index 3f72093..7a089b6 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
@@ -66,15 +66,18 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
Graph graph = GraphFactory.open(configuration);
+ ///
assertEquals(6, graph.traversal().withComputer(Computer.compute(SparkGraphComputer.class)).V().out().count().next().longValue());
- ////////
+ assertFalse(Spark.hasRDD(Constants.getGraphLocation(rddName)));
+ assertEquals(0, Spark.getContext().getPersistentRDDs().size());
+ //
+ assertEquals(2, graph.traversal().withComputer(Computer.compute(SparkGraphComputer.class)).V().out().out().count().next().longValue());
assertFalse(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertEquals(0, Spark.getContext().getPersistentRDDs().size());
///////
Spark.close();
}
-
@Test
public void shouldPersistRDDBasedOnStorageLevel() throws Exception {
Spark.create("local[4]");
@@ -104,7 +107,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(StorageLevel.fromString(storageLevel), Spark.getRDD(Constants.getGraphLocation(rddName)).getStorageLevel());
assertEquals(counter, Spark.getRDDs().size());
assertEquals(counter, Spark.getContext().getPersistentRDDs().size());
- //System.out.println(SparkContextStorage.open().ls());
}
Spark.close();
}
@@ -164,6 +166,44 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(6, graph.traversal().withComputer(SparkGraphComputer.class).V().out().count().next().longValue());
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertEquals(1, Spark.getContext().getPersistentRDDs().size());
+ ///////
+ graph = GraphFactory.open(configuration);
+ graph.compute(SparkGraphComputer.class)
+ .result(GraphComputer.ResultGraph.NEW)
+ .persist(GraphComputer.Persist.EDGES)
+ .program(TraversalVertexProgram.build()
+ .traversal(graph.traversal().withComputer(SparkGraphComputer.class),
+ "gremlin-groovy",
+ "g.V().count()").create(graph)).submit().get();
+ assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
+ assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName2)));
+ assertEquals(2, Spark.getContext().getPersistentRDDs().size());
+ ///////
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, PersistedInputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, rddName);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName2);
+ graph = GraphFactory.open(configuration);
+ assertEquals(6, graph.traversal().withComputer(SparkGraphComputer.class).V().out().count().next().longValue());
+ assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
+ assertEquals(1, Spark.getContext().getPersistentRDDs().size());
+ ///////
+ graph = GraphFactory.open(configuration);
+ graph.compute(SparkGraphComputer.class)
+ .result(GraphComputer.ResultGraph.NEW)
+ .persist(GraphComputer.Persist.EDGES)
+ .program(TraversalVertexProgram.build()
+ .traversal(graph.traversal().withComputer(SparkGraphComputer.class),
+ "gremlin-groovy",
+ "g.V().count()").create(graph)).submit().get();
+ assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
+ assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName2)));
+ assertEquals(2, Spark.getContext().getPersistentRDDs().size());
+ ///////
+ graph = GraphFactory.open(configuration);
+ assertEquals(6, graph.traversal().withComputer(SparkGraphComputer.class).V().out().count().next().longValue());
+ assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
+ assertEquals(1, Spark.getContext().getPersistentRDDs().size());
Spark.close();
}
[16/18] incubator-tinkerpop git commit: got fold() and group()
working with SparkStarBarrierInterceptor. Ensured that
SparkStarBarrierInterceptor doesn't cache or partition the inputRDD. Added
some logic to ensure that such configurations are propagated
Posted by ok...@apache.org.
got fold() and group() working with SparkStarBarrierInterceptor. Ensured that SparkStarBarrierInterceptor doesn't cache or partition the inputRDD. Added some logic to ensure that such configurations are propagated through chained OLAP jobs. Also, reverted some work I did on DefaultTraversal. I will raise another issue for the (minor) problem. Integration tests pass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/3a8995b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/3a8995b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/3a8995b0
Branch: refs/heads/master
Commit: 3a8995b0ef54fd2bfa10dddd879d1a2de2a23636
Parents: 1e6aa16
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu May 5 14:15:01 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu May 5 14:15:01 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphGraphComputer.java | 3 ++
.../traversal/util/DefaultTraversal.java | 7 ++-
.../process/computer/SparkGraphComputer.java | 5 ++
.../optimization/SparkInterceptorStrategy.java | 8 ++-
.../SparkStarBarrierInterceptor.java | 56 ++++++++++++++------
.../SparkInterceptorStrategyTest.java | 32 ++++++++---
6 files changed, 82 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3a8995b0/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index 3ed8618..6bd2a04 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -136,6 +136,9 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
throw new IllegalStateException(e.getMessage(), e);
}
this.memory.setRuntime(System.currentTimeMillis() - startTime);
+ // clear properties that should not be propagated in an OLAP chain
+ apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_GRAPH_FILTER);
+ apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR);
return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), this.memory.asImmutable());
}, exec);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3a8995b0/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
index 78db94a..5ea5668 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversal.java
@@ -297,16 +297,15 @@ public class DefaultTraversal<S, E> implements Traversal.Admin<S, E> {
@Override
public boolean equals(final Object other) {
- return other != null && other instanceof Traversal.Admin && this.equals(((Traversal.Admin) other));
+ return other != null && other.getClass().equals(this.getClass()) && this.equals(((Traversal.Admin) other));
}
@Override
public int hashCode() {
- int result = 1;
+ int result = this.getClass().hashCode();
for (final Step step : this.asAdmin().getSteps()) {
result ^= step.hashCode();
}
return result;
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3a8995b0/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 4b91a10..5bc040f 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -354,6 +354,11 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
}
// update runtime and return the newly computed graph
finalMemory.setRuntime(System.currentTimeMillis() - startTime);
+ // clear properties that should not be propagated in an OLAP chain
+ apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_GRAPH_FILTER);
+ apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR);
+ apacheConfiguration.clearProperty(Constants.GREMLIN_SPARK_SKIP_GRAPH_CACHE);
+ apacheConfiguration.clearProperty(Constants.GREMLIN_SPARK_SKIP_PARTITIONER);
return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), finalMemory.asImmutable());
} finally {
if (!apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3a8995b0/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index 14a23e2..19d21b3 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -46,8 +46,12 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
- if (SparkStarBarrierInterceptor.isLegal(computerTraversal))
- step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkStarBarrierInterceptor.class.getCanonicalName()));
+ if (SparkStarBarrierInterceptor.isLegal(computerTraversal)) {
+ step.setComputer(step.getComputer()
+ .configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true)
+ .configure(Constants.GREMLIN_SPARK_SKIP_GRAPH_CACHE, true)
+ .configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkStarBarrierInterceptor.class.getCanonicalName()));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3a8995b0/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
index 76272be..070f3b2 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
@@ -31,8 +31,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.FoldStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupCountStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MaxGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MeanGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MinGlobalStep;
@@ -41,16 +43,17 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
-import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
+import org.apache.tinkerpop.gremlin.util.function.ArrayListSupplier;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BinaryOperator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -67,10 +70,8 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
final Traversal.Admin<Vertex, Object> traversal = (Traversal.Admin) vertexProgram.getTraversal().getPure().clone();
final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds(); // any V(1,2,3)-style ids to filter on
final ReducingBarrierStep endStep = (ReducingBarrierStep) traversal.getEndStep(); // needed for the final traverser generation
- final Traversal.Admin<Object, Object> endStepTraversal = new DefaultTraversal<>();
- endStepTraversal.addStep(endStep).applyStrategies();
traversal.removeStep(0); // remove GraphStep
- traversal.removeStep(traversal.getSteps().size() - 1); // remove CountGlobalStep
+ traversal.removeStep(traversal.getSteps().size() - 1); // remove ReducingBarrierStep
traversal.applyStrategies(); // compile
boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
///////////////////////////////
@@ -89,6 +90,7 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
};
}
});
+ // USE SPARK DSL FOR THE RESPECTIVE END REDUCING BARRIER STEP OF THE TRAVERSAL
final Object result;
if (endStep instanceof CountGlobalStep)
result = nextRDD.map(Traverser::bulk).fold(0l, (a, b) -> a + b);
@@ -109,23 +111,45 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
result = nextRDD
.map(traverser -> (Number) traverser.get())
.fold(Integer.MIN_VALUE, NumberHelper::max);
- else if (endStep instanceof GroupCountStep)
+ else if (endStep instanceof FoldStep) {
+ final BinaryOperator biOperator = endStep.getBiOperator();
+ result = nextRDD.map(traverser -> {
+ if (endStep.getSeedSupplier() instanceof ArrayListSupplier) {
+ final List list = new ArrayList<>();
+ for (long i = 0; i < traverser.bulk(); i++) {
+ list.add(traverser.get());
+ }
+ return list;
+ } else {
+ return traverser.get();
+ }
+ }).fold(endStep.getSeedSupplier().get(), biOperator::apply);
+ } else if (endStep instanceof GroupStep) {
+ ((GroupStep) endStep).onGraphComputer();
+ final GroupStep.GroupBiOperator<Object, Object> biOperator = (GroupStep.GroupBiOperator) endStep.getBiOperator();
+ result = ((GroupStep) endStep).generateFinalResult(nextRDD.
+ mapPartitions(partitions -> {
+ final GroupStep<Object, Object, Object> clone = (GroupStep) endStep.clone();
+ return () -> IteratorUtils.map(partitions, clone::projectTraverser);
+ }).fold(((GroupStep<Object, Object, Object>) endStep).getSeedSupplier().get(), biOperator::apply));
+ } else if (endStep instanceof GroupCountStep) {
+ final GroupCountStep.GroupCountBiOperator<Object> biOperator = GroupCountStep.GroupCountBiOperator.instance();
result = nextRDD
- .mapPartitions(partition -> {
- final Traversal.Admin<?, Map<?, Long>> clone = (Traversal.Admin) endStepTraversal.clone();
- return () -> IteratorUtils.map(partition, traverser -> TraversalUtil.apply((Traverser.Admin) traverser, clone));
+ .mapPartitions(partitions -> {
+ final GroupCountStep<Object, Object> clone = (GroupCountStep) endStep.clone();
+ return () -> IteratorUtils.map(partitions, clone::projectTraverser);
})
- .fold(((GroupCountStep<?, ?>) endStep).getSeedSupplier().get(), (a, b) -> GroupCountStep.GroupCountBiOperator.instance().apply((Map) a, (Map) b));
- else
+ .fold(((GroupCountStep<Object, Object>) endStep).getSeedSupplier().get(), biOperator::apply);
+ } else
throw new IllegalArgumentException("The end step is an unsupported barrier: " + endStep);
memory.setInExecute(false);
///////////////////////////////
// generate the HALTED_TRAVERSERS for the memory
final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
- haltedTraversers.add(traversal.getTraverserGenerator().generate(result, endStep, 1l));
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(result, endStep, 1l)); // all reducing barrier steps produce a result of bulk 1
memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
- memory.incrIteration(); // any local star graph reduction take a single iteration
+ memory.incrIteration(); // any local star graph reduction takes a single iteration
return inputRDD;
}
@@ -139,8 +163,10 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
!endStep.getClass().equals(MeanGlobalStep.class) &&
!endStep.getClass().equals(MaxGlobalStep.class) &&
!endStep.getClass().equals(MinGlobalStep.class) &&
+ !endStep.getClass().equals(FoldStep.class) &&
+ !endStep.getClass().equals(GroupStep.class) &&
!endStep.getClass().equals(GroupCountStep.class))
- // TODO: group(), fold(), and tree()
+ // TODO: tree()
return false;
if (TraversalHelper.getStepsOfAssignableClassRecursively(Scope.global, Barrier.class, traversal).size() != 1)
return false;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3a8995b0/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index a43f081..24e3663 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.VertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
@@ -47,6 +48,7 @@ import java.util.Map;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -66,7 +68,9 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
///
Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
+ GraphTraversalSource g = graph.traversal().withComputer().withoutStrategies(SparkSingleIterationStrategy.class);
+ assertFalse(g.getStrategies().toList().contains(SparkSingleIterationStrategy.instance()));
+ assertFalse(g.V().count().explain().toString().contains(SparkSingleIterationStrategy.class.getSimpleName()));
assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
/// groupCount(m)-test
@@ -92,7 +96,9 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
///
Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
+ GraphTraversalSource g = graph.traversal().withComputer().withoutStrategies(SparkSingleIterationStrategy.class);
+ assertFalse(g.getStrategies().toList().contains(SparkSingleIterationStrategy.instance()));
+ assertFalse(g.V().count().explain().toString().contains(SparkSingleIterationStrategy.class.getSimpleName()));
assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
/// SparkCountInterceptor matches
@@ -124,6 +130,11 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
put("person", 4l);
}}, g.V().<String>groupCount().by(T.label));
test(SparkStarBarrierInterceptor.class, Collections.singletonMap("person", 2l), g.V().has("person", "age", P.lt(30)).<String>groupCount().by(T.label));
+ test(SparkStarBarrierInterceptor.class, new HashMap<String, Long>() {{
+ put("software", 2l);
+ put("person", 4l);
+ }}, g.V().<String, Long>group().by(T.label).by(__.count()));
+ test(SparkStarBarrierInterceptor.class, 123l, g.V().hasLabel("person").values("age").fold(0l, Operator.sum));
/// No interceptor matches
test(2l, g.V().out().out().count());
test(6l, g.E().count());
@@ -142,14 +153,19 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
private static <R> void test(Class<? extends VertexProgramInterceptor> expectedInterceptor, R expectedResult, final Traversal<?, R> traversal) throws Exception {
final Traversal.Admin<?, ?> clone = traversal.asAdmin().clone();
clone.applyStrategies();
- final String interceptor = (String) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, clone).get()
- .getComputer()
- .getConfiguration()
- .getOrDefault(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, null);
- if (null == expectedInterceptor)
+ final Map<String, Object> configuration = TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, clone).get().getComputer().getConfiguration();
+ final String interceptor = (String) configuration.getOrDefault(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, null);
+ if (null == expectedInterceptor) {
assertNull(interceptor);
- else
+ assertFalse((Boolean) configuration.getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false));
+ assertFalse((Boolean) configuration.getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false));
+ } else {
assertEquals(expectedInterceptor, Class.forName(interceptor));
+ if (interceptor.equals(SparkStarBarrierInterceptor.class.getCanonicalName())) {
+ assertTrue((Boolean) configuration.getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false));
+ assertTrue((Boolean) configuration.getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false));
+ }
+ }
assertEquals(expectedResult, traversal.next());
}
[14/18] incubator-tinkerpop git commit: found severe issues with
TraversalHelper.isLocalStarGraph(). Given that this is used extensively by
GraphComputer implementations, this was a good find. Crazy enough,
no test cases for that method. Added a host of
Posted by ok...@apache.org.
found severe issues with TraversalHelper.isLocalStarGraph(). Given that this is used extensively by GraphComputer implementations, this was a good find. Crazy enough, no test cases for that method. Added a host of them for traverals of varying complexity. Added spark.skipPartitioner and spark.skipGraphCache which are both current modulated by SparkSingleIterationStrategy. Opened up some IgnoreEngines from @pietermartin that we due to bad TraversalHelper.isLocalStarGraph(). More tests cases for the new SparkXXXStrategy work. Lots of logic cleanup in SparkGraphComputer given the distinction between skipPartiioner and skipGraphCache.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/854b259c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/854b259c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/854b259c
Branch: refs/heads/master
Commit: 854b259cb0645d1d50e5365392b49a373e2131a5
Parents: 6e122e4
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu May 5 09:54:41 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu May 5 09:54:41 2016 -0600
----------------------------------------------------------------------
.../process/traversal/util/TraversalHelper.java | 53 +++++----
.../process/util/TraversalHelperTest.java | 36 ++++++
.../traversal/step/branch/OptionalTest.java | 2 -
.../tinkerpop/gremlin/hadoop/Constants.java | 3 +-
.../process/computer/SparkGraphComputer.java | 27 ++---
.../optimization/SparkInterceptorStrategy.java | 7 +-
.../SparkPartitionAwareStrategy.java | 88 ---------------
.../SparkSingleIterationStrategy.java | 88 +++++++++++++++
.../SparkInterceptorStrategyTest.java | 4 +-
.../SparkPartitionAwareStrategyTest.java | 113 -------------------
.../SparkSingleIterationStrategyTest.java | 104 +++++++++++++++++
11 files changed, 282 insertions(+), 243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
index 09ea337..9a98e54 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalHelper.java
@@ -24,6 +24,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.lambda.ElementValueTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.lambda.TokenTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.ByModulating;
import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder;
import org.apache.tinkerpop.gremlin.process.traversal.step.Scoping;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
@@ -33,8 +34,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.filter.NotStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WherePredicateStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTraversalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LabelStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyMapStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
@@ -84,41 +87,53 @@ public final class TraversalHelper {
}
public static boolean isLocalStarGraph(final Traversal.Admin<?, ?> traversal) {
- return isLocalStarGraph(traversal, 'v');
+ return 'x' != isLocalStarGraph(traversal, 'v');
}
- private static boolean isLocalStarGraph(final Traversal.Admin<?, ?> traversal, char state) {
+ private static char isLocalStarGraph(final Traversal.Admin<?, ?> traversal, char state) {
if (state == 'u' &&
(traversal instanceof ElementValueTraversal ||
(traversal instanceof TokenTraversal && !((TokenTraversal) traversal).getToken().equals(T.id))))
- return false;
+ return 'x';
for (final Step step : traversal.getSteps()) {
- if (step instanceof RepeatStep &&
- ((RepeatStep<?>) step).getGlobalChildren().stream()
- .flatMap(t -> t.getSteps().stream())
- .filter(s -> s instanceof VertexStep)
- .findAny()
- .isPresent()) // TODO: is this sufficient?
- return false;
- else if (step instanceof PropertiesStep && state == 'u')
- return false;
+ if ((step instanceof PropertiesStep || step instanceof LabelStep || step instanceof PropertyMapStep) && state == 'u')
+ return 'x';
else if (step instanceof VertexStep) {
- if (state == 'u') return false;
+ if (state == 'u') return 'x';
state = ((VertexStep) step).returnsVertex() ? 'u' : 'e';
} else if (step instanceof EdgeVertexStep) {
state = 'u';
} else if (step instanceof HasContainerHolder && state == 'u') {
if (((HasContainerHolder) step).getHasContainers().stream()
- .filter(c -> !c.getKey().equals(T.id.getAccessor())) // TODO: are labels available?
- .findAny().isPresent()) return false;
+ .filter(c -> !c.getKey().equals(T.id.getAccessor()))
+ .findAny().isPresent()) return 'x';
} else if (step instanceof TraversalParent) {
final char currState = state;
- if (((TraversalParent) step).getLocalChildren().stream()
- .filter(t -> !isLocalStarGraph(t.asAdmin(), currState))
- .findAny().isPresent()) return false;
+ Set<Character> states = ((TraversalParent) step).getLocalChildren().stream()
+ .map(t -> isLocalStarGraph(t.asAdmin(), currState))
+ .collect(Collectors.toSet());
+ if (states.contains('x'))
+ return 'x';
+ else if (!(step instanceof ByModulating)) {
+ if (states.contains('u'))
+ state = 'u';
+ else if (states.contains('e'))
+ state = 'e';
+ }
+ states = ((TraversalParent) step).getGlobalChildren().stream()
+ .map(t -> isLocalStarGraph(t.asAdmin(), currState))
+ .collect(Collectors.toSet());
+ if (states.contains('x'))
+ return 'x';
+ else if (states.contains('u'))
+ state = 'u';
+ else if (states.contains('e'))
+ state = 'e';
+ if (state != currState && step instanceof RepeatStep)
+ return 'x';
}
}
- return true;
+ return state;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/util/TraversalHelperTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/util/TraversalHelperTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/util/TraversalHelperTest.java
index 6d3577c..8505215 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/util/TraversalHelperTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/util/TraversalHelperTest.java
@@ -39,10 +39,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.structure.PropertyType;
+import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.out;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -290,4 +292,38 @@ public class TraversalHelperTest {
Mockito.when(s.toString()).thenReturn("0123456789");
assertEquals("0123...", TraversalHelper.getShortName(s, 7));
}
+
+ @Test
+ public void shouldIdentifyStarGraphTraversals() {
+ assertTrue(TraversalHelper.isLocalStarGraph(__.identity().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.id().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.out().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.label().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.bothE().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.values().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.properties().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.repeat(__.identity()).asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.repeat(__.has("name")).asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.out().repeat(__.identity()).asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.out().id().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.label().union(__.out(), __.in()).asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.label().union(__.out(), __.in()).id().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.coalesce(out("likes"), out("knows"), out("created")).groupCount().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.local(__.out()).groupCount().asAdmin()));
+ assertTrue(TraversalHelper.isLocalStarGraph(__.local(__.out()).groupCount().by(T.id).asAdmin()));
+ // assertTrue(TraversalHelper.isLocalStarGraph(__.out().repeat(__.has("name")).asAdmin()));
+ //
+ assertFalse(TraversalHelper.isLocalStarGraph(__.out().label().asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.out().values().asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.out().valueMap().asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.repeat(__.out()).asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.repeat(__.has("name").out()).asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.repeat(__.has("name").union(__.out(), __.in())).asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.union(__.out(), __.in()).label().asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.union(__.out(), __.in().out()).asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.union(__.out(), __.out().union(__.in(), __.out())).asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.union(__.values(), __.out().union(__.in(), __.out())).out().asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.coalesce(out("likes"), out("knows"), out("created")).groupCount().by("name").asAdmin()));
+ assertFalse(TraversalHelper.isLocalStarGraph(__.local(__.out()).groupCount().by("name").asAdmin()));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/OptionalTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/OptionalTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/OptionalTest.java
index fbb2816..660af6c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/OptionalTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/OptionalTest.java
@@ -74,7 +74,6 @@ public abstract class OptionalTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_hasLabelXpersonX_optionalXoutXknowsX_optionalXoutXcreatedXXX_path() {
final Traversal<Vertex, Path> traversal = get_g_V_hasLabelXpersonX_optionalXoutXknowsX_optionalXoutXcreatedXXX_path();
printTraversalForm(traversal);
@@ -98,7 +97,6 @@ public abstract class OptionalTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_optionalXout_optionalXoutXX_path() {
final Traversal<Vertex, Path> traversal = get_g_V_optionalXout_optionalXoutXX_path();
printTraversalForm(traversal);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index fb4f15a..9c79b53 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -58,7 +58,8 @@ public final class Constants {
public static final String GREMLIN_SPARK_PERSIST_CONTEXT = "gremlin.spark.persistContext";
public static final String GREMLIN_SPARK_GRAPH_STORAGE_LEVEL = "gremlin.spark.graphStorageLevel";
public static final String GREMLIN_SPARK_PERSIST_STORAGE_LEVEL = "gremlin.spark.persistStorageLevel";
- public static final String GREMLIN_SPARK_SKIP_PARTITIONER = "gremlin.spark.skipPartitioner";
+ public static final String GREMLIN_SPARK_SKIP_PARTITIONER = "gremlin.spark.skipPartitioner"; // don't partition the loadedGraphRDD
+ public static final String GREMLIN_SPARK_SKIP_GRAPH_CACHE = "gremlin.spark.skipGraphCache"; // don't cache the loadedGraphRDD (ignores graphStorageLevel)
public static final String SPARK_SERIALIZER = "spark.serializer";
public static String getGraphLocation(final String location) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 82b3369..4b91a10 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -57,7 +57,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedE
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkInterceptorStrategy;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkPartitionAwareStrategy;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.SparkSingleIterationStrategy;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputFormatRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputOutputHelper;
@@ -73,7 +73,6 @@ import org.apache.tinkerpop.gremlin.structure.io.Storage;
import java.io.File;
import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -99,7 +98,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
static {
TraversalStrategies.GlobalCache.registerStrategies(SparkGraphComputer.class,
TraversalStrategies.GlobalCache.getStrategies(GraphComputer.class).clone().addStrategies(
- SparkPartitionAwareStrategy.instance(),
+ SparkSingleIterationStrategy.instance(),
SparkInterceptorStrategy.instance()));
}
@@ -148,6 +147,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
final boolean outputToHDFS = FileOutputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, Object.class));
final boolean outputToSpark = PersistedOutputRDD.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, Object.class));
final boolean skipPartitioner = apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false);
+ final boolean skipPersist = apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_SKIP_GRAPH_CACHE, false);
String inputLocation = null;
if (inputFromSpark)
inputLocation = Constants.getSearchGraphLocation(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION), sparkContextStorage).orElse(null);
@@ -213,7 +213,6 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
Spark.create(sparkContext.sc()); // this is the context RDD holder that prevents GC
updateLocalConfiguration(sparkContext, sparkConfiguration);
// create a message-passing friendly rdd from the input rdd
- JavaPairRDD<Object, VertexWritable> computedGraphRDD = null;
boolean partitioned = false;
JavaPairRDD<Object, VertexWritable> loadedGraphRDD = inputRDD.readGraphRDD(apacheConfiguration, sparkContext);
// if there are vertex or edge filters, filter the loaded graph rdd prior to partitioning and persisting
@@ -244,10 +243,11 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
loadedGraphRDD = loadedGraphRDD.repartition(this.workers);
}
// persist the vertex program loaded graph as specified by configuration or else use default cache() which is MEMORY_ONLY
- if (!skipPartitioner && (!inputFromSpark || partitioned || filtered))
+ if (!skipPersist && (!inputFromSpark || partitioned || filtered))
loadedGraphRDD = loadedGraphRDD.persist(StorageLevel.fromString(hadoopConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_STORAGE_LEVEL, "MEMORY_ONLY")));
-
+ // final graph with view (for persisting and/or mapReducing -- may be null and thus, possible to save space/time)
+ JavaPairRDD<Object, VertexWritable> computedGraphRDD = null;
////////////////////////////////
// process the vertex program //
////////////////////////////////
@@ -288,17 +288,18 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
memory.broadcastMemory(sparkContext);
}
}
- // write the computed graph to the respective output (rdd or output format)
- computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
+ // if the graph will be continued to be used (persisted or mapreduced), then generate a view+graph
+ if ((null != outputRDD && !this.persist.equals(Persist.NOTHING)) || !this.mapReducers.isEmpty())
+ computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
}
/////////////////
memory.complete(); // drop all transient memory keys
- if (null != outputRDD && !this.persist.equals(Persist.NOTHING)) {
+ // write the computed graph to the respective output (rdd or output format)
+ if (null != outputRDD && !this.persist.equals(Persist.NOTHING))
outputRDD.writeGraphRDD(apacheConfiguration, computedGraphRDD);
- }
}
- final boolean computedGraphCreated = computedGraphRDD != null;
+ final boolean computedGraphCreated = computedGraphRDD != null && computedGraphRDD != loadedGraphRDD;
if (!computedGraphCreated)
computedGraphRDD = loadedGraphRDD;
@@ -338,11 +339,11 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
// unpersist the loaded graph if it will not be used again (no PersistedInputRDD)
// if the graphRDD was loaded from Spark, but then partitioned or filtered, its a different RDD
- if ((!inputFromSpark || partitioned || filtered) && computedGraphCreated)
+ if (!inputFromSpark || partitioned || filtered)
loadedGraphRDD.unpersist();
// unpersist the computed graph if it will not be used again (no PersistedOutputRDD)
// if the computed graph is the loadedGraphRDD because it was not mutated and not-unpersisted, then don't unpersist the computedGraphRDD/loadedGraphRDD
- if (!outputToSpark || (this.persist.equals(GraphComputer.Persist.NOTHING) && loadedGraphRDD != computedGraphRDD))
+ if ((!outputToSpark || this.persist.equals(GraphComputer.Persist.NOTHING)) && computedGraphCreated)
computedGraphRDD.unpersist();
// delete any file system or rdd data if persist nothing
if (null != outputLocation && this.persist.equals(GraphComputer.Persist.NOTHING)) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index 11c03f7..14a23e2 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -29,8 +29,6 @@ import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.op
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
-import java.util.List;
-
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -43,9 +41,8 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
- final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance());
- final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
- for (final TraversalVertexProgramStep step : steps) {
+ final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined
+ for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {
final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
deleted file mode 100644
index a7380c4..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaFlatMapStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaMapStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkPartitionAwareStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
-
- private static final SparkPartitionAwareStrategy INSTANCE = new SparkPartitionAwareStrategy();
-
- private static final Set<Class<? extends Step>> MESSAGEPASS_CLASSES = new HashSet<>(Arrays.asList(
- EdgeVertexStep.class,
- LambdaMapStep.class, // maybe?
- LambdaFlatMapStep.class // maybe?
- // VertexStep is special as you need to see if the return class is Edge or Vertex (logic below)
- ));
-
- private SparkPartitionAwareStrategy() {
- }
-
- @Override
- public void apply(final Traversal.Admin<?, ?> traversal) {
- final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
- for (final TraversalVertexProgramStep step : steps) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(traversal.getGraph().orElse(EmptyGraph.instance())).getTraversal().get().clone();
- if (!computerTraversal.isLocked())
- computerTraversal.applyStrategies();
- boolean messagePasses = MESSAGEPASS_CLASSES.stream()
- .flatMap(clazz -> TraversalHelper.<Step<?, ?>>getStepsOfAssignableClassRecursively((Class) clazz, computerTraversal).stream())
- .filter(s -> TraversalHelper.isGlobalChild(((Step) s).getTraversal().asAdmin()))
- .findAny()
- .isPresent();
- if (!messagePasses) {
- for (final VertexStep vertexStep : TraversalHelper.getStepsOfAssignableClassRecursively(VertexStep.class, computerTraversal)) {
- if (TraversalHelper.isGlobalChild(vertexStep.getTraversal()) &&
- (vertexStep.returnsVertex() || !vertexStep.getDirection().equals(Direction.OUT))) { // in edges require message pass in OLAP
- messagePasses = true;
- break;
- }
- }
- }
- if (!messagePasses) // if no message passing, don't partition (save time)
- step.setComputer(step.getComputer().configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true));
- }
- }
-
- public static SparkPartitionAwareStrategy instance() {
- return INSTANCE;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
new file mode 100644
index 0000000..153282e
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaFlatMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.LambdaMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkSingleIterationStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> implements TraversalStrategy.ProviderOptimizationStrategy {
+
+ private static final SparkSingleIterationStrategy INSTANCE = new SparkSingleIterationStrategy();
+
+ private static final Set<Class> MULTI_ITERATION_CLASSES = new HashSet<>(Arrays.asList(
+ EdgeVertexStep.class,
+ LambdaMapStep.class, // maybe?
+ LambdaFlatMapStep.class // maybe?
+ // VertexStep is special as you need to see if the return class is Edge or Vertex (logic below)
+ ));
+
+ private SparkSingleIterationStrategy() {
+ }
+
+ @Override
+ public void apply(final Traversal.Admin<?, ?> traversal) {
+ final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined
+ for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
+ if (!computerTraversal.isLocked())
+ computerTraversal.applyStrategies();
+ boolean doesMessagePass = TraversalHelper.hasStepOfAssignableClassRecursively(Scope.global, MULTI_ITERATION_CLASSES, computerTraversal);
+ if (!doesMessagePass) {
+ for (final VertexStep vertexStep : TraversalHelper.getStepsOfAssignableClassRecursively(Scope.global, VertexStep.class, computerTraversal)) {
+ if (vertexStep.returnsVertex() || !vertexStep.getDirection().equals(Direction.OUT)) { // in-edges require message pass in OLAP
+ doesMessagePass = true;
+ break;
+ }
+ }
+ }
+ if (!doesMessagePass) {
+ step.setComputer(step.getComputer()
+ // if no message passing, don't partition the loaded graph
+ .configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true)
+ // if no message passing, don't cache the loaded graph
+ .configure(Constants.GREMLIN_SPARK_SKIP_GRAPH_CACHE, true));
+ }
+ }
+ }
+
+ public static SparkSingleIterationStrategy instance() {
+ return INSTANCE;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index b48a2bb..a43f081 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -61,7 +61,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString()));
configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
///
@@ -87,7 +87,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString()));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString()));
configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
///
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java
deleted file mode 100644
index 045f424..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategyTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
-import org.junit.Test;
-
-import java.util.UUID;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class SparkPartitionAwareStrategyTest extends AbstractSparkTest {
-
- @Test
- public void shouldSuccessfullyEvaluateSkipPartitionedTraversals() throws Exception {
- final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
- Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
- configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
- ///
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
- assertTrue(g.getStrategies().toList().contains(SparkPartitionAwareStrategy.instance()));
- assertTrue(g.V().count().explain().toString().contains(SparkPartitionAwareStrategy.class.getSimpleName()));
- //
- assertEquals(6l, g.V().count().next().longValue());
- assertEquals(2l, g.V().out().out().count().next().longValue());
- }
-
-
- @Test
- public void shouldSkipPartitionExceptedTraversals() throws Exception {
- final String outputLocation = TestHelper.makeTestDataDirectory(SparkPartitionAwareStrategyTest.class, UUID.randomUUID().toString());
- Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
- configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
-
- Graph graph = GraphFactory.open(configuration);
- GraphTraversalSource g = graph.traversal().withComputer();
-
- assertTrue(skipPartitioner(g.V().limit(10)));
- assertTrue(skipPartitioner(g.V().values("age").groupCount()));
- assertTrue(skipPartitioner(g.V().groupCount().by(__.out().count())));
- assertTrue(skipPartitioner(g.V().outE()));
- assertTrue(skipPartitioner(g.V().count()));
- assertTrue(skipPartitioner(g.V().out().count()));
- assertTrue(skipPartitioner(g.V().local(__.inE()).count()));
- assertTrue(skipPartitioner(g.V().outE().inV().count()));
- ////
- assertFalse(skipPartitioner(g.V().outE().inV()));
- assertFalse(skipPartitioner(g.V().both()));
- assertFalse(skipPartitioner(g.V().both().count()));
- assertFalse(skipPartitioner(g.V().out().id()));
- assertFalse(skipPartitioner(g.V().out().out().count()));
- assertFalse(skipPartitioner(g.V().in().count()));
- assertFalse(skipPartitioner(g.V().inE().count()));
-
-
- }
-
- private static boolean skipPartitioner(final Traversal<?, ?> traversal) {
- traversal.asAdmin().applyStrategies();
- return (Boolean) TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
- .getComputer()
- .getConfiguration()
- .getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/854b259c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java
new file mode 100644
index 0000000..4e43438
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkSingleIterationStrategyTest extends AbstractSparkTest {
+
+ @Test
+ public void shouldSuccessfullyEvaluateSingleIterationTraversals() throws Exception {
+ final String outputLocation = TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString());
+ Configuration configuration = getBaseConfiguration();
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
+ configuration.setProperty(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
+
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal().withComputer().withoutStrategies(SparkInterceptorStrategy.class);
+ assertFalse(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
+ assertFalse(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
+ assertTrue(g.getStrategies().toList().contains(SparkSingleIterationStrategy.instance()));
+ assertTrue(g.V().count().explain().toString().contains(SparkSingleIterationStrategy.class.getSimpleName()));
+
+ test(true, g.V().limit(10));
+ test(true, g.V().values("age").groupCount());
+ test(true, g.V().groupCount().by(__.out().count()));
+ test(true, g.V().outE());
+ test(true, 6l, g.V().count());
+ test(true, 6l, g.V().out().count());
+ test(true, 6l, g.V().local(__.inE()).count());
+ test(true, 6l, g.V().outE().inV().count());
+ ////
+ test(false, g.V().outE().inV());
+ test(false, g.V().both());
+ test(false, 12l, g.V().both().count());
+ test(false, g.V().out().id());
+ test(false, 2l, g.V().out().out().count());
+ test(false, 6l, g.V().in().count());
+ test(false, 6l, g.V().inE().count());
+ }
+
+ private static <R> void test(boolean singleIteration, final Traversal<?, R> traversal) {
+ test(singleIteration, null, traversal);
+ }
+
+ private static <R> void test(boolean singleIteration, R expectedResult, final Traversal<?, R> traversal) {
+ traversal.asAdmin().applyStrategies();
+ final Map<String, Object> configuration = TraversalHelper.getFirstStepOfAssignableClass(TraversalVertexProgramStep.class, traversal.asAdmin()).get()
+ .getComputer()
+ .getConfiguration();
+ assertEquals(singleIteration, configuration.getOrDefault(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, false));
+ assertEquals(singleIteration, configuration.getOrDefault(Constants.GREMLIN_SPARK_SKIP_GRAPH_CACHE, false));
+ final List<R> result = traversal.toList();
+ if (null != expectedResult)
+ assertEquals(expectedResult, result.get(0));
+ }
+
+}
[09/18] incubator-tinkerpop git commit: just generalized
SparkCountInterceptor to work with ANY star graph computation that ends with
count(). The concept is simple,
just do flatMapToPair(traversal).count(). tada. This is so clean and
pretty.... next is
Posted by ok...@apache.org.
just generalized SparkCountInterceptor to work with ANY star graph computation that ends with count(). The concept is simple, just do flatMapToPair(traversal).count(). tada. This is so clean and pretty.... next is going to be generalizing to work with any ReducingBarrierStep, not just count(). Also, need to add some test cases that use sideEffects and ensure that Memory is updated accordingly. However, besides that, this is such a dead simple concept that I actually think this is how we should redesign SparkGraphComputer. Break up the traverasl into starGraphTraversals and then concatenate the jobs via Spark join()s.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6f9e5fa0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6f9e5fa0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6f9e5fa0
Branch: refs/heads/master
Commit: 6f9e5fa0c7ffa01b7fbd3c3fed9539f6ad12bd4c
Parents: 2717b28
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 11:26:00 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 11:26:00 2016 -0600
----------------------------------------------------------------------
.../traversal/TraversalVertexProgram.java | 19 ++----
.../optimization/SparkInterceptorStrategy.java | 5 +-
.../SparkPartitionAwareStrategy.java | 4 +-
.../interceptor/SparkCountInterceptor.java | 72 +++++++-------------
.../SparkInterceptorStrategyTest.java | 8 +--
5 files changed, 39 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6f9e5fa0/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index d640e51..f6ed25d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -123,23 +123,12 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
}
/**
- * A helper method to yield a {@link Traversal} from the {@link Graph} and provided {@link Configuration}.
+ * Get the {@link PureTraversal} associated with the current instance of the {@link TraversalVertexProgram}.
*
- * @param graph the graph that the traversal will run against
- * @param configuration The configuration containing the TRAVERSAL key.
- * @return the traversal supplied by the configuration
+ * @return the pure traversal of the instantiated program
*/
- public static Traversal.Admin<?, ?> getTraversal(final Graph graph, final Configuration configuration) {
- return VertexProgram.<TraversalVertexProgram>createVertexProgram(graph, configuration).traversal.get();
- }
-
- /**
- * Get the {@link Traversal} associated with the current instance of the traversal vertex program.
- *
- * @return the traversal of the instantiated program
- */
- public Traversal.Admin<?, ?> getTraversal() {
- return this.traversal.get();
+ public PureTraversal<?, ?> getTraversal() {
+ return this.traversal;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6f9e5fa0/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index b3d250a..89b831d 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -46,7 +46,10 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance());
final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
for (final TraversalVertexProgramStep step : steps) {
- if (SparkCountInterceptor.isLegal(step.generateProgram(graph).getTraversal()))
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
+ if (!computerTraversal.isLocked())
+ computerTraversal.applyStrategies();
+ if (SparkCountInterceptor.isLegal(computerTraversal))
step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkCountInterceptor.class.getCanonicalName()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6f9e5fa0/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
index de3e737..a7380c4 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkPartitionAwareStrategy.java
@@ -59,7 +59,9 @@ public final class SparkPartitionAwareStrategy extends AbstractTraversalStrategy
public void apply(final Traversal.Admin<?, ?> traversal) {
final List<TraversalVertexProgramStep> steps = TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal);
for (final TraversalVertexProgramStep step : steps) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(traversal.getGraph().orElse(EmptyGraph.instance())).getTraversal();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(traversal.getGraph().orElse(EmptyGraph.instance())).getTraversal().get().clone();
+ if (!computerTraversal.isLocked())
+ computerTraversal.applyStrategies();
boolean messagePasses = MESSAGEPASS_CLASSES.stream()
.flatMap(clazz -> TraversalHelper.<Step<?, ?>>getStepsOfAssignableClassRecursively((Class) clazz, computerTraversal).stream())
.filter(s -> TraversalHelper.isGlobalChild(((Step) s).getTraversal().asAdmin()))
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6f9e5fa0/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
index 7ca4b61..6f91c42 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
@@ -24,22 +24,18 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
-import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
+import java.util.Iterator;
import java.util.List;
/**
@@ -54,60 +50,40 @@ public final class SparkCountInterceptor implements SparkVertexProgramIntercepto
@Override
public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
vertexProgram.setup(memory);
- final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal();
- JavaPairRDD<Object, VertexWritable> nextRDD = inputRDD;
- long count = -1;
- for (final Step step : traversal.getSteps()) {
- if (step instanceof GraphStep) {
- final Object[] ids = ((GraphStep) step).getIds();
- if (ids.length > 0)
- nextRDD = nextRDD.filter(tuple -> ElementHelper.idExists(tuple._2().get().id(), ids));
- } else if (step instanceof HasStep) {
- final List<HasContainer> hasContainers = ((HasStep<?>) step).getHasContainers();
- if (hasContainers.size() > 0)
- nextRDD = nextRDD.filter(tuple -> HasContainer.testAll(tuple._2().get(), hasContainers));
- } else if (step instanceof PropertiesStep) {
- final String[] keys = ((PropertiesStep) step).getPropertyKeys();
- count = nextRDD.mapValues(vertex -> IteratorUtils.count(vertex.get().properties(keys))).fold(new Tuple2<>("sum", 0l), (a, b) -> new Tuple2<>("sum", a._2() + b._2()))._2();
- } else if (step instanceof VertexStep) {
- final String[] labels = ((VertexStep) step).getEdgeLabels();
- final Direction direction = ((VertexStep) step).getDirection();
- count = nextRDD.mapValues(vertex -> IteratorUtils.count(vertex.get().edges(direction, labels))).fold(new Tuple2<>("sum", 0l), (a, b) -> new Tuple2<>("sum", a._2() + b._2()))._2();
- } else if (step instanceof CountGlobalStep) {
- if (count == -1)
- count = nextRDD.count();
- }
- }
- assert count != -1;
+ final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal().getPure().clone();
+ final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds();
+ final CountGlobalStep countGlobalStep = (CountGlobalStep) traversal.getEndStep();
+ traversal.removeStep(0); // remove GraphStep
+ traversal.removeStep(traversal.getSteps().size() - 1); // remove CountGlobalStep
+ traversal.applyStrategies(); // compile
+ boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
+ final long count = inputRDD
+ .filter(tuple -> ElementHelper.idExists(tuple._2().get().id(), graphStepIds))
+ .flatMapValues(vertexWritable -> {
+ if (identityTraversal) // g.V.count()-style (identity)
+ return () -> (Iterator) IteratorUtils.of(vertexWritable);
+ else { // add the vertex to head of the traversal
+ final Traversal.Admin<Vertex, ?> clone = traversal.clone();
+ clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
+ return () -> clone;
+ }
+ }).count();
+ // generate the HALTED_TRAVERSERS for the memory
final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
- haltedTraversers.add(traversal.getTraverserGenerator().generate(count, (Step) traversal.getEndStep(), 1l));
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(count, countGlobalStep, 1l));
memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
- memory.incrIteration();
+ memory.incrIteration(); // any local star graph reduction take a single iteration
return inputRDD;
}
public static boolean isLegal(final Traversal.Admin<?, ?> traversal) {
- if (!TraversalHelper.getLabels(traversal).isEmpty())
- return false;
final List<Step> steps = traversal.getSteps();
if (!steps.get(0).getClass().equals(GraphStep.class) || ((GraphStep) steps.get(0)).returnsEdge())
return false;
if (!steps.get(steps.size() - 1).getClass().equals(CountGlobalStep.class))
return false;
- int vertexPropertiesStepCount = 0;
- for (int i = 1; i < steps.size() - 1; i++) {
- final Step<?, ?> step = steps.get(i);
- final Class<? extends Step> stepClass = step.getClass();
- if (!stepClass.equals(HasStep.class) && !stepClass.equals(PropertiesStep.class) && !stepClass.equals(VertexStep.class))
- return false;
- if ((stepClass.equals(VertexStep.class) || stepClass.equals(PropertiesStep.class)) && (++vertexPropertiesStepCount > 1 || !step.getNextStep().getClass().equals(CountGlobalStep.class)))
- return false;
- if (stepClass.equals(HasStep.class) && !step.getPreviousStep().getClass().equals(GraphStep.class) && !step.getPreviousStep().getClass().equals(HasStep.class))
- return false;
- }
- return true;
+ return TraversalHelper.isLocalStarGraph(traversal);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6f9e5fa0/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index c1cf8cf..b2a4820 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -80,17 +80,17 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
test(SparkCountInterceptor.class, 5l, g.V(1, 4).out("knows", "created").count());
test(SparkCountInterceptor.class, 1l, g.V(2).in("knows").count());
test(SparkCountInterceptor.class, 0l, g.V(6).has("name", "peter").in().count());
+ test(SparkCountInterceptor.class, 6l, g.V().as("a").values("name").as("b").count());
+ test(SparkCountInterceptor.class, 6l, g.V().as("a").count());
+ test(SparkCountInterceptor.class, 1l, g.V().has("name", "marko").as("a").values("name").as("b").count());
+ test(SparkCountInterceptor.class, 4l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age").out("created").count());
/// No interceptor matches
test(2l, g.V().out().out().count());
test(6l, g.E().count());
- test(6l, g.V().as("a").values("name").as("b").count());
- test(6l, g.V().as("a").count());
- test(1l, g.V().has("name", "marko").as("a").values("name").as("b").count());
test(2l, g.V().out().out().count());
test(6l, g.V().out().values("name").count());
test(2l, g.V().out("knows").values("name").count());
test(3l, g.V().in().has("name", "marko").count());
- test(4l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age").out("created").count()); // TODO: filter(values()) should be okay.
}
private static <R> void test(Class<? extends VertexProgramInterceptor> expectedInterceptor, R expectedResult, final Traversal<?, R> traversal) throws Exception {
[12/18] incubator-tinkerpop git commit: SparkCountInterceptor is now
called SparkStarBarrierInterceptor and works with count(), max(), sum(), min(),
and mean(). groupCount(), group(), and tree() are still left to do.
Posted by ok...@apache.org.
SparkCountInterceptor is now called SparkStarBarrierInterceptor and works with count(), max(), sum(), min(), and mean(). groupCount(), group(), and tree() are still left to do.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6a256f8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6a256f8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6a256f8f
Branch: refs/heads/master
Commit: 6a256f8fcbe6a87a8d794466a7b559ff9ba8d2d6
Parents: 01fda96
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 4 15:45:27 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 4 15:45:27 2016 -0600
----------------------------------------------------------------------
.../step/sideEffect/SackValueStep.java | 1 -
.../optimization/SparkInterceptorStrategy.java | 6 +-
.../interceptor/SparkCountInterceptor.java | 103 --------------
.../SparkStarBarrierInterceptor.java | 142 +++++++++++++++++++
.../SparkInterceptorStrategyTest.java | 47 +++---
5 files changed, 171 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a256f8f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SackValueStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SackValueStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SackValueStep.java
index 1b02fe2..7f030d6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SackValueStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SackValueStep.java
@@ -58,7 +58,6 @@ public final class SackValueStep<S, A, B> extends SideEffectStep<S> implements T
@Override
protected void sideEffect(final Traverser.Admin<S> traverser) {
traverser.sack(this.sackFunction.apply(traverser.sack(), null == this.sackTraversal ? (B) traverser.get() : TraversalUtil.apply(traverser, this.sackTraversal)));
-
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a256f8f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index 89b831d..11c03f7 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -25,7 +25,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkStarBarrierInterceptor;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
@@ -49,8 +49,8 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
- if (SparkCountInterceptor.isLegal(computerTraversal))
- step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkCountInterceptor.class.getCanonicalName()));
+ if (SparkStarBarrierInterceptor.isLegal(computerTraversal))
+ step.setComputer(step.getComputer().configure(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR, SparkStarBarrierInterceptor.class.getCanonicalName()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a256f8f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
deleted file mode 100644
index f85d6db..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkCountInterceptor.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor;
-
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
-import org.apache.tinkerpop.gremlin.process.traversal.Scope;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupGlobalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.CollectingBarrierStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkCountInterceptor implements SparkVertexProgramInterceptor<TraversalVertexProgram> {
-
- public SparkCountInterceptor() {
-
- }
-
- @Override
- public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
- vertexProgram.setup(memory);
- final Traversal.Admin<Vertex, Long> traversal = (Traversal.Admin) vertexProgram.getTraversal().getPure().clone();
- final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds(); // any V(1,2,3)-style ids to filter on
- final CountGlobalStep countGlobalStep = (CountGlobalStep) traversal.getEndStep(); // needed for the final traverser generation
- traversal.removeStep(0); // remove GraphStep
- traversal.removeStep(traversal.getSteps().size() - 1); // remove CountGlobalStep
- traversal.applyStrategies(); // compile
- boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
-
- ((MemoryTraversalSideEffects) traversal.getSideEffects()).setMemory(memory, true); // any intermediate sideEffect steps are backed by SparkMemory
- memory.setInExecute(true);
- final long count = inputRDD
- .filter(tuple -> ElementHelper.idExists(tuple._2().get().id(), graphStepIds))
- .flatMapValues(vertexWritable -> {
- if (identityTraversal) // g.V.count()-style (identity)
- return () -> (Iterator) IteratorUtils.of(vertexWritable);
- else { // add the vertex to head of the traversal
- return () -> { // and iterate it for its results
- final Traversal.Admin<Vertex, ?> clone = traversal.clone(); // need a unique clone for each vertex to isolate the computation
- clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
- return clone;
- };
- }
- }).count();
- memory.setInExecute(false);
-
- // generate the HALTED_TRAVERSERS for the memory
- final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
- haltedTraversers.add(traversal.getTraverserGenerator().generate(count, countGlobalStep, 1l));
- memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
- memory.incrIteration(); // any local star graph reduction take a single iteration
- return inputRDD;
- }
-
- public static boolean isLegal(final Traversal.Admin<?, ?> traversal) {
- final List<Step> steps = traversal.getSteps();
- if (!steps.get(0).getClass().equals(GraphStep.class) || ((GraphStep) steps.get(0)).returnsEdge())
- return false;
- if (!steps.get(steps.size() - 1).getClass().equals(CountGlobalStep.class))
- return false;
- if (TraversalHelper.hasStepOfAssignableClassRecursively(Scope.global, Arrays.asList(CollectingBarrierStep.class, DedupGlobalStep.class), traversal))
- return false;
- return TraversalHelper.isLocalStarGraph(traversal);
-
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a256f8f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
new file mode 100644
index 0000000..3649c8f
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.NumberHelper;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.MaxGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.MeanGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.MinGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkMemory;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.SparkVertexProgramInterceptor;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkStarBarrierInterceptor implements SparkVertexProgramInterceptor<TraversalVertexProgram> {
+
+ public SparkStarBarrierInterceptor() {
+
+ }
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> apply(final TraversalVertexProgram vertexProgram, final JavaPairRDD<Object, VertexWritable> inputRDD, final SparkMemory memory) {
+ vertexProgram.setup(memory);
+ final Traversal.Admin<Vertex, Object> traversal = (Traversal.Admin) vertexProgram.getTraversal().getPure().clone();
+ final Object[] graphStepIds = ((GraphStep) traversal.getStartStep()).getIds(); // any V(1,2,3)-style ids to filter on
+ final ReducingBarrierStep endStep = (ReducingBarrierStep) traversal.getEndStep(); // needed for the final traverser generation
+ final Traversal.Admin<Object, Object> endStepTraversal = new DefaultTraversal<>();
+ endStepTraversal.addStep(endStep).applyStrategies();
+ traversal.removeStep(0); // remove GraphStep
+ traversal.removeStep(traversal.getSteps().size() - 1); // remove CountGlobalStep
+ traversal.applyStrategies(); // compile
+ boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
+ ///////////////////////////////
+ ((MemoryTraversalSideEffects) traversal.getSideEffects()).setMemory(memory, true); // any intermediate sideEffect steps are backed by SparkMemory
+ memory.setInExecute(true);
+ final JavaRDD<Traverser.Admin<Object>> nextRDD = inputRDD.values()
+ .filter(vertexWritable -> ElementHelper.idExists(vertexWritable.get().id(), graphStepIds))
+ .flatMap(vertexWritable -> {
+ if (identityTraversal) // g.V.count()-style (identity)
+ return () -> IteratorUtils.of(traversal.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
+ else { // add the vertex to head of the traversal
+ return () -> { // and iterate it for its results
+ final Traversal.Admin<Vertex, ?> clone = traversal.clone(); // need a unique clone for each vertex to isolate the computation
+ clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), EmptyStep.instance(), 1l));
+ return (Step) clone.getEndStep();
+ };
+ }
+ });
+ final Object result;
+ if (endStep instanceof CountGlobalStep)
+ result = nextRDD.map(Traverser::bulk).fold(0l, (a, b) -> a + b);
+ else if (endStep instanceof SumGlobalStep)
+ result = nextRDD
+ .map(traverser -> NumberHelper.mul(traverser.bulk(), (Number) traverser.get()))
+ .fold(0, NumberHelper::add);
+ else if (endStep instanceof MeanGlobalStep)
+ result = nextRDD
+ .map(traverser -> new MeanGlobalStep.MeanNumber((Number) traverser.get(), traverser.bulk()))
+ .fold(new MeanGlobalStep.MeanNumber(), MeanGlobalStep.MeanNumber::add)
+ .getFinal();
+ else if (endStep instanceof MinGlobalStep)
+ result = nextRDD
+ .map(traverser -> (Number) traverser.get())
+ .fold(Integer.MAX_VALUE, NumberHelper::min);
+ else if (endStep instanceof MaxGlobalStep)
+ result = nextRDD
+ .map(traverser -> (Number) traverser.get())
+ .fold(Integer.MIN_VALUE, NumberHelper::max);
+ else
+ throw new IllegalArgumentException("The end step is an unsupported barrier: " + endStep);
+ memory.setInExecute(false);
+ ///////////////////////////////
+
+ // generate the HALTED_TRAVERSERS for the memory
+ final TraverserSet<Long> haltedTraversers = new TraverserSet<>();
+ haltedTraversers.add(traversal.getTraverserGenerator().generate(result, endStep, 1l));
+ memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
+ memory.incrIteration(); // any local star graph reduction take a single iteration
+ return inputRDD;
+ }
+
+ public static boolean isLegal(final Traversal.Admin<?, ?> traversal) {
+ final Step<?, ?> startStep = traversal.getStartStep();
+ final Step<?, ?> endStep = traversal.getEndStep();
+ if (!startStep.getClass().equals(GraphStep.class) || ((GraphStep) startStep).returnsEdge())
+ return false;
+ if (!endStep.getClass().equals(CountGlobalStep.class) &&
+ !endStep.getClass().equals(SumGlobalStep.class) &&
+ !endStep.getClass().equals(MeanGlobalStep.class) &&
+ !endStep.getClass().equals(MaxGlobalStep.class) &&
+ !endStep.getClass().equals(MinGlobalStep.class))
+ // TODO: group(), groupCount(), fold(), and tree()
+ return false;
+ if (TraversalHelper.getStepsOfAssignableClassRecursively(Scope.global, Barrier.class, traversal).size() != 1)
+ return false;
+ if (traversal.getTraverserRequirements().contains(TraverserRequirement.SACK))
+ return false;
+ return TraversalHelper.isLocalStarGraph(traversal);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6a256f8f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index 4529fd3..f18a944 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -33,7 +33,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkCountInterceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization.interceptor.SparkStarBarrierInterceptor;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.T;
@@ -69,7 +69,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
/// groupCount(m)-test
Traversal.Admin<Vertex, Long> traversal = g.V().groupCount("m").by(T.label).count().asAdmin();
- test(SparkCountInterceptor.class, 6l, traversal);
+ test(SparkStarBarrierInterceptor.class, 6l, traversal);
assertEquals(1, traversal.getSideEffects().keys().size());
assertTrue(traversal.getSideEffects().exists("m"));
assertTrue(traversal.getSideEffects().keys().contains("m"));
@@ -94,25 +94,29 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
assertTrue(g.getStrategies().toList().contains(SparkInterceptorStrategy.instance()));
assertTrue(g.V().count().explain().toString().contains(SparkInterceptorStrategy.class.getSimpleName()));
/// SparkCountInterceptor matches
- test(SparkCountInterceptor.class, 6l, g.V().count());
- test(SparkCountInterceptor.class, 2l, g.V().hasLabel("software").count());
- test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).count());
- test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).values("name").count());
- test(SparkCountInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name").count());
- test(SparkCountInterceptor.class, 4l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name", "age").count());
- test(SparkCountInterceptor.class, 3l, g.V().hasLabel("person").has("age", P.gt(30)).out().count());
- test(SparkCountInterceptor.class, 0l, g.V().hasLabel("person").has("age", P.gt(30)).out("knows").count());
- test(SparkCountInterceptor.class, 3l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age", P.gt(30)).out("created").count());
- test(SparkCountInterceptor.class, 3l, g.V(1).out().count());
- test(SparkCountInterceptor.class, 2l, g.V(1).out("knows").count());
- test(SparkCountInterceptor.class, 3l, g.V(1).out("knows", "created").count());
- test(SparkCountInterceptor.class, 5l, g.V(1, 4).out("knows", "created").count());
- test(SparkCountInterceptor.class, 1l, g.V(2).in("knows").count());
- test(SparkCountInterceptor.class, 0l, g.V(6).has("name", "peter").in().count());
- test(SparkCountInterceptor.class, 6l, g.V().as("a").values("name").as("b").count());
- test(SparkCountInterceptor.class, 6l, g.V().as("a").count());
- test(SparkCountInterceptor.class, 1l, g.V().has("name", "marko").as("a").values("name").as("b").count());
- test(SparkCountInterceptor.class, 4l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age").out("created").count());
+ test(SparkStarBarrierInterceptor.class, 6l, g.V().count());
+ test(SparkStarBarrierInterceptor.class, 2l, g.V().hasLabel("software").count());
+ test(SparkStarBarrierInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).count());
+ test(SparkStarBarrierInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).values("name").count());
+ test(SparkStarBarrierInterceptor.class, 2l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name").count());
+ test(SparkStarBarrierInterceptor.class, 4l, g.V().hasLabel("person").has("age", P.gt(30)).properties("name", "age").count());
+ test(SparkStarBarrierInterceptor.class, 3l, g.V().hasLabel("person").has("age", P.gt(30)).out().count());
+ test(SparkStarBarrierInterceptor.class, 0l, g.V().hasLabel("person").has("age", P.gt(30)).out("knows").count());
+ test(SparkStarBarrierInterceptor.class, 3l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age", P.gt(30)).out("created").count());
+ test(SparkStarBarrierInterceptor.class, 3l, g.V(1).out().count());
+ test(SparkStarBarrierInterceptor.class, 2l, g.V(1).out("knows").count());
+ test(SparkStarBarrierInterceptor.class, 3l, g.V(1).out("knows", "created").count());
+ test(SparkStarBarrierInterceptor.class, 5l, g.V(1, 4).out("knows", "created").count());
+ test(SparkStarBarrierInterceptor.class, 1l, g.V(2).in("knows").count());
+ test(SparkStarBarrierInterceptor.class, 0l, g.V(6).has("name", "peter").in().count());
+ test(SparkStarBarrierInterceptor.class, 6l, g.V().as("a").values("name").as("b").count());
+ test(SparkStarBarrierInterceptor.class, 6l, g.V().as("a").count());
+ test(SparkStarBarrierInterceptor.class, 1l, g.V().has("name", "marko").as("a").values("name").as("b").count());
+ test(SparkStarBarrierInterceptor.class, 4l, g.V().has(T.label, P.not(P.within("robot", "android")).and(P.within("person", "software"))).hasLabel("person").has("age").out("created").count());
+ test(SparkStarBarrierInterceptor.class, 123l, g.V().has("age").values("age").sum());
+ test(SparkStarBarrierInterceptor.class, 67l, g.V().has("age").has("age", P.gt(30)).values("age").sum());
+ test(SparkStarBarrierInterceptor.class, 27, g.V().hasLabel("person").values("age").min());
+ test(SparkStarBarrierInterceptor.class, 35, g.V().hasLabel("person").values("age").max());
/// No interceptor matches
test(2l, g.V().out().out().count());
test(6l, g.E().count());
@@ -123,6 +127,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
test(6l, g.V().repeat(__.dedup()).times(2).count());
test(6l, g.V().dedup().count());
test(4l, g.V().hasLabel("person").order().by("age").count());
+ test(1l, g.V().count().count());
}
private static <R> void test(Class<? extends VertexProgramInterceptor> expectedInterceptor, R expectedResult, final Traversal<?, R> traversal) throws Exception {
[18/18] incubator-tinkerpop git commit: updated CHANGELOG.
Posted by ok...@apache.org.
updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e560970f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e560970f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e560970f
Branch: refs/heads/master
Commit: e560970f487f20cb7769f831a7a7586109e462a1
Parents: 197f59c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri May 6 10:08:39 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri May 6 10:08:39 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e560970f/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 3a9024e..0491bac 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,11 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.2.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `VertexProgramInterceptor` interface as a general pattern for `GraphComputer` providers to use for bypassing `GraphComputer` semantics where appropriate.
+* Added `SparkStarBarrierInterceptor` that uses Spark DSL for local star graph traversals that end with a `ReducingBarrierStep`.
+* Added `SparkInterceptorStrategy` which identifies which interceptor to use (if any) given the submitted `VertexProgram`.
+* Added `SparkSingleIterationStrategy` that does not partition nor cache the graph RDD if the traversal does not message pass.
+* Added more helper methods to `TraversalHelper` for handling scoped traversal children.
* Deprecated all "performance" tests based on "JUnit Benchmarks".
* `SparkGraphComputer` no longer shuffles empty views or empty outgoing messages in order to save time and space.
* `TraversalVertexProgram` no longer maintains empty halted traverser properties in order to save space.