You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/12 23:15:27 UTC
incubator-tinkerpop git commit: added another PageRankTest test case
and made MapReduce.map() a non-default implementation so users know they have
to implement that method.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1140 0a2243037 -> 491bef661
added another PageRankTest test case and made MapReduce.map() a non-default implementation so users know they have to implement that method.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/491bef66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/491bef66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/491bef66
Branch: refs/heads/TINKERPOP-1140
Commit: 491bef66112d75d42cde87ae000e63587e9b3bfe
Parents: 0a22430
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Feb 12 15:15:18 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Feb 12 15:15:18 2016 -0700
----------------------------------------------------------------------
.../gremlin/process/computer/GraphComputer.java | 4 ---
.../gremlin/process/computer/MapReduce.java | 4 +--
.../traversal/TraversalVertexProgram.java | 6 ----
.../decoration/VertexProgramStrategy.java | 3 ++
.../step/map/GroovyPageRankTest.groovy | 6 ++++
.../process/computer/GraphComputerTest.java | 34 ++------------------
.../traversal/step/map/PageRankTest.java | 22 +++++++++++++
.../computer/AbstractHadoopGraphComputer.java | 5 ---
.../process/computer/CombineIteratorTest.java | 13 ++++++++
.../process/computer/TinkerGraphComputer.java | 4 ---
10 files changed, 49 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
index 62fc260..ddb09a0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
@@ -272,10 +272,6 @@ public interface GraphComputer {
public static IllegalArgumentException edgeFilterAccessesAdjacentVertices(final Traversal<Vertex, Edge> edgeFilter) {
return new IllegalArgumentException("The provided edge filter traversal accesses data on adjacent vertices: " + edgeFilter);
}
-
- public static IllegalStateException mapReduceJobsMustHaveAMapStage(final MapReduce mapReduce) {
- throw new IllegalStateException("MapReduce jobs must have a map stage: " + mapReduce);
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MapReduce.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MapReduce.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MapReduce.java
index 3e1eae6..f6e6c0c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MapReduce.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MapReduce.java
@@ -87,12 +87,12 @@ public interface MapReduce<MK, MV, RK, RV, R> extends Cloneable {
/**
* The map() method is logically executed at all vertices in the graph in parallel.
* The map() method emits key/value pairs given some analysis of the data in the vertices (and/or its incident edges).
+ * All {@link MapReduce} classes must at least provide an implementation of {@code MapReduce#map(Vertex, MapEmitter)}.
*
* @param vertex the current vertex being map() processed.
* @param emitter the component that allows for key/value pairs to be emitted to the next stage.
*/
- public default void map(final Vertex vertex, final MapEmitter<MK, MV> emitter) {
- }
+ public void map(final Vertex vertex, final MapEmitter<MK, MV> emitter);
/**
* The combine() method is logically executed at all "machines" in parallel.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index cdd7362..1ed12d6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -267,12 +267,6 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
};
}
- /*public <S, E> Traversal.Admin<S, E> computerResultTraversal(final ComputerResult result) {
- final Traversal.Admin<S, E> traversal = (Traversal.Admin<S, E>) this.getTraversal();
- ((ComputerResultStep) traversal.getEndStep()).populateTraversers(result);
- return traversal;
- }*/
-
//////////////
public static Builder build() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategy.java
index 8fbfb4f..7d4d554 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategy.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy;
@@ -69,6 +70,8 @@ public final class VertexProgramStrategy extends AbstractTraversalStrategy<Trave
pageRankVertexProgramStep.setGraphComputerFunction(this.graphComputerFunction);
graphStep.getLabels().forEach(pageRankVertexProgramStep::addLabel);
traversal.removeStep(0); // remove the graph step
+ if (traversal.getSteps().size() == 1) // todo: in the future, this should just be a mapreduce job added to the PageRankVertexProgram step
+ traversal.addStep(new IdentityStep<>(traversal));
}
if (null != this.graphComputerFunction) { // if the function is null, then its been serialized and thus, already in a graph computer
Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
index 0efcd2d..45fe088 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
@@ -31,6 +31,12 @@ public abstract class GroovyPageRankTest {
public static class Traversals extends PageRankTest {
@Override
+ public Traversal<Vertex, Vertex> get_g_V_pageRank() {
+ TraversalScriptHelper.compute("g.V.pageRank", g)
+ }
+
+
+ @Override
public Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name() {
TraversalScriptHelper.compute("g.V.pageRank.order.by(PageRankVertexProgram.PAGE_RANK, decr).name", g)
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
index c910744..33343b1 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
@@ -77,8 +77,7 @@ import static org.junit.Assert.fail;
"vertexPropertiesCanNotBeUpdatedInMapReduce",
"computerRequiresMoreWorkersThanSupported",
"vertexFilterAccessesIncidentEdges",
- "edgeFilterAccessesAdjacentVertices",
- "mapReduceJobsMustHaveAMapStage"
+ "edgeFilterAccessesAdjacentVertices"
})
@ExceptionCoverage(exceptionClass = Graph.Exceptions.class, methods = {
"graphDoesNotSupportProvidedGraphComputer"
@@ -104,35 +103,6 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
}
}
- @Test
- @LoadGraphWith(MODERN)
- public void shouldNotAllowMapReduceJobsWithoutAMapStage() throws Exception {
- try {
- graphProvider.getGraphComputer(graph).mapReduce(new BadMapReduce()).submit().get();
- fail("Should throw an IllegalStateException saying that MapReduce jobs must have a map stage");
- } catch (Exception e) {
- assertTrue(true);
- }
- }
-
- public static class BadMapReduce extends StaticMapReduce {
-
- @Override
- public boolean doStage(final Stage stage) {
- return false;
- }
-
- @Override
- public String getMemoryKey() {
- return "nothing";
- }
-
- @Override
- public Object generateFinalResult(final Iterator keyValues) {
- return new Object();
- }
- }
-
/////////////////////////////////////////////
@Test
@LoadGraphWith(MODERN)
@@ -1909,5 +1879,7 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
assertEquals(6, graph3.traversal().V().values(PeerPressureVertexProgram.VOTE_STRENGTH).count().next().intValue());
assertEquals(6, graph3.traversal().V().values(PageRankVertexProgram.PAGE_RANK).count().next().intValue());
assertEquals(6, graph3.traversal().V().values(PageRankVertexProgram.EDGE_COUNT).count().next().intValue());
+
+ // TODO: add a test the shows DAG behavior -- splitting another TraversalVertexProgram off of the PeerPressureVertexProgram job.
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
index b28d6d3..51fa4b7 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
@@ -41,12 +41,28 @@ import static org.junit.Assert.assertTrue;
@RunWith(GremlinProcessRunner.class)
public abstract class PageRankTest extends AbstractGremlinProcessTest {
+ public abstract Traversal<Vertex, Vertex> get_g_V_pageRank();
+
public abstract Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name();
public abstract Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name_limitX2X();
@Test
@LoadGraphWith(MODERN)
+ public void g_V_pageRank() {
+ final Traversal<Vertex, Vertex> traversal = get_g_V_pageRank();
+ printTraversalForm(traversal);
+ int counter = 0;
+ while (traversal.hasNext()) {
+ final Vertex vertex = traversal.next();
+ counter++;
+ assertTrue(vertex.property(PageRankVertexProgram.PAGE_RANK).isPresent());
+ }
+ assertEquals(6, counter);
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
public void g_V_pageRank_order_byXpageRank_decrX_name() {
final Traversal<Vertex, String> traversal = get_g_V_pageRank_order_byXpageRank_decrX_name();
printTraversalForm(traversal);
@@ -72,6 +88,12 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
}
public static class Traversals extends PageRankTest {
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_V_pageRank() {
+ return g.V().pageRank();
+ }
+
@Override
public Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name() {
return g.V().pageRank().order().by(PageRankVertexProgram.PAGE_RANK, Order.decr).values("name");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index 24455af..ac83dd6 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -133,11 +133,6 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
// if too many workers are requested, throw appropriate exception
if (this.workers > this.features().getMaxWorkers())
throw GraphComputer.Exceptions.computerRequiresMoreWorkersThanSupported(this.workers, this.features().getMaxWorkers());
- // all map reducers must have a map stage
- for (final MapReduce mapReduce : this.mapReducers) {
- if (!mapReduce.doStage(MapReduce.Stage.MAP))
- throw GraphComputer.Exceptions.mapReduceJobsMustHaveAMapStage(mapReduce);
- }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/CombineIteratorTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/CombineIteratorTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/CombineIteratorTest.java
index e5458b0..118a0cb 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/CombineIteratorTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/CombineIteratorTest.java
@@ -22,6 +22,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.junit.Test;
import scala.Tuple2;
@@ -148,12 +149,19 @@ public class CombineIteratorTest {
emitter.emit(key, counter);
}
+
+
@Override
public boolean doStage(final Stage stage) {
return true;
}
@Override
+ public void map(final Vertex vertex, final MapEmitter<String, Long> emitter) {
+
+ }
+
+ @Override
public String getMemoryKey() {
return "test";
}
@@ -181,6 +189,11 @@ public class CombineIteratorTest {
}
@Override
+ public void map(final Vertex vertex, final MapEmitter<NullObject, Long> emitter) {
+
+ }
+
+ @Override
public String getMemoryKey() {
return "test";
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/491bef66/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index c9bf7ec..e721a76 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -131,10 +131,6 @@ public final class TinkerGraphComputer implements GraphComputer {
// ensure requested workers are not larger than supported workers
if (this.workers > this.features().getMaxWorkers())
throw GraphComputer.Exceptions.computerRequiresMoreWorkersThanSupported(this.workers, this.features().getMaxWorkers());
- for (final MapReduce mapReduce : this.mapReducers) {
- if (!mapReduce.doStage(MapReduce.Stage.MAP))
- throw GraphComputer.Exceptions.mapReduceJobsMustHaveAMapStage(mapReduce);
- }
// initialize the memory
this.memory = new TinkerMemory(this.vertexProgram, this.mapReducers);