You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/22 18:38:54 UTC
incubator-tinkerpop git commit: EDGE_COUNT and VOTE_STRENGTH in
PageRankVertexProgram and PeerPressureVertexProgram are not transient and the
respective property keys are private static. Extended
GraphComputerTest.shouldSupportTransientKeys() with a MapR
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1166 706759c1d -> b03c1adc4
EDGE_COUNT and VOTE_STRENGTH in PageRankVertexProgram and PeerPressureVertexProgram are not transient and the respective property keys are private static. Extended GraphComputerTest.shouldSupportTransientKeys() with a MapReduce job that ensure that the transient vertex properties are not accessible during MapReduce.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b03c1adc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b03c1adc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b03c1adc
Branch: refs/heads/TINKERPOP-1166
Commit: b03c1adc466fa48fffcfeb7e17b71a243f66b76f
Parents: 706759c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Feb 22 10:38:50 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Feb 22 10:38:50 2016 -0700
----------------------------------------------------------------------
.../peerpressure/PeerPressureVertexProgram.java | 5 +--
.../ranking/pagerank/PageRankVertexProgram.java | 5 +--
.../process/computer/GraphComputerTest.java | 40 ++++++++++++++++----
.../pagerank/PageRankVertexProgramTest.java | 2 -
.../io/PersistedInputOutputRDDTest.java | 6 ---
.../process/computer/TinkerGraphComputer.java | 10 ++---
6 files changed, 42 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b03c1adc/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
index de4fdb9..c702aef 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
@@ -60,9 +60,8 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
private final Set<MessageScope> COUNT_SCOPE = new HashSet<>(Collections.singletonList(this.countScope));
public static final String CLUSTER = "gremlin.peerPressureVertexProgram.cluster";
- public static final String VOTE_STRENGTH = "gremlin.peerPressureVertexProgram.voteStrength";
-
- public static final String PROPERTY = "gremlin.peerPressureVertexProgram.property";
+ private static final String VOTE_STRENGTH = "gremlin.peerPressureVertexProgram.voteStrength";
+ private static final String PROPERTY = "gremlin.peerPressureVertexProgram.property";
private static final String MAX_ITERATIONS = "gremlin.peerPressureVertexProgram.maxIterations";
private static final String DISTRIBUTE_VOTE = "gremlin.peerPressureVertexProgram.distributeVote";
private static final String EDGE_TRAVERSAL = "gremlin.peerPressureVertexProgram.edgeTraversal";
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b03c1adc/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgram.java
index bb642e2..7609eba 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgram.java
@@ -52,9 +52,8 @@ public class PageRankVertexProgram implements VertexProgram<Double> {
public static final String PAGE_RANK = "gremlin.pageRankVertexProgram.pageRank";
- public static final String EDGE_COUNT = "gremlin.pageRankVertexProgram.edgeCount";
-
- public static final String PROPERTY = "gremlin.pageRankVertexProgram.property";
+ private static final String EDGE_COUNT = "gremlin.pageRankVertexProgram.edgeCount";
+ private static final String PROPERTY = "gremlin.pageRankVertexProgram.property";
private static final String VERTEX_COUNT = "gremlin.pageRankVertexProgram.vertexCount";
private static final String ALPHA = "gremlin.pageRankVertexProgram.alpha";
private static final String TOTAL_ITERATIONS = "gremlin.pageRankVertexProgram.totalIterations";
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b03c1adc/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
index 097284b..2f5f71f 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
@@ -1866,7 +1866,7 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
assertEquals(6, graph1.traversal().V().count().next().intValue());
assertEquals(6, graph1.traversal().E().count().next().intValue());
assertEquals(6, graph1.traversal().V().values(PageRankVertexProgram.PAGE_RANK).count().next().intValue());
- assertEquals(0, graph1.traversal().V().values(PageRankVertexProgram.EDGE_COUNT).count().next().intValue());
+ assertEquals(18, graph1.traversal().V().values().count().next().intValue());
//
final ComputerResult result2 = graph1.compute(graphProvider.getGraphComputer(graph1).getClass())
.program(PeerPressureVertexProgram.build().maxIterations(4).create(graph1)).persist(GraphComputer.Persist.EDGES).result(GraphComputer.ResultGraph.NEW).submit().get();
@@ -1876,9 +1876,8 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
assertEquals(6, graph2.traversal().V().count().next().intValue());
assertEquals(6, graph2.traversal().E().count().next().intValue());
assertEquals(6, graph2.traversal().V().values(PeerPressureVertexProgram.CLUSTER).count().next().intValue());
- assertEquals(0, graph2.traversal().V().values(PeerPressureVertexProgram.VOTE_STRENGTH).count().next().intValue());
assertEquals(6, graph2.traversal().V().values(PageRankVertexProgram.PAGE_RANK).count().next().intValue());
- assertEquals(0, graph2.traversal().V().values(PageRankVertexProgram.EDGE_COUNT).count().next().intValue());
+ assertEquals(24, graph2.traversal().V().values().count().next().intValue());
//
final ComputerResult result3 = graph2.compute(graphProvider.getGraphComputer(graph2).getClass())
.program(TraversalVertexProgram.build().traversal(g.V().groupCount("m").by(__.values(PageRankVertexProgram.PAGE_RANK).count()).label().asAdmin()).create(graph2)).persist(GraphComputer.Persist.EDGES).result(GraphComputer.ResultGraph.NEW).submit().get();
@@ -1896,9 +1895,8 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
assertEquals(6, graph3.traversal().E().count().next().intValue());
assertEquals(6, graph3.traversal().V().values(TraversalVertexProgram.HALTED_TRAVERSERS).count().next().intValue());
assertEquals(6, graph3.traversal().V().values(PeerPressureVertexProgram.CLUSTER).count().next().intValue());
- assertEquals(0, graph3.traversal().V().values(PeerPressureVertexProgram.VOTE_STRENGTH).count().next().intValue());
assertEquals(6, graph3.traversal().V().values(PageRankVertexProgram.PAGE_RANK).count().next().intValue());
- assertEquals(0, graph3.traversal().V().values(PageRankVertexProgram.EDGE_COUNT).count().next().intValue());
+ assertEquals(36, graph3.traversal().V().values().count().next().intValue()); // traverser side-effects
// TODO: add a test the shows DAG behavior -- splitting another TraversalVertexProgram off of the PeerPressureVertexProgram job.
}
@@ -1969,7 +1967,7 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
public void shouldSupportTransientKeys() throws Exception {
- final ComputerResult result = graphProvider.getGraphComputer(graph).program(new VertexProgramO()).submit().get();
+ final ComputerResult result = graphProvider.getGraphComputer(graph).program(new VertexProgramO()).mapReduce(new MapReduceK()).submit().get();
result.graph().vertices().forEachRemaining(vertex -> {
assertFalse(vertex.property("v1").isPresent());
assertFalse(vertex.property("v2").isPresent());
@@ -1998,7 +1996,7 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
assertFalse(result.memory().exists("m2"));
assertTrue(result.memory().exists("m3"));
assertEquals(24l, result.memory().<Long>get("m3").longValue());
- assertEquals(1, result.memory().keys().size());
+ assertEquals(2, result.memory().keys().size()); // mapReduceK
}
private static class VertexProgramO extends StaticVertexProgram {
@@ -2106,4 +2104,32 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
return GraphComputer.Persist.VERTEX_PROPERTIES;
}
}
+
+ public static class MapReduceK extends StaticMapReduce {
+
+ @Override
+ public boolean doStage(final Stage stage) {
+ return stage.equals(Stage.MAP);
+ }
+
+ @Override
+ public void map(final Vertex vertex, final MapEmitter emitter) {
+ assertFalse(vertex.property("v1").isPresent());
+ assertFalse(vertex.property("v2").isPresent());
+ assertTrue(vertex.property("v3").isPresent());
+ assertTrue(vertex.property("name").isPresent());
+ assertEquals(3, IteratorUtils.count(vertex.properties()));
+ assertEquals(3, IteratorUtils.count(vertex.values()));
+ }
+
+ @Override
+ public String getMemoryKey() {
+ return "mapReduceK";
+ }
+
+ @Override
+ public Object generateFinalResult(final Iterator keyValues) {
+ return "anObject";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b03c1adc/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgramTest.java
index cd89fa9..41dfaa0 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ranking/pagerank/PageRankVertexProgramTest.java
@@ -27,7 +27,6 @@ import org.junit.Test;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
@@ -43,7 +42,6 @@ public class PageRankVertexProgramTest extends AbstractGremlinProcessTest {
result.graph().traversal().V().forEachRemaining(v -> {
assertEquals(3, v.keys().size()); // name, age/lang, pageRank
assertTrue(v.keys().contains("name"));
- assertFalse(v.keys().contains(PageRankVertexProgram.EDGE_COUNT));
assertTrue(v.keys().contains(PageRankVertexProgram.PAGE_RANK));
assertEquals(1, IteratorUtils.count(v.values("name")));
assertEquals(1, IteratorUtils.count(v.values(PageRankVertexProgram.PAGE_RANK)));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b03c1adc/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
index 6567b58..040dd39 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
@@ -209,7 +209,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(0l, g.E().count().next().longValue());
assertEquals("marko", g.V().has("name", "marko").values("name").next());
assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
- assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue());
////
Spark.close();
}
@@ -249,7 +248,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(6l, g.E().count().next().longValue());
assertEquals("marko", g.V().has("name", "marko").values("name").next());
assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
- assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue());
////
Spark.close();
}
@@ -275,7 +273,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(6l, g.V().count().next().longValue());
assertEquals(6l, g.E().count().next().longValue());
assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
- assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue());
////
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertEquals(1, Spark.getContext().getPersistentRDDs().size());
@@ -291,7 +288,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(6l, g.V().count().next().longValue());
assertEquals(6l, g.E().count().next().longValue());
assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
- assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue());
////
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName2)));
@@ -304,7 +300,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(6l, g.V().count().next().longValue());
assertEquals(0l, g.E().count().next().longValue());
assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
- assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue());
////
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName2)));
@@ -318,7 +313,6 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest {
assertEquals(0l, g.V().count().next().longValue());
assertEquals(0l, g.E().count().next().longValue());
assertEquals(0l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
- assertEquals(0l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue());
////
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertFalse(Spark.hasRDD(Constants.getGraphLocation(rddName2)));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b03c1adc/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index fe5dd8d..5fb477c 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -136,9 +136,10 @@ public final class TinkerGraphComputer implements GraphComputer {
this.memory = new TinkerMemory(this.vertexProgram, this.mapReducers);
return CompletableFuture.<ComputerResult>supplyAsync(() -> {
final long time = System.currentTimeMillis();
+ final TinkerGraphComputerView view;
try (final TinkerWorkerPool workers = new TinkerWorkerPool(this.workers)) {
if (null != this.vertexProgram) {
- TinkerHelper.createGraphComputerView(this.graph, this.graphFilter, this.vertexProgram.getVertexComputeKeys());
+ view = TinkerHelper.createGraphComputerView(this.graph, this.graphFilter, this.vertexProgram.getVertexComputeKeys());
// execute the vertex program
this.vertexProgram.setup(this.memory);
while (true) {
@@ -167,9 +168,10 @@ public final class TinkerGraphComputer implements GraphComputer {
this.memory.incrIteration();
}
}
+ view.complete(); // drop all transient vertex compute keys
} else {
// MapReduce only
- TinkerHelper.createGraphComputerView(this.graph, this.graphFilter, Collections.emptySet());
+ view = TinkerHelper.createGraphComputerView(this.graph, this.graphFilter, Collections.emptySet());
}
// execute mapreduce jobs
@@ -212,9 +214,7 @@ public final class TinkerGraphComputer implements GraphComputer {
this.memory.setRuntime(System.currentTimeMillis() - time);
this.memory.complete(); // drop all transient properties and set iteration
// determine the resultant graph based on the result graph/persist state
- final TinkerGraphComputerView view = TinkerHelper.getGraphComputerView(this.graph); // can return a null view if no vertexprogram is used
- if (null != view) view.complete(); // drop all transient properties
- final Graph resultGraph = null == view ? this.graph : view.processResultGraphPersist(this.resultGraph, this.persist);
+ final Graph resultGraph = view.processResultGraphPersist(this.resultGraph, this.persist);
TinkerHelper.dropGraphComputerView(this.graph); // drop the view from the original source graph
return new DefaultComputerResult(resultGraph, this.memory.asImmutable());