You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/09/30 04:46:45 UTC
incubator-tinkerpop git commit: GraphComputer test that verifies that
workers are created and destroyed from 0 to 10 workers. This can't be done in
Giraph past 1 because of local-test-mode. In Spark,
the dataset is too small and only 1 worker is chosen.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/thread-issue-tinkergraph c160c3310 -> 641540838
GraphComputer test that verifies that workers are created and destroyed from 0 to 10 workers. This can't be done in Giraph past 1 because of local-test-mode. In Spark, the dataset is too small and only 1 worker is chosen. I OPT_OUT right now in HadoopGraph but I think I can get Giraph working (for 1 worker) and forcing Spark to spawn workers (maybe).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/64154083
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/64154083
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/64154083
Branch: refs/heads/thread-issue-tinkergraph
Commit: 6415408380b1dcbd3baa8d98ee526bf97ac34056
Parents: c160c33
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 29 20:46:34 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 29 20:46:34 2015 -0600
----------------------------------------------------------------------
.../gremlin/process/computer/GraphComputer.java | 4 ++--
.../process/computer/GraphComputerTest.java | 17 ++++++++---------
.../gremlin/hadoop/structure/HadoopGraph.java | 4 ++++
3 files changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/64154083/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
index 75c72df..18d4f74 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
@@ -117,8 +117,8 @@ public interface GraphComputer {
public interface Features {
- public default boolean supportsWorkerSpecification() {
- return true;
+ public default boolean supportsWorkerCount(int workers) {
+ return workers != 0;
}
public default boolean supportsGlobalMessageScopes() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/64154083/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
index 2e3a3d8..b859f4b 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
@@ -42,7 +42,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.GRATEFUL;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
@@ -1372,19 +1371,19 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(GRATEFUL)
public void shouldSupportWorkerCount() throws Exception {
- final GraphComputer computer = graph.compute(graphComputerClass.get());
- if (computer.features().supportsWorkerSpecification()) {
- ComputerResult result = computer.program(new VertexProgramL()).workers(1).submit().get();
- assertEquals(1l, (long) result.memory().get("workerCount"));
- ////
- result = graph.compute(graphComputerClass.get()).program(new VertexProgramL()).workers(2).submit().get();
- assertEquals(2l, (long) result.memory().get("workerCount"));
+ assertFalse(graph.compute(graphComputerClass.get()).features().supportsWorkerCount(0));
+ for (int i = 0; i < 10; i++) { // the GraphComputer should not support 0 workers
+ final GraphComputer computer = graph.compute(graphComputerClass.get());
+ if (computer.features().supportsWorkerCount(i)) {
+ ComputerResult result = computer.program(new VertexProgramL()).workers(i).submit().get();
+ assertEquals(Integer.valueOf(i).longValue(), (long) result.memory().get("workerCount"));
+ }
}
}
public static class VertexProgramL implements VertexProgram {
- final Set<String> threadIds = new ConcurrentSkipListSet<>();
+ final Set<String> threadIds = new HashSet<>();
@Override
public void setup(final Memory memory) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/64154083/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 3c6cbb0..7927a15 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -153,6 +153,10 @@ import java.util.stream.Stream;
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyProfileTest$Traversals",
method = "g_V_out_out_profile_grateful",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
+ method = "shouldSupportWorkerCount",
+ reason = "It is not possible to control Spark and Giraph worker counts in integration testing.")
public final class HadoopGraph implements Graph {
public static final Logger LOGGER = LoggerFactory.getLogger(HadoopGraph.class);