You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/04 15:41:43 UTC
[1/8] incubator-tinkerpop git commit: Make the handleIterator method
throw a checked exception.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/spark 3b3ddb4d4 -> e0e08ebaa
Make the handleIterator method throw a checked exception.
This will make the expected error handling more explicity to those extending the AbstractEvalOpProcessor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6efdbca7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6efdbca7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6efdbca7
Branch: refs/heads/spark
Commit: 6efdbca732a4e7ed1984753ac87037d25bf534bf
Parents: 332f812
Author: Stephen Mallette <sp...@apache.org>
Authored: Wed Mar 4 07:12:37 2015 -0500
Committer: Stephen Mallette <sp...@apache.org>
Committed: Wed Mar 4 07:12:37 2015 -0500
----------------------------------------------------------------------
.../gremlin/server/op/AbstractEvalOpProcessor.java | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6efdbca7/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index 9258b35..2b0ab32 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -158,7 +158,11 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
stopWatch.start();
- handleIterator(ctx, msg, settings, itty, resultIterationBatchSize, stopWatch);
+ try {
+ handleIterator(ctx, msg, settings, itty, resultIterationBatchSize, stopWatch);
+ } catch (TimeoutException te) {
+ throw new RuntimeException(te);
+ }
stopWatch.stop();
}, executor);
@@ -177,7 +181,13 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
}, executor);
}
- protected void handleIterator(final ChannelHandlerContext ctx, final RequestMessage msg, final Settings settings, final Iterator itty, final int resultIterationBatchSize, final StopWatch stopWatch) {
+ /**
+ * Called by {@link #evalOpInternal} when iterating a result set.
+ *
+ * @throws TimeoutException if the time taken to serialize the entire result set exceeds the allowable time.
+ */
+ protected void handleIterator(final ChannelHandlerContext ctx, final RequestMessage msg, final Settings settings,
+ final Iterator itty, final int resultIterationBatchSize, final StopWatch stopWatch) throws TimeoutException {
List<Object> aggregate = new ArrayList<>(resultIterationBatchSize);
while (itty.hasNext()) {
aggregate.add(itty.next());
@@ -193,7 +203,7 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
stopWatch.split();
if (stopWatch.getSplitTime() > settings.serializedResponseTimeout)
- throw new RuntimeException(new TimeoutException("Serialization of the entire response exceeded the serializeResponseTimeout setting"));
+ throw new TimeoutException("Serialization of the entire response exceeded the serializeResponseTimeout setting");
stopWatch.unsplit();
}
[7/8] incubator-tinkerpop git commit: random tweaks. about to merge
master. will work from there now.
Posted by ok...@apache.org.
random tweaks. about to merge master. will work from there now.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/76a75ee1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/76a75ee1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/76a75ee1
Branch: refs/heads/spark
Commit: 76a75ee147d8664ae557ac3e6005db742bbc2f55
Parents: 3b3ddb4
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 4 07:41:18 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 4 07:41:18 2015 -0700
----------------------------------------------------------------------
hadoop-gremlin/conf/spark-gryo.properties | 5 +----
.../hadoop/process/computer/spark/SparkGraphProvider.java | 1 +
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/76a75ee1/hadoop-gremlin/conf/spark-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/spark-gryo.properties b/hadoop-gremlin/conf/spark-gryo.properties
index a25482a..bfaa6ee 100644
--- a/hadoop-gremlin/conf/spark-gryo.properties
+++ b/hadoop-gremlin/conf/spark-gryo.properties
@@ -29,10 +29,7 @@ gremlin.hadoop.inputLocation=hdfs://localhost:9000/user/marko/tinkerpop-modern-v
gremlin.hadoop.outputLocation=output
# the vertex program to execute
-# gremlin.vertexProgram=org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram
-gremlin.vertexProgram=org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram
-gremlin.traversalVertexProgram.traversalSupplier.type=CLASS
-gremlin.traversalVertexProgram.traversalSupplier.object=org.apache.tinkerpop.gremlin.hadoop.process.computer.example.TraversalSupplier1
+gremlin.vertexProgram=org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram
# It is possible to provide Spark configuration parameters for use with SparkGraphComputer
##########################################################################################
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/76a75ee1/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphProvider.java
index 255b0a5..6a1b47f 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphProvider.java
@@ -47,6 +47,7 @@ public final class SparkGraphProvider extends HadoopGraphProvider {
///////////
put(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
put("spark.master", "local[4]");
+ put("spark.serializer","org.apache.spark.serializer.JavaSerializer");
}};
}
}
[3/8] incubator-tinkerpop git commit: Moved stopwatch functionality
into handleIterator.
Posted by ok...@apache.org.
Moved stopwatch functionality into handleIterator.
It is now completely the responsibility of the implementer to break result iteration if it exceeds serialization time. A good implementation will do that.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/14c54a55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/14c54a55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/14c54a55
Branch: refs/heads/spark
Commit: 14c54a5528ac05f2527707135f57528935f9c34f
Parents: 30e0604
Author: Stephen Mallette <sp...@apache.org>
Authored: Wed Mar 4 07:54:05 2015 -0500
Committer: Stephen Mallette <sp...@apache.org>
Committed: Wed Mar 4 07:54:05 2015 -0500
----------------------------------------------------------------------
.../server/op/AbstractEvalOpProcessor.java | 24 ++++++++++++--------
1 file changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/14c54a55/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index e6dd7de..a353e6d 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -148,20 +148,13 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
final CompletableFuture<Void> iterationFuture = evalFuture.thenAcceptAsync(o -> {
final Iterator itty = IteratorUtils.convertToIterator(o);
- // timer for the total serialization time
- final StopWatch stopWatch = new StopWatch();
-
logger.debug("Preparing to iterate results from - {} - in thread [{}]", msg, Thread.currentThread().getName());
- stopWatch.start();
-
try {
- handleIterator(context, itty, stopWatch);
+ handleIterator(context, itty);
} catch (TimeoutException te) {
throw new RuntimeException(te);
}
-
- stopWatch.stop();
}, executor);
iterationFuture.handleAsync((r, ex) -> {
@@ -179,15 +172,24 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
}
/**
- * Called by {@link #evalOpInternal} when iterating a result set.
+ * Called by {@link #evalOpInternal} when iterating a result set. Implementers should respect the
+ * {@link Settings#serializedResponseTimeout} configuration and break the serialization process if
+ * it begins to take too long to do so, throwing a {@link java.util.concurrent.TimeoutException} in such
+ * cases.
*
+ * @param context The Gremlin Server {@link Context} object containing settings, request message, etc.
+ * @param itty The result to iterator
* @throws TimeoutException if the time taken to serialize the entire result set exceeds the allowable time.
*/
- protected void handleIterator(final Context context, final Iterator itty, final StopWatch stopWatch) throws TimeoutException {
+ protected void handleIterator(final Context context, final Iterator itty) throws TimeoutException {
final ChannelHandlerContext ctx = context.getChannelHandlerContext();
final RequestMessage msg = context.getRequestMessage();
final Settings settings = context.getSettings();
+ // timer for the total serialization time
+ final StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+
// the batch size can be overridden by the request
final int resultIterationBatchSize = (Integer) msg.optionalArgs(Tokens.ARGS_BATCH_SIZE).orElse(settings.resultIterationBatchSize);
List<Object> aggregate = new ArrayList<>(resultIterationBatchSize);
@@ -209,5 +211,7 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
stopWatch.unsplit();
}
+
+ stopWatch.stop();
}
}
[8/8] incubator-tinkerpop git commit: Merge branch 'master' into spark
Posted by ok...@apache.org.
Merge branch 'master' into spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e0e08eba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e0e08eba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e0e08eba
Branch: refs/heads/spark
Commit: e0e08ebaaa2b0713f5c42f5cc1f742b97d6821ce
Parents: 76a75ee 2eb3dba
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 4 07:41:32 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 4 07:41:32 2015 -0700
----------------------------------------------------------------------
.../server/op/AbstractEvalOpProcessor.java | 46 ++++++++++++++------
.../server/op/session/SessionOpProcessor.java | 2 +-
2 files changed, 33 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[5/8] incubator-tinkerpop git commit: Make SessionOpProcessor evalOp
protected.
Posted by ok...@apache.org.
Make SessionOpProcessor evalOp protected.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4f6301fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4f6301fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4f6301fa
Branch: refs/heads/spark
Commit: 4f6301fa92124a50cad75f2fbbb2e595eaa35781
Parents: 14c54a5
Author: rjbriody <bo...@datastax.com>
Authored: Wed Mar 4 09:26:33 2015 -0500
Committer: rjbriody <bo...@datastax.com>
Committed: Wed Mar 4 09:26:33 2015 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/server/op/session/SessionOpProcessor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4f6301fa/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
index 33617df..5a56e6b 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
@@ -101,7 +101,7 @@ public class SessionOpProcessor extends AbstractEvalOpProcessor {
return Optional.empty();
}
- private void evalOp(final Context context) throws OpProcessorException {
+ protected void evalOp(final Context context) throws OpProcessorException {
final RequestMessage msg = context.getRequestMessage();
final Session session = getSession(context, msg);
[2/8] incubator-tinkerpop git commit: Simplify the method signature
of handleIterator.
Posted by ok...@apache.org.
Simplify the method signature of handleIterator.
Reduced signature to just three parameters and at the same time passed the all important Gremlin Server Context object to allow maximum flexibility for those extending the class.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/30e0604d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/30e0604d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/30e0604d
Branch: refs/heads/spark
Commit: 30e0604d2b2fe838dd96a8e4cb88ebf1e8f01f71
Parents: 6efdbca
Author: Stephen Mallette <sp...@apache.org>
Authored: Wed Mar 4 07:23:16 2015 -0500
Committer: Stephen Mallette <sp...@apache.org>
Committed: Wed Mar 4 07:23:16 2015 -0500
----------------------------------------------------------------------
.../gremlin/server/op/AbstractEvalOpProcessor.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/30e0604d/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index 2b0ab32..e6dd7de 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -130,7 +130,6 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
final Timer.Context timerContext = evalOpTimer.time();
final ChannelHandlerContext ctx = context.getChannelHandlerContext();
final RequestMessage msg = context.getRequestMessage();
- final Settings settings = context.getSettings();
final GremlinExecutor gremlinExecutor = gremlinExecutorSupplier.get();
final ExecutorService executor = gremlinExecutor.getExecutorService();
@@ -148,8 +147,6 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
final CompletableFuture<Void> iterationFuture = evalFuture.thenAcceptAsync(o -> {
final Iterator itty = IteratorUtils.convertToIterator(o);
- // the batch size can be overridden by the request
- final int resultIterationBatchSize = (Integer) msg.optionalArgs(Tokens.ARGS_BATCH_SIZE).orElse(settings.resultIterationBatchSize);
// timer for the total serialization time
final StopWatch stopWatch = new StopWatch();
@@ -159,7 +156,7 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
stopWatch.start();
try {
- handleIterator(ctx, msg, settings, itty, resultIterationBatchSize, stopWatch);
+ handleIterator(context, itty, stopWatch);
} catch (TimeoutException te) {
throw new RuntimeException(te);
}
@@ -186,8 +183,13 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
*
* @throws TimeoutException if the time taken to serialize the entire result set exceeds the allowable time.
*/
- protected void handleIterator(final ChannelHandlerContext ctx, final RequestMessage msg, final Settings settings,
- final Iterator itty, final int resultIterationBatchSize, final StopWatch stopWatch) throws TimeoutException {
+ protected void handleIterator(final Context context, final Iterator itty, final StopWatch stopWatch) throws TimeoutException {
+ final ChannelHandlerContext ctx = context.getChannelHandlerContext();
+ final RequestMessage msg = context.getRequestMessage();
+ final Settings settings = context.getSettings();
+
+ // the batch size can be overridden by the request
+ final int resultIterationBatchSize = (Integer) msg.optionalArgs(Tokens.ARGS_BATCH_SIZE).orElse(settings.resultIterationBatchSize);
List<Object> aggregate = new ArrayList<>(resultIterationBatchSize);
while (itty.hasNext()) {
aggregate.add(itty.next());
[6/8] incubator-tinkerpop git commit: Merge branch 'master' of
https://github.com/rjbriody/incubator-tinkerpop
Posted by ok...@apache.org.
Merge branch 'master' of https://github.com/rjbriody/incubator-tinkerpop
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2eb3dba9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2eb3dba9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2eb3dba9
Branch: refs/heads/spark
Commit: 2eb3dba95af5b3d14e7d0f326cdbdc7e2f383d9e
Parents: d2884ba 4f6301f
Author: Stephen Mallette <sp...@apache.org>
Authored: Wed Mar 4 09:30:24 2015 -0500
Committer: Stephen Mallette <sp...@apache.org>
Committed: Wed Mar 4 09:30:24 2015 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/server/op/session/SessionOpProcessor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[4/8] incubator-tinkerpop git commit: Update javadoc a bit.
Posted by ok...@apache.org.
Update javadoc a bit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d2884ba7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d2884ba7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d2884ba7
Branch: refs/heads/spark
Commit: d2884ba7b1b00790ce4fcb67e6ae76e2e5a9a3de
Parents: 14c54a5
Author: Stephen Mallette <sp...@apache.org>
Authored: Wed Mar 4 09:26:00 2015 -0500
Committer: Stephen Mallette <sp...@apache.org>
Committed: Wed Mar 4 09:26:00 2015 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d2884ba7/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index a353e6d..8c335f3 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -117,7 +117,9 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
/**
* A generalized implementation of the "eval" operation. It handles script evaluation and iteration of results
* so as to write {@link ResponseMessage} objects down the Netty pipeline. It also handles script timeouts,
- * iteration timeouts, metrics and building bindings.
+ * iteration timeouts, metrics and building bindings. Note that result iteration is delegated to the
+ * {@link #handleIterator} method, so those extending this class could override that method for better control
+ * over result iteration.
*
* @param context The current Gremlin Server {@link Context}
* @param gremlinExecutorSupplier A function that returns the {@link GremlinExecutor} to use in executing the