You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2016/02/25 15:00:49 UTC
[01/43] incubator-tinkerpop git commit: More fixes for in-session
binding concurrency. [Forced Update!]
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1107 9c78b807c -> 73489b3a8 (forced update)
More fixes for in-session binding concurrency.
Tried to again ensure that multiple threads don't act on bindings at the same time. Fixed a logging message for session close that was logging a rollback that may not have actually happened.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a5a00bf1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a5a00bf1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a5a00bf1
Branch: refs/heads/TINKERPOP-1107
Commit: a5a00bf1c42c286b5c1d042003c2521f7dc317d3
Parents: 888ac6a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Feb 15 16:23:38 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Feb 16 12:17:14 2016 -0500
----------------------------------------------------------------------
.../server/op/AbstractEvalOpProcessor.java | 49 ++++++++++++--------
.../gremlin/server/op/session/Session.java | 6 ++-
.../server/op/session/SessionOpProcessor.java | 4 +-
.../server/GremlinServerIntegrateTest.java | 27 +++++++++++
4 files changed, 64 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a5a00bf1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index 1e37fa0..b8f48eb 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -46,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.Bindings;
+import javax.script.SimpleBindings;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -188,30 +189,40 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
final String script = (String) args.get(Tokens.ARGS_GREMLIN);
final String language = args.containsKey(Tokens.ARGS_LANGUAGE) ? (String) args.get(Tokens.ARGS_LANGUAGE) : null;
- final Bindings bindings = bindingsSupplier.get();
+ final Bindings bindings = new SimpleBindings();
// sessionless requests are always transaction managed, but in-session requests are configurable.
final boolean managedTransactionsForRequest = manageTransactions ?
true : (Boolean) args.getOrDefault(Tokens.ARGS_MANAGE_TRANSACTION, false);
- final CompletableFuture<Object> evalFuture = gremlinExecutor.eval(script, language, bindings, null, o -> {
- final Iterator itty = IteratorUtils.asIterator(o);
-
- logger.debug("Preparing to iterate results from - {} - in thread [{}]", msg, Thread.currentThread().getName());
-
- try {
- handleIterator(context, itty);
- } catch (TimeoutException ex) {
- final String errorMessage = String.format("Response iteration exceeded the configured threshold for request [%s] - %s", msg, ex.getMessage());
- logger.warn(errorMessage);
- ctx.writeAndFlush(ResponseMessage.build(msg).code(ResponseStatusCode.SERVER_ERROR_TIMEOUT).statusMessage(errorMessage).create());
- if (managedTransactionsForRequest) attemptRollback(msg, context.getGraphManager(), settings.strictTransactionManagement);
- } catch (Exception ex) {
- logger.warn(String.format("Exception processing a script on request [%s].", msg), ex);
- ctx.writeAndFlush(ResponseMessage.build(msg).code(ResponseStatusCode.SERVER_ERROR).statusMessage(ex.getMessage()).create());
- if (managedTransactionsForRequest) attemptRollback(msg, context.getGraphManager(), settings.strictTransactionManagement);
- }
- });
+ final GremlinExecutor.LifeCycle lifeCycle = GremlinExecutor.LifeCycle.build()
+ .beforeEval(b -> {
+ try {
+ b.putAll(bindingsSupplier.get());
+ } catch (OpProcessorException ope) {
+ ope.printStackTrace();
+ }
+ })
+ .withResult(o -> {
+ final Iterator itty = IteratorUtils.asIterator(o);
+
+ logger.debug("Preparing to iterate results from - {} - in thread [{}]", msg, Thread.currentThread().getName());
+
+ try {
+ handleIterator(context, itty);
+ } catch (TimeoutException ex) {
+ final String errorMessage = String.format("Response iteration exceeded the configured threshold for request [%s] - %s", msg, ex.getMessage());
+ logger.warn(errorMessage);
+ ctx.writeAndFlush(ResponseMessage.build(msg).code(ResponseStatusCode.SERVER_ERROR_TIMEOUT).statusMessage(errorMessage).create());
+ if (managedTransactionsForRequest) attemptRollback(msg, context.getGraphManager(), settings.strictTransactionManagement);
+ } catch (Exception ex) {
+ logger.warn(String.format("Exception processing a script on request [%s].", msg), ex);
+ ctx.writeAndFlush(ResponseMessage.build(msg).code(ResponseStatusCode.SERVER_ERROR).statusMessage(ex.getMessage()).create());
+ if (managedTransactionsForRequest) attemptRollback(msg, context.getGraphManager(), settings.strictTransactionManagement);
+ }
+ }).create();
+
+ final CompletableFuture<Object> evalFuture = gremlinExecutor.eval(script, language, bindings, lifeCycle);
evalFuture.handle((v, t) -> {
timerContext.stop();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a5a00bf1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
index 70fa593..c29b45a 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
@@ -122,8 +122,10 @@ public class Session {
// that thread of execution from this session
try {
executor.submit(() -> {
- logger.info("Rolling back open transactions on {} before killing session: {}", kv.getKey(), session);
- if (g.tx().isOpen()) g.tx().rollback();
+ if (g.tx().isOpen()) {
+ logger.info("Rolling back open transactions on {} before killing session: {}", kv.getKey(), session);
+ g.tx().rollback();
+ }
}).get(30000, TimeUnit.MILLISECONDS);
} catch (Exception ex) {
logger.warn("An error occurred while attempting rollback when closing session: " + session, ex);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a5a00bf1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
index d5e5e7c..b7e0d06 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/SessionOpProcessor.java
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.script.Bindings;
+import javax.script.SimpleBindings;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@@ -149,7 +150,8 @@ public class SessionOpProcessor extends AbstractEvalOpProcessor {
context.getChannelHandlerContext().channel().attr(StateKey.SESSION).set(session);
evalOpInternal(context, session::getGremlinExecutor, () -> {
- final Bindings bindings = session.getBindings();
+ final Bindings bindings = new SimpleBindings();
+ bindings.putAll(session.getBindings());
// parameter bindings override session bindings if present
Optional.ofNullable((Map<String, Object>) msg.getArgs().get(Tokens.ARGS_BINDINGS)).ifPresent(bindings::putAll);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a5a00bf1/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
index 6ca7dff..7714a93 100644
--- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
+++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
@@ -780,4 +780,31 @@ public class GremlinServerIntegrateTest extends AbstractGremlinServerIntegration
assertEquals("jason", v.value("name"));
}
}
+
+ @Test
+ public void shouldEnsureSessionBindingsAreThreadSafe() throws Exception {
+ final Cluster cluster = Cluster.open();
+ final Client client = cluster.connect(name.getMethodName());
+
+ client.submitAsync("a=100;b=1000;c=10000;null");
+ final int requests = 1000;
+ final List<CompletableFuture<ResultSet>> futures = new ArrayList<>(requests);
+ IntStream.range(0, requests).forEach(i -> {
+ try {
+ futures.add(client.submitAsync("a+b+c"));
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ });
+
+ assertEquals(requests, futures.size());
+
+ for(CompletableFuture<ResultSet> f : futures) {
+ final Result r = f.get().one();
+ assertEquals(11100, r.getInt());
+ }
+
+ client.close();
+ cluster.close();
+ }
}
[07/43] incubator-tinkerpop git commit: splitted
implementations.asciidoc into implementations-hadoop.asciidoc and
implementations-neo4j.asciidoc
Posted by sp...@apache.org.
splitted implementations.asciidoc into implementations-hadoop.asciidoc and implementations-neo4j.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bbf5b3f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bbf5b3f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bbf5b3f4
Branch: refs/heads/TINKERPOP-1107
Commit: bbf5b3f4d61c0aa2266b7430a1e083e8f4c01920
Parents: 70a3065
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Thu Feb 18 23:52:55 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Thu Feb 18 23:52:55 2016 +0100
----------------------------------------------------------------------
.../reference/implementations-hadoop.asciidoc | 929 +++++++++
.../reference/implementations-neo4j.asciidoc | 921 +++++++++
docs/src/reference/implementations.asciidoc | 1835 ------------------
docs/src/reference/index.asciidoc | 3 +-
4 files changed, 1852 insertions(+), 1836 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bbf5b3f4/docs/src/reference/implementations-hadoop.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-hadoop.asciidoc b/docs/src/reference/implementations-hadoop.asciidoc
new file mode 100644
index 0000000..376f377
--- /dev/null
+++ b/docs/src/reference/implementations-hadoop.asciidoc
@@ -0,0 +1,929 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+[[hadoop-gremlin]]
+Hadoop-Gremlin
+--------------
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>hadoop-gremlin</artifactId>
+ <version>x.y.z</version>
+</dependency>
+----
+
+image:hadoop-logo-notext.png[width=100,float=left] link:http://hadoop.apache.org/[Hadoop] is a distributed
+computing framework that is used to process data represented across a multi-machine compute cluster. When the
+data in the Hadoop cluster represents a TinkerPop3 graph, then Hadoop-Gremlin can be used to process the graph
+using both TinkerPop3's OLTP and OLAP graph computing models.
+
+IMPORTANT: This section assumes that the user has a Hadoop 2.x cluster functioning. For more information on getting
+started with Hadoop, please see the
+link:http://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-common/SingleCluster.html[Single Node Setup]
+tutorial. Moreover, if using `GiraphGraphComputer` or `SparkGraphComputer` it is advisable that the reader also
+familiarize their self with Giraph (link:http://giraph.apache.org/quick_start.html[Getting Started]) and Spark
+(link:http://spark.apache.org/docs/latest/quick-start.html[Quick Start]).
+
+Installing Hadoop-Gremlin
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `HADOOP_GREMLIN_LIBS` references locations that contains jars that should be uploaded to a respective
+distributed cache (link:http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html[YARN] or SparkServer).
+Note that the locations in `HADOOP_GREMLIN_LIBS` can be a colon-separated (`:`) and all jars from all locations will
+be loaded into the cluster. Typically, only the jars of the respective GraphComputer are required to be loaded (e.g.
+`GiraphGraphComputer` plugin lib directory).
+
+[source,shell]
+export HADOOP_GREMLIN_LIBS=/usr/local/gremlin-console/ext/giraph-gremlin/lib
+
+If using <<gremlin-console,Gremlin Console>>, it is important to install the Hadoop-Gremlin plugin. Note that
+Hadoop-Gremlin requires a Gremlin Console restart after installing.
+
+[source,text]
+----
+$ bin/gremlin.sh
+
+ \,,,/
+ (o o)
+-----oOOo-(3)-oOOo-----
+plugin activated: tinkerpop.server
+plugin activated: tinkerpop.utilities
+plugin activated: tinkerpop.tinkergraph
+gremlin> :install org.apache.tinkerpop hadoop-gremlin x.y.z
+==>loaded: [org.apache.tinkerpop, hadoop-gremlin, x.y.z] - restart the console to use [tinkerpop.hadoop]
+gremlin> :q
+$ bin/gremlin.sh
+
+ \,,,/
+ (o o)
+-----oOOo-(3)-oOOo-----
+plugin activated: tinkerpop.server
+plugin activated: tinkerpop.utilities
+plugin activated: tinkerpop.tinkergraph
+gremlin> :plugin use tinkerpop.hadoop
+==>tinkerpop.hadoop activated
+gremlin>
+----
+
+Properties Files
+~~~~~~~~~~~~~~~~
+
+`HadoopGraph` makes use of properties files which ultimately get turned into Apache configurations and/or
+Hadoop configurations. The example properties file presented below is located at `conf/hadoop/hadoop-gryo.properties`.
+
+[source,text]
+gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
+gremlin.hadoop.inputLocation=tinkerpop-modern.kryo
+gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
+gremlin.hadoop.outputLocation=output
+gremlin.hadoop.graphOutputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
+gremlin.hadoop.jarsInDistributedCache=true
+####################################
+# Spark Configuration #
+####################################
+spark.master=local[4]
+spark.executor.memory=1g
+spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
+####################################
+# SparkGraphComputer Configuration #
+####################################
+gremlin.spark.graphInputRDD=org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat
+gremlin.spark.graphOutputRDD=org.apache.tinkerpop.gremlin.spark.structure.io.OutputRDDFormat
+gremlin.spark.persistContext=true
+#####################################
+# GiraphGraphComputer Configuration #
+#####################################
+giraph.minWorkers=2
+giraph.maxWorkers=2
+giraph.useOutOfCoreGraph=true
+giraph.useOutOfCoreMessages=true
+mapreduce.map.java.opts=-Xmx1024m
+mapreduce.reduce.java.opts=-Xmx1024m
+giraph.numInputThreads=2
+giraph.numComputeThreads=2
+
+A review of the Hadoop-Gremlin specific properties are provided in the table below. For the respective OLAP
+engines (<<sparkgraphcomputer,`SparkGraphComputer`>> or <<giraphgraphcomputer,`GiraphGraphComputer`>>) refer
+to their respective documentation for configuration options.
+
+[width="100%",cols="2,10",options="header"]
+|=========================================================
+|Property |Description
+|gremlin.graph |The class of the graph to construct using GraphFactory.
+|gremlin.hadoop.inputLocation |The location of the input file(s) for Hadoop-Gremlin to read the graph from.
+|gremlin.hadoop.graphInputFormat |The format that the graph input file(s) are represented in.
+|gremlin.hadoop.outputLocation |The location to write the computed HadoopGraph to.
+|gremlin.hadoop.graphOutputFormat |The format that the output file(s) should be represented in.
+|gremlin.hadoop.jarsInDistributedCache |Whether to upload the Hadoop-Gremlin jars to a distributed cache (necessary if jars are not on the machines' classpaths).
+|=========================================================
+
+
+
+Along with the properties above, the numerous link:http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/core-default.xml[Hadoop specific properties]
+can be added as needed to tune and parameterize the executed Hadoop-Gremlin job on the respective Hadoop cluster.
+
+IMPORTANT: As the size of the graphs being processed becomes large, it is important to fully understand how the
+underlying OLAP engine (e.g. Spark, Giraph, etc.) works and understand the numerous parameterizations offered by
+these systems. Such knowledge can help alleviate out of memory exceptions, slow load times, slow processing times,
+garbage collection issues, etc.
+
+OLTP Hadoop-Gremlin
+~~~~~~~~~~~~~~~~~~~
+
+image:hadoop-pipes.png[width=180,float=left] It is possible to execute OLTP operations over a `HadoopGraph`.
+However, realize that the underlying HDFS files are not random access and thus, to retrieve a vertex, a linear scan
+is required. OLTP operations are useful for peeking into the graph prior to executing a long running OLAP job -- e.g.
+`g.V().valueMap().limit(10)`.
+
+CAUTION: OLTP operations on `HadoopGraph` are not efficient. They require linear scans to execute and are unreasonable
+for large graphs. In such large graph situations, make use of <<traversalvertexprogram,TraversalVertexProgram>>
+which is the OLAP Gremlin machine.
+
+[gremlin-groovy]
+----
+hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo')
+hdfs.ls()
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+g = graph.traversal()
+g.V().count()
+g.V().out().out().values('name')
+g.V().group().by{it.value('name')[1]}.by('name').next()
+----
+
+OLAP Hadoop-Gremlin
+~~~~~~~~~~~~~~~~~~~
+
+image:hadoop-furnace.png[width=180,float=left] Hadoop-Gremlin was designed to execute OLAP operations via
+`GraphComputer`. The OLTP examples presented previously are reproduced below, but using `TraversalVertexProgram`
+for the execution of the Gremlin traversal.
+
+A `Graph` in TinkerPop3 can support any number of `GraphComputer` implementations. Out of the box, Hadoop-Gremlin
+supports the following three implementations.
+
+* <<mapreducegraphcomputer,`MapReduceGraphComputer`>>: Leverages Hadoop's MapReduce engine to execute TinkerPop3 OLAP
+computations. (*coming soon*)
+** The graph must fit within the total disk space of the Hadoop cluster (supports massive graphs). Message passing is
+coordinated via MapReduce jobs over the on-disk graph (slow traversals).
+* <<sparkgraphcomputer,`SparkGraphComputer`>>: Leverages Apache Spark to execute TinkerPop3 OLAP computations.
+** The graph may fit within the total RAM of the cluster (supports larger graphs). Message passing is coordinated via
+Spark map/reduce/join operations on in-memory and disk-cached data (average speed traversals).
+* <<giraphgraphcomputer,`GiraphGraphComputer`>>: Leverages Apache Giraph to execute TinkerPop3 OLAP computations.
+** The graph should fit within the total RAM of the Hadoop cluster (graph size restriction), though "out-of-core"
+processing is possible. Message passing is coordinated via ZooKeeper for the in-memory graph (speedy traversals).
+
+TIP: image:gremlin-sugar.png[width=50,float=left] For those wanting to use the <<sugar-plugin,SugarPlugin>> with
+their submitted traversal, do `:remote config useSugar true` as well as `:plugin use tinkerpop.sugar` at the start of
+the Gremlin Console session if it is not already activated.
+
+Note that `SparkGraphComputer` and `GiraphGraphComputer` are loaded via their respective plugins. Typically only
+one plugin or the other is loaded depending on the desired `GraphComputer` to use.
+
+[source,text]
+----
+$ bin/gremlin.sh
+
+ \,,,/
+ (o o)
+-----oOOo-(3)-oOOo-----
+plugin activated: tinkerpop.server
+plugin activated: tinkerpop.utilities
+plugin activated: tinkerpop.tinkergraph
+plugin activated: tinkerpop.hadoop
+gremlin> :install org.apache.tinkerpop giraph-gremlin x.y.z
+==>loaded: [org.apache.tinkerpop, giraph-gremlin, x.y.z] - restart the console to use [tinkerpop.giraph]
+gremlin> :install org.apache.tinkerpop spark-gremlin x.y.z
+==>loaded: [org.apache.tinkerpop, spark-gremlin, x.y.z] - restart the console to use [tinkerpop.spark]
+gremlin> :q
+$ bin/gremlin.sh
+
+ \,,,/
+ (o o)
+-----oOOo-(3)-oOOo-----
+plugin activated: tinkerpop.server
+plugin activated: tinkerpop.utilities
+plugin activated: tinkerpop.tinkergraph
+plugin activated: tinkerpop.hadoop
+gremlin> :plugin use tinkerpop.giraph
+==>tinkerpop.giraph activated
+gremlin> :plugin use tinkerpop.spark
+==>tinkerpop.spark activated
+----
+
+WARNING: Hadoop, Spark, and Giraph all depend on many of the same libraries (e.g. ZooKeeper, Snappy, Netty, Guava,
+etc.). Unfortunately, typically these dependencies are not to the same versions of the respective libraries. As such,
+it is best to *not* have both Spark and Giraph plugins loaded in the same console session nor in the same Java
+project (though intelligent `<exclusion>`-usage can help alleviate conflicts in a Java project).
+
+CAUTION: It is important to note that when doing an OLAP traversal, any resulting vertices, edges, or properties will be
+attached to the source graph. For Hadoop-based graphs, this may lead to linear search times on massive graphs. Thus,
+if vertex, edge, or property objects are to be returns (as a final result), it is best to `.id()` to get the id
+of the object and not the actual attached object.
+
+[[mapreducegraphcomputer]]
+MapReduceGraphComputer
+^^^^^^^^^^^^^^^^^^^^^^
+
+*COMING SOON*
+
+[[sparkgraphcomputer]]
+SparkGraphComputer
+^^^^^^^^^^^^^^^^^^
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>spark-gremlin</artifactId>
+ <version>x.y.z</version>
+</dependency>
+----
+
+image:spark-logo.png[width=175,float=left] link:http://spark.apache.org[Spark] is an Apache Software Foundation
+project focused on general-purpose OLAP data processing. Spark provides a hybrid in-memory/disk-based distributed
+computing model that is similar to Hadoop's MapReduce model. Spark maintains a fluent function chaining DSL that is
+arguably easier for developers to work with than native Hadoop MapReduce. Spark-Gremlin provides an implementation of
+the bulk-synchronous parallel, distributed message passing algorithm within Spark and thus, any `VertexProgram` can be
+executed over `SparkGraphComputer`.
+
+If `SparkGraphComputer` will be used as the `GraphComputer` for `HadoopGraph` then its `lib` directory should be
+specified in `HADOOP_GREMLIN_LIBS`.
+
+[source,shell]
+export HADOOP_GREMLIN_LIBS=$HADOOP_GREMLIN_LIBS:/usr/local/gremlin-console/ext/spark-gremlin/lib
+
+Furthermore the `lib/` directory should be distributed across all machines in the SparkServer cluster. For this purpose TinkerPop
+provides a helper script, which takes the Spark installation directory and the the Spark machines as input:
+
+[source,shell]
+bin/init-tp-spark.sh /usr/local/spark spark@10.0.0.1 spark@10.0.0.2 spark@10.0.0.3
+
+Once the `lib/` directory is distributed, `SparkGraphComputer` can be used as follows.
+
+[gremlin-groovy]
+----
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+g = graph.traversal(computer(SparkGraphComputer))
+g.V().count()
+g.V().out().out().values('name')
+----
+
+For using lambdas in Gremlin-Groovy, simply provide `:remote connect` a `TraversalSource` which leverages SparkGraphComputer.
+
+[gremlin-groovy]
+----
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+g = graph.traversal(computer(SparkGraphComputer))
+:remote connect tinkerpop.hadoop graph g
+:> g.V().group().by{it.value('name')[1]}.by('name')
+----
+
+The `SparkGraphComputer` algorithm leverages Spark's caching abilities to reduce the amount of data shuffled across
+the wire on each iteration of the <<vertexprogram,`VertexProgram`>>. When the graph is loaded as a Spark RDD
+(Resilient Distributed Dataset) it is immediately cached as `graphRDD`. The `graphRDD` is a distributed adjacency
+list which encodes the vertex, its properties, and all its incident edges. On the first iteration, each vertex
+(in parallel) is passed through `VertexProgram.execute()`. This yields an output of the vertex's mutated state
+(i.e. updated compute keys -- `propertyX`) and its outgoing messages. This `viewOutgoingRDD` is then reduced to
+`viewIncomingRDD` where the outgoing messages are sent to their respective vertices. If a `MessageCombiner` exists
+for the vertex program, then messages are aggregated locally and globally to ultimately yield one incoming message
+for the vertex. This reduce sequence is the "message pass." If the vertex program does not terminate on this
+iteration, then the `viewIncomingRDD` is joined with the cached `graphRDD` and the process continues. When there
+are no more iterations, there is a final join and the resultant RDD is stripped of its edges and messages. This
+`mapReduceRDD` is cached and is processed by each <<mapreduce,`MapReduce`>> job in the
+<<graphcomputer,`GraphComputer`>> computation.
+
+image::spark-algorithm.png[width=775]
+
+[width="100%",cols="2,10",options="header"]
+|========================================================
+|Property |Description
+|gremlin.spark.graphInputRDD |A class for creating RDD's from underlying graph data, defaults to Hadoop `InputFormat`.
+|gremlin.spark.graphOutputRDD |A class for output RDD's, defaults to Hadoop `OutputFormat`.
+|gremlin.spark.graphStorageLevel |What `StorageLevel` to use for the cached graph during job execution (default `MEMORY_ONLY`).
+|gremlin.spark.persistContext |Whether to create a new `SparkContext` for every `SparkGraphComputer` or to reuse an existing one.
+|gremlin.spark.persistStorageLevel |What `StorageLevel` to use when persisted RDDs via `PersistedOutputRDD` (default `MEMORY_ONLY`).
+|========================================================
+
+InputRDD and OutputRDD
+++++++++++++++++++++++
+
+If the provider/user does not want to use Hadoop `InputFormats`, it is possible to leverage Spark's RDD
+constructs directly. There is a `gremlin.spark.graphInputRDD` configuration that references a `Class<? extends
+InputRDD>`. An `InputRDD` provides a read method that takes a `SparkContext` and returns a graphRDD. Likewise, use
+`gremlin.spark.graphOutputRDD` and the respective `OutputRDD`.
+
+If the graph system provider uses an `InputRDD`, the RDD should maintain an associated `org.apache.spark.Partitioner`. By doing so,
+`SparkGraphComputer` will not partition the loaded graph across the cluster as it has already been partitioned by the graph system provider.
+This can save a significant amount of time and space resources.
+If the `InputRDD` does not have a registered partitioner, `SparkGraphComputer` will partition the graph using
+a `org.apache.spark.HashPartitioner` with the number of partitions being either the number of existing partitions in the input (e.g. input splits)
+or the user specified number of `GraphComputer.workers()`.
+
+Using a Persisted Context
++++++++++++++++++++++++++
+
+It is possible to persist the graph RDD between jobs within the `SparkContext` (e.g. SparkServer) by leveraging `PersistedOutputRDD`.
+Note that `gremlin.spark.persistContext` should be set to `true` or else the persisted RDD will be destroyed when the `SparkContext` closes.
+The persisted RDD is named by the `gremlin.hadoop.outputLocation` configuration. Similarly, `PersistedInputRDD` is used with respective
+`gremlin.hadoop.inputLocation` to retrieve the persisted RDD from the `SparkContext`.
+
+When using a persistent `SparkContext` the configuration used by the original Spark Configuration will be inherited by all threaded
+references to that Spark Context. The exception to this rule are those properties which have a specific thread local effect.
+
+.Thread Local Properties
+. spark.jobGroup.id
+. spark.job.description
+. spark.job.interruptOnCancel
+. spark.scheduler.pool
+
+Finally, there is a `spark` object that can be used to manage persisted RDDs (see <<interacting-with-spark, Interacting with Spark>>).
+
+[[bulkdumpervertexprogramusingspark]]
+Exporting with BulkDumperVertexProgram
+++++++++++++++++++++++++++++++++++++++
+
+The <<bulkdumpervertexprogram, BulkDumperVertexProgram>> exports a whole graph in any of the supported Hadoop GraphOutputFormats (`GraphSONOutputFormat`,
+`GryoOutputFormat` or `ScriptOutputFormat`). The example below takes a Hadoop graph as the input (in `GryoInputFormat`) and exports it as a GraphSON file
+(`GraphSONOutputFormat`).
+
+[gremlin-groovy]
+----
+hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo')
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+graph.configuration().setProperty('gremlin.hadoop.graphOutputFormat', 'org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat')
+graph.compute(SparkGraphComputer).program(BulkDumperVertexProgram.build().create()).submit().get()
+hdfs.ls('output')
+hdfs.head('output/~g')
+----
+
+Loading with BulkLoaderVertexProgram
+++++++++++++++++++++++++++++++++++++
+
+The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load large
+amounts of data to and from different `Graph` implementations. The following code demonstrates how to load the
+Grateful Dead graph from HadoopGraph into TinkerGraph over Spark:
+
+[gremlin-groovy]
+----
+hdfs.copyFromLocal('data/grateful-dead.kryo', 'grateful-dead.kryo')
+readGraph = GraphFactory.open('conf/hadoop/hadoop-grateful-gryo.properties')
+writeGraph = 'conf/tinkergraph-gryo.properties'
+blvp = BulkLoaderVertexProgram.build().
+ keepOriginalIds(false).
+ writeGraph(writeGraph).create(readGraph)
+readGraph.compute(SparkGraphComputer).workers(1).program(blvp).submit().get()
+:set max-iteration 10
+graph = GraphFactory.open(writeGraph)
+g = graph.traversal()
+g.V().valueMap()
+graph.close()
+----
+
+[source,properties]
+----
+# hadoop-grateful-gryo.properties
+
+#
+# Hadoop Graph Configuration
+#
+gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
+gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
+gremlin.hadoop.inputLocation=grateful-dead.kryo
+gremlin.hadoop.outputLocation=output
+gremlin.hadoop.jarsInDistributedCache=true
+
+#
+# SparkGraphComputer Configuration
+#
+spark.master=local[1]
+spark.executor.memory=1g
+spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
+----
+
+[source,properties]
+----
+# tinkergraph-gryo.properties
+
+gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
+gremlin.tinkergraph.graphFormat=gryo
+gremlin.tinkergraph.graphLocation=/tmp/tinkergraph.kryo
+----
+
+IMPORTANT: The path to TinkerGraph jars needs to be included in the `HADOOP_GREMLIN_LIBS` for the above example to work.
+
+[[giraphgraphcomputer]]
+GiraphGraphComputer
+^^^^^^^^^^^^^^^^^^^
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>giraph-gremlin</artifactId>
+ <version>x.y.z</version>
+</dependency>
+----
+
+image:giraph-logo.png[width=100,float=left] link:http://giraph.apache.org[Giraph] is an Apache Software Foundation
+project focused on OLAP-based graph processing. Giraph makes use of the distributed graph computing paradigm made
+popular by Google's Pregel. In Giraph, developers write "vertex programs" that get executed at each vertex in
+parallel. These programs communicate with one another in a bulk synchronous parallel (BSP) manner. This model aligns
+with TinkerPop3's `GraphComputer` API. TinkerPop3 provides an implementation of `GraphComputer` that works for Giraph
+called `GiraphGraphComputer`. Moreover, with TinkerPop3's <<mapreduce,MapReduce>>-framework, the standard
+Giraph/Pregel model is extended to support an arbitrary number of MapReduce phases to aggregate and yield results
+from the graph. Below are examples using `GiraphGraphComputer` from the <<gremlin-console,Gremlin-Console>>.
+
+WARNING: Giraph uses a large number of Hadoop counters. The default for Hadoop is 120. In `mapred-site.xml` it is
+possible to increase the limit it via the `mapreduce.job.counters.max` property. A good value to use is 1000. This
+is a cluster-wide property so be sure to restart the cluster after updating.
+
+WARNING: The maximum number of workers can be no larger than the number of map-slots in the Hadoop cluster minus 1.
+For example, if the Hadoop cluster has 4 map slots, then `giraph.maxWorkers` can not be larger than 3. One map-slot
+is reserved for the master compute node and all other slots can be allocated as workers to execute the VertexPrograms
+on the vertices of the graph.
+
+If `GiraphGraphComputer` will be used as the `GraphComputer` for `HadoopGraph` then its `lib` directory should be
+specified in `HADOOP_GREMLIN_LIBS`.
+
+[source,shell]
+export HADOOP_GREMLIN_LIBS=$HADOOP_GREMLIN_LIBS:/usr/local/gremlin-console/ext/giraph-gremlin/lib
+
+Or, the user can specify the directory in the Gremlin Console.
+
+[source,groovy]
+System.setProperty('HADOOP_GREMLIN_LIBS',System.getProperty('HADOOP_GREMLIN_LIBS') + ':' + '/usr/local/gremlin-console/ext/giraph-gremlin/lib')
+
+[gremlin-groovy]
+----
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+g = graph.traversal(computer(GiraphGraphComputer))
+g.V().count()
+g.V().out().out().values('name')
+----
+
+IMPORTANT: The examples above do not use lambdas (i.e. closures in Gremlin-Groovy). This makes the traversal
+serializable and thus, able to be distributed to all machines in the Hadoop cluster. If a lambda is required in a
+traversal, then the traversal must be sent as a `String` and compiled locally at each machine in the cluster. The
+following example demonstrates the `:remote` command which allows for submitting Gremlin traversals as a `String`.
+
+[gremlin-groovy]
+----
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+g = graph.traversal(computer(GiraphGraphComputer))
+:remote connect tinkerpop.hadoop graph g
+:> g.V().group().by{it.value('name')[1]}.by('name')
+result
+result.memory.runtime
+result.memory.keys()
+result.memory.get('~reducing')
+----
+
+NOTE: If the user explicitly specifies `giraph.maxWorkers` and/or `giraph.numComputeThreads` in the configuration,
+then these values will be used by Giraph. However, if these are not specified and the user never calls
+`GraphComputer.workers()` then `GiraphGraphComputer` will try to compute the number of workers/threads to use based
+on the cluster's profile.
+
+Loading with BulkLoaderVertexProgram
+++++++++++++++++++++++++++++++++++++
+
+The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load
+large amounts of data to and from different `Graph` implementations. The following code demonstrates how to load
+the Grateful Dead graph from HadoopGraph into TinkerGraph over Giraph:
+
+[gremlin-groovy]
+----
+hdfs.copyFromLocal('data/grateful-dead.kryo', 'grateful-dead.kryo')
+readGraph = GraphFactory.open('conf/hadoop/hadoop-grateful-gryo.properties')
+writeGraph = 'conf/tinkergraph-gryo.properties'
+blvp = BulkLoaderVertexProgram.build().
+ keepOriginalIds(false).
+ writeGraph(writeGraph).create(readGraph)
+readGraph.compute(GiraphGraphComputer).workers(1).program(blvp).submit().get()
+:set max-iteration 10
+graph = GraphFactory.open(writeGraph)
+g = graph.traversal()
+g.V().valueMap()
+graph.close()
+----
+
+[source,properties]
+----
+# hadoop-grateful-gryo.properties
+
+#
+# Hadoop Graph Configuration
+#
+gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
+gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
+gremlin.hadoop.graphOutputFormat=org.apache.hadoop.mapreduce.lib.output.NullOutputFormat
+gremlin.hadoop.inputLocation=grateful-dead.kryo
+gremlin.hadoop.outputLocation=output
+gremlin.hadoop.jarsInDistributedCache=true
+
+#
+# GiraphGraphComputer Configuration
+#
+giraph.minWorkers=1
+giraph.maxWorkers=1
+giraph.useOutOfCoreGraph=true
+giraph.useOutOfCoreMessages=true
+mapred.map.child.java.opts=-Xmx1024m
+mapred.reduce.child.java.opts=-Xmx1024m
+giraph.numInputThreads=4
+giraph.numComputeThreads=4
+giraph.maxMessagesInMemory=100000
+----
+
+[source,properties]
+----
+# tinkergraph-gryo.properties
+
+gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
+gremlin.tinkergraph.graphFormat=gryo
+gremlin.tinkergraph.graphLocation=/tmp/tinkergraph.kryo
+----
+
+NOTE: The path to TinkerGraph needs to be included in the `HADOOP_GREMLIN_LIBS` for the above example to work.
+
+Input/Output Formats
+~~~~~~~~~~~~~~~~~~~~
+
+image:adjacency-list.png[width=300,float=right] Hadoop-Gremlin provides various I/O formats -- i.e. Hadoop
+`InputFormat` and `OutputFormat`. All of the formats make use of an link:http://en.wikipedia.org/wiki/Adjacency_list[adjacency list]
+representation of the graph where each "row" represents a single vertex, its properties, and its incoming and
+outgoing edges.
+
+{empty} +
+
+[[gryo-io-format]]
+Gryo I/O Format
+^^^^^^^^^^^^^^^
+
+* **InputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat`
+* **OutputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat`
+
+<<gryo-reader-writer,Gryo>> is a binary graph format that leverages link:https://github.com/EsotericSoftware/kryo[Kryo]
+to make a compact, binary representation of a vertex. It is recommended that users leverage Gryo given its space/time
+savings over text-based representations.
+
+NOTE: The `GryoInputFormat` is splittable.
+
+[[graphson-io-format]]
+GraphSON I/O Format
+^^^^^^^^^^^^^^^^^^^
+
+* **InputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat`
+* **OutputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat`
+
+<<graphson-reader-writer,GraphSON>> is a JSON based graph format. GraphSON is a space-expensive graph format in that
+it is a text-based markup language. However, it is convenient for many developers to work with as its structure is
+simple (easy to create and parse).
+
+The data below represents an adjacency list representation of the classic TinkerGraph toy graph in GraphSON format.
+
+[source,json]
+----
+{"id":1,"label":"person","outE":{"created":[{"id":9,"inV":3,"properties":{"weight":0.4}}],"knows":[{"id":7,"inV":2,"properties":{"weight":0.5}},{"id":8,"inV":4,"properties":{"weight":1.0}}]},"properties":{"name":[{"id":0,"value":"marko"}],"age":[{"id":1,"value":29}]}}
+{"id":2,"label":"person","inE":{"knows":[{"id":7,"outV":1,"properties":{"weight":0.5}}]},"properties":{"name":[{"id":2,"value":"vadas"}],"age":[{"id":3,"value":27}]}}
+{"id":3,"label":"software","inE":{"created":[{"id":9,"outV":1,"properties":{"weight":0.4}},{"id":11,"outV":4,"properties":{"weight":0.4}},{"id":12,"outV":6,"properties":{"weight":0.2}}]},"properties":{"name":[{"id":4,"value":"lop"}],"lang":[{"id":5,"value":"java"}]}}
+{"id":4,"label":"person","inE":{"knows":[{"id":8,"outV":1,"properties":{"weight":1.0}}]},"outE":{"created":[{"id":10,"inV":5,"properties":{"weight":1.0}},{"id":11,"inV":3,"properties":{"weight":0.4}}]},"properties":{"name":[{"id":6,"value":"josh"}],"age":[{"id":7,"value":32}]}}
+{"id":5,"label":"software","inE":{"created":[{"id":10,"outV":4,"properties":{"weight":1.0}}]},"properties":{"name":[{"id":8,"value":"ripple"}],"lang":[{"id":9,"value":"java"}]}}
+{"id":6,"label":"person","outE":{"created":[{"id":12,"inV":3,"properties":{"weight":0.2}}]},"properties":{"name":[{"id":10,"value":"peter"}],"age":[{"id":11,"value":35}]}}
+----
+
+[[script-io-format]]
+Script I/O Format
+^^^^^^^^^^^^^^^^^
+
+* **InputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptInputFormat`
+* **OutputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptOutputFormat`
+
+`ScriptInputFormat` and `ScriptOutputFormat` take an arbitrary script and use that script to either read or write
+`Vertex` objects, respectively. This can be considered the most general `InputFormat`/`OutputFormat` possible in that
+Hadoop-Gremlin uses the user provided script for all reading/writing.
+
+ScriptInputFormat
++++++++++++++++++
+
+The data below represents an adjacency list representation of the classic TinkerGraph toy graph. First line reads,
+"vertex `1`, labeled `person` having 2 property values (`marko` and `29`) has 3 outgoing edges; the first edge is
+labeled `knows`, connects the current vertex `1` with vertex `2` and has a property value `0.4`, and so on."
+
+[source]
+1:person:marko:29 knows:2:0.5,knows:4:1.0,created:3:0.4
+2:person:vadas:27
+3:project:lop:java
+4:person:josh:32 created:3:0.4,created:5:1.0
+5:project:ripple:java
+6:person:peter:35 created:3:0.2
+
+There is no corresponding `InputFormat` that can parse this particular file (or some adjacency list variant of it).
+As such, `ScriptInputFormat` can be used. With `ScriptInputFormat` a script is stored in HDFS and leveraged by each
+mapper in the Hadoop job. The script must have the following method defined:
+
+[source,groovy]
+def parse(String line, ScriptElementFactory factory) { ... }
+
+`ScriptElementFactory` is a legacy from previous versions and, although it's still functional, it should no longer be used.
+In order to create vertices and edges, the `parse()` method gets access to a global variable named `graph`, which holds
+the local `StarGraph` for the current line/vertex.
+
+An appropriate `parse()` for the above adjacency list file is:
+
+[source,groovy]
+def parse(line, factory) {
+ def parts = line.split(/ /)
+ def (id, label, name, x) = parts[0].split(/:/).toList()
+ def v1 = graph.addVertex(T.id, id, T.label, label)
+ if (name != null) v1.property('name', name) // first value is always the name
+ if (x != null) {
+ // second value depends on the vertex label; it's either
+ // the age of a person or the language of a project
+ if (label.equals('project')) v1.property('lang', x)
+ else v1.property('age', Integer.valueOf(x))
+ }
+ if (parts.length == 2) {
+ parts[1].split(/,/).grep { !it.isEmpty() }.each {
+ def (eLabel, refId, weight) = it.split(/:/).toList()
+ def v2 = graph.addVertex(T.id, refId)
+ v1.addOutEdge(eLabel, v2, 'weight', Double.valueOf(weight))
+ }
+ }
+ return v1
+}
+
+The resultant `Vertex` denotes whether the line parsed yielded a valid Vertex. As such, if the line is not valid
+(e.g. a comment line, a skip line, etc.), then simply return `null`.
+
+ScriptOutputFormat Support
+++++++++++++++++++++++++++
+
+The principle above can also be used to convert a vertex to an arbitrary `String` representation that is ultimately
+streamed back to a file in HDFS. This is the role of `ScriptOutputFormat`. `ScriptOutputFormat` requires that the
+provided script maintains a method with the following signature:
+
+[source,groovy]
+def stringify(Vertex vertex) { ... }
+
+An appropriate `stringify()` to produce output in the same format that was shown in the `ScriptInputFormat` sample is:
+
+[source,groovy]
+def stringify(vertex) {
+ def v = vertex.values('name', 'age', 'lang').inject(vertex.id(), vertex.label()).join(':')
+ def outE = vertex.outE().map {
+ def e = it.get()
+ e.values('weight').inject(e.label(), e.inV().next().id()).join(':')
+ }.join(',')
+ return [v, outE].join('\t')
+}
+
+
+
+Storage Systems
+~~~~~~~~~~~~~~~
+
+Hadoop-Gremlin provides two implementations of the `Storage` API:
+
+* `FileSystemStorage`: Access HDFS and local file system data.
+* `SparkContextStorage`: Access Spark persisted RDD data.
+
+[[interacting-with-hdfs]]
+Interacting with HDFS
+^^^^^^^^^^^^^^^^^^^^^
+
+The distributed file system of Hadoop is called link:http://en.wikipedia.org/wiki/Apache_Hadoop#Hadoop_distributed_file_system[HDFS].
+The results of any OLAP operation are stored in HDFS accessible via `hdfs`. For local file system access, there is `local`.
+
+[gremlin-groovy]
+----
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+graph.compute(SparkGraphComputer).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey('clusterCount').create()).submit().get();
+hdfs.ls()
+hdfs.ls('output')
+hdfs.head('output', GryoInputFormat)
+hdfs.head('output', 'clusterCount', SequenceFileInputFormat)
+hdfs.rm('output')
+hdfs.ls()
+----
+
+[[interacting-with-spark]]
+Interacting with Spark
+^^^^^^^^^^^^^^^^^^^^^^
+
+If a Spark context is persisted, then Spark RDDs will remain the Spark cache and accessible over subsequent jobs.
+RDDs are retrieved and saved to the `SparkContext` via `PersistedInputRDD` and `PersistedOutputRDD` respectivly.
+Persisted RDDs can be accessed using `spark`.
+
+[gremlin-groovy]
+----
+Spark.create('local[4]')
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+graph.configuration().setProperty('gremlin.spark.graphOutputRDD', PersistedOutputRDD.class.getCanonicalName())
+graph.configuration().clearProperty('gremlin.hadoop.graphOutputFormat')
+graph.configuration().setProperty('gremlin.spark.persistContext',true)
+graph.compute(SparkGraphComputer).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey('clusterCount').create()).submit().get();
+spark.ls()
+spark.ls('output')
+spark.head('output', PersistedInputRDD)
+spark.head('output', 'clusterCount', PersistedInputRDD)
+spark.rm('output')
+spark.ls()
+----
+
+A Command Line Example
+~~~~~~~~~~~~~~~~~~~~~~
+
+image::pagerank-logo.png[width=300]
+
+The classic link:http://en.wikipedia.org/wiki/PageRank[PageRank] centrality algorithm can be executed over the
+TinkerPop graph from the command line using `GiraphGraphComputer`.
+
+WARNING: Be sure that the `HADOOP_GREMLIN_LIBS` references the location `lib` directory of the respective
+`GraphComputer` engine being used or else the requisite dependencies will not be uploaded to the Hadoop cluster.
+
+[source,text]
+----
+$ hdfs dfs -copyFromLocal data/tinkerpop-modern.json tinkerpop-modern.json
+$ hdfs dfs -ls
+Found 2 items
+-rw-r--r-- 1 marko supergroup 2356 2014-07-28 13:00 /user/marko/tinkerpop-modern.json
+$ hadoop jar target/giraph-gremlin-x.y.z-job.jar org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer ../hadoop-gremlin/conf/hadoop-graphson.properties
+15/09/11 08:02:08 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
+15/09/11 08:02:11 INFO computer.GiraphGraphComputer: HadoopGremlin(Giraph): PageRankVertexProgram[alpha=0.85,iterations=30]
+15/09/11 08:02:12 INFO mapreduce.JobSubmitter: number of splits:3
+15/09/11 08:02:12 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1441915907347_0028
+15/09/11 08:02:12 INFO impl.YarnClientImpl: Submitted application application_1441915907347_0028
+15/09/11 08:02:12 INFO job.GiraphJob: Tracking URL: http://markos-macbook:8088/proxy/application_1441915907347_0028/
+15/09/11 08:02:12 INFO job.GiraphJob: Waiting for resources... Job will start only when it gets all 3 mappers
+15/09/11 08:03:54 INFO mapreduce.Job: Running job: job_1441915907347_0028
+15/09/11 08:03:55 INFO mapreduce.Job: Job job_1441915907347_0028 running in uber mode : false
+15/09/11 08:03:55 INFO mapreduce.Job: map 33% reduce 0%
+15/09/11 08:03:57 INFO mapreduce.Job: map 67% reduce 0%
+15/09/11 08:04:01 INFO mapreduce.Job: map 100% reduce 0%
+15/09/11 08:06:17 INFO mapreduce.Job: Job job_1441915907347_0028 completed successfully
+15/09/11 08:06:17 INFO mapreduce.Job: Counters: 80
+ File System Counters
+ FILE: Number of bytes read=0
+ FILE: Number of bytes written=483918
+ FILE: Number of read operations=0
+ FILE: Number of large read operations=0
+ FILE: Number of write operations=0
+ HDFS: Number of bytes read=1465
+ HDFS: Number of bytes written=1760
+ HDFS: Number of read operations=39
+ HDFS: Number of large read operations=0
+ HDFS: Number of write operations=20
+ Job Counters
+ Launched map tasks=3
+ Other local map tasks=3
+ Total time spent by all maps in occupied slots (ms)=458105
+ Total time spent by all reduces in occupied slots (ms)=0
+ Total time spent by all map tasks (ms)=458105
+ Total vcore-seconds taken by all map tasks=458105
+ Total megabyte-seconds taken by all map tasks=469099520
+ Map-Reduce Framework
+ Map input records=3
+ Map output records=0
+ Input split bytes=132
+ Spilled Records=0
+ Failed Shuffles=0
+ Merged Map outputs=0
+ GC time elapsed (ms)=1594
+ CPU time spent (ms)=0
+ Physical memory (bytes) snapshot=0
+ Virtual memory (bytes) snapshot=0
+ Total committed heap usage (bytes)=527958016
+ Giraph Stats
+ Aggregate edges=0
+ Aggregate finished vertices=0
+ Aggregate sent message message bytes=13535
+ Aggregate sent messages=186
+ Aggregate vertices=6
+ Current master task partition=0
+ Current workers=2
+ Last checkpointed superstep=0
+ Sent message bytes=438
+ Sent messages=6
+ Superstep=31
+ Giraph Timers
+ Initialize (ms)=2996
+ Input superstep (ms)=5209
+ Setup (ms)=59
+ Shutdown (ms)=9324
+ Superstep 0 GiraphComputation (ms)=3861
+ Superstep 1 GiraphComputation (ms)=4027
+ Superstep 10 GiraphComputation (ms)=4000
+ Superstep 11 GiraphComputation (ms)=4004
+ Superstep 12 GiraphComputation (ms)=3999
+ Superstep 13 GiraphComputation (ms)=4000
+ Superstep 14 GiraphComputation (ms)=4005
+ Superstep 15 GiraphComputation (ms)=4003
+ Superstep 16 GiraphComputation (ms)=4001
+ Superstep 17 GiraphComputation (ms)=4007
+ Superstep 18 GiraphComputation (ms)=3998
+ Superstep 19 GiraphComputation (ms)=4006
+ Superstep 2 GiraphComputation (ms)=4007
+ Superstep 20 GiraphComputation (ms)=3996
+ Superstep 21 GiraphComputation (ms)=4006
+ Superstep 22 GiraphComputation (ms)=4002
+ Superstep 23 GiraphComputation (ms)=3998
+ Superstep 24 GiraphComputation (ms)=4003
+ Superstep 25 GiraphComputation (ms)=4001
+ Superstep 26 GiraphComputation (ms)=4003
+ Superstep 27 GiraphComputation (ms)=4005
+ Superstep 28 GiraphComputation (ms)=4002
+ Superstep 29 GiraphComputation (ms)=4001
+ Superstep 3 GiraphComputation (ms)=3988
+ Superstep 30 GiraphComputation (ms)=4248
+ Superstep 4 GiraphComputation (ms)=4010
+ Superstep 5 GiraphComputation (ms)=3998
+ Superstep 6 GiraphComputation (ms)=3996
+ Superstep 7 GiraphComputation (ms)=4005
+ Superstep 8 GiraphComputation (ms)=4009
+ Superstep 9 GiraphComputation (ms)=3994
+ Total (ms)=138788
+ File Input Format Counters
+ Bytes Read=0
+ File Output Format Counters
+ Bytes Written=0
+$ hdfs dfs -cat output/~g/*
+{"id":1,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.15000000000000002}],"name":[{"id":0,"value":"marko"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":3.0}],"age":[{"id":1,"value":29}]}}
+{"id":5,"label":"software","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":35,"value":0.23181250000000003}],"name":[{"id":8,"value":"ripple"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":6,"value":0.0}],"lang":[{"id":9,"value":"java"}]}}
+{"id":3,"label":"software","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.4018125}],"name":[{"id":4,"value":"lop"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":0.0}],"lang":[{"id":5,"value":"java"}]}}
+{"id":4,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.19250000000000003}],"name":[{"id":6,"value":"josh"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":2.0}],"age":[{"id":7,"value":32}]}}
+{"id":2,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":35,"value":0.19250000000000003}],"name":[{"id":2,"value":"vadas"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":6,"value":0.0}],"age":[{"id":3,"value":27}]}}
+{"id":6,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":35,"value":0.15000000000000002}],"name":[{"id":10,"value":"peter"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":6,"value":1.0}],"age":[{"id":11,"value":35}]}}
+----
+
+Vertex 4 ("josh") is isolated below:
+
+[source,js]
+----
+{
+ "id":4,
+ "label":"person",
+ "properties": {
+ "gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.19250000000000003}],
+ "name":[{"id":6,"value":"josh"}],
+ "gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":2.0}],
+ "age":[{"id":7,"value":32}]}
+ }
+}
+----
+
+Hadoop-Gremlin for Graph System Providers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Hadoop-Gremlin is centered around `InputFormats` and `OutputFormats`. If a 3rd-party graph system provider wishes to
+leverage Hadoop-Gremlin (and its respective `GraphComputer` engines), then they need to provide, at minimum, a
+Hadoop2 `InputFormat<NullWritable,VertexWritable>` for their graph system. If the provider wishes to persist computed
+results back to their graph system (and not just to HDFS via a `FileOutputFormat`), then a graph system specific
+`OutputFormat<NullWritable,VertexWritable>` must be developed as well.
+
+Conceptually, `HadoopGraph` is a wrapper around a `Configuration` object. There is no "data" in the `HadoopGraph` as
+the `InputFormat` specifies where and how to get the graph data at OLAP (and OLTP) runtime. Thus, `HadoopGraph` is a
+small object with little overhead. Graph system providers should realize `HadoopGraph` as the gateway to the OLAP
+features offered by Hadoop-Gremlin. For example, a graph system specific `Graph.compute(Class<? extends GraphComputer>
+graphComputerClass)`-method may look as follows:
+
+[source,java]
+----
+public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) throws IllegalArgumentException {
+ try {
+ if (AbstractHadoopGraphComputer.class.isAssignableFrom(graphComputerClass))
+ return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
+ else
+ throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
+ } catch (final Exception e) {
+ throw new IllegalArgumentException(e.getMessage(),e);
+ }
+}
+----
+
+Note that the configurations for Hadoop are assumed to be in the `Graph.configuration()` object. If this is not the
+case, then the `Configuration` provided to `HadoopGraph.open()` should be dynamically created within the
+`compute()`-method. It is in the provided configuration that `HadoopGraph` gets the various properties which
+determine how to read and write data to and from Hadoop. For instance, `gremlin.hadoop.graphInputFormat` and
+`gremlin.hadoop.graphOutputFormat`.
+
+IMPORTANT: A graph system provider's `OutputFormat` should implement the `PersistResultGraphAware` interface which
+determines which persistence options are available to the user. For the standard file-based `OutputFormats` provided
+by Hadoop-Gremlin (e.g. <<gryo-io-format,`GryoOutputFormat`>>, <<graphson-io-format,`GraphSONOutputFormat`>>,
+and <<script-io-format,`ScriptInputOutputFormat`>>) `ResultGraph.ORIGINAL` is not supported as the original graph
+data files are not random access and are, in essence, immutable. Thus, these file-based `OutputFormats` only support
+`ResultGraph.NEW` which creates a copy of the data specified by the `Persist` enum.
+
[35/43] incubator-tinkerpop git commit: Made some adjustments based
on pull request feedback.
Posted by sp...@apache.org.
Made some adjustments based on pull request feedback.
Fixed changelog to use the right name of the archetype. Better logging control/output for the server archetype. Documentation improvements. Included log4j as a compile time dependency.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/68b16eb9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/68b16eb9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/68b16eb9
Branch: refs/heads/TINKERPOP-1107
Commit: 68b16eb90570c4ed4cd36ab9b93e650bddc568ae
Parents: bb3bf09
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Feb 24 13:54:30 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Feb 24 13:54:30 2016 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 +-
.../src/reference/gremlin-applications.asciidoc | 2 +-
.../META-INF/maven/archetype-metadata.xml | 4 +-
.../archetype-resources/README.asciidoc | 10 +++--
.../conf/log4j-test.properties | 21 +++++++++
.../archetype-resources/conf/log4j.properties | 2 +-
.../main/resources/archetype-resources/pom.xml | 13 +++---
.../archetype-resources/src/main/java/App.java | 11 ++++-
.../src/main/resources/gremlin-server.yaml | 45 --------------------
.../src/test/resources/gremlin-server.yaml | 45 ++++++++++++++++++++
10 files changed, 93 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 67965cf..910d9b6 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,7 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.2 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-* Added the `gremlin-archetype-driver` archetype that demonstrates
+* Added the `gremlin-archetype-server` archetype that demonstrates
* Added the `gremlin-archetype-tinkergraph` archetype that demonstrates a basic project that uses TinkerGraph.
* Added `gremlin-archetype` module to house TinkerPop "examples".
* Fixed a bug where multiple "close" requests were being sent by the driver on `Client.close()`.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/docs/src/reference/gremlin-applications.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/gremlin-applications.asciidoc b/docs/src/reference/gremlin-applications.asciidoc
index 02bf1f0..06fe0cc 100644
--- a/docs/src/reference/gremlin-applications.asciidoc
+++ b/docs/src/reference/gremlin-applications.asciidoc
@@ -1897,7 +1897,7 @@ You can use Maven to generate these example projects with a command like:
[source,shell]
$ mvn archetype:generate -DarchetypeGroupId=org.apache.tinkerpop -DarchetypeArtifactId=gremlin-archetype-server
- -DarchetypeVersion=3.1.2-incubating -DgroupId=com.my -DartifactId=app
+ -DarchetypeVersion=x.y.z -DgroupId=com.my -DartifactId=app -Dversion=0.1 -DinteractiveMode=false
This command will generate a new Maven project in a directory called "app" with a `pom.xml` specifying a `groupId` of
`com.my`. Please see the `README.asciidoc` in the root of each generated project for information on how to build and
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
index a6c9d46..93d66a8 100644
--- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
@@ -37,10 +37,10 @@ limitations under the License.
<directory>src/main/java</directory>
</fileSet>
<fileSet filtered="true" packaged="true">
- <directory>src/main/resources</directory>
+ <directory>src/test/java</directory>
</fileSet>
<fileSet filtered="true" packaged="true">
- <directory>src/test/java</directory>
+ <directory>src/test/resources</directory>
</fileSet>
</fileSets>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
index 42f369d..fd543bc 100644
--- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
@@ -33,6 +33,11 @@ Prerequisites
Building and Running
--------------------
+Build the project as follows:
+
+[source,text]
+mvn clean package
+
Start Gremlin Server in the directory it was unpackaged and specify usage of the "modern" graph configuration:
[source,text]
@@ -60,8 +65,7 @@ $ bin/gremlin-server.sh conf/gremlin-server-modern.yaml
[INFO] GremlinServer$1 - Gremlin Server configured with worker thread pool of 1, gremlin pool of 8 and boss thread pool of 1.
[INFO] GremlinServer$1 - Channel started at port 8182.
-Build and run this project as follows:
+Run this project as follows:
[source,text]
-mvn clean package
-mvn exec:java -Dexec.mainClass="${package}.App"
\ No newline at end of file
+mvn exec:java -Dexec.mainClass="${package}.App" -Dlog4j.configuration=file:conf/log4j.properties
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j-test.properties b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
index ef436fe..2b58359 100644
--- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-log4j.rootLogger=WARN, stdout
+log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
index 5c47aea..ca29604 100644
--- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
@@ -35,15 +35,14 @@ limitations under the License.
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-server</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-server</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -71,7 +70,7 @@ limitations under the License.
<version>2.17</version>
<configuration>
<systemPropertyVariables>
- <log4j.configuration>file:conf/log4j.properties</log4j.configuration>
+ <log4j.configuration>file:conf/log4j-test.properties</log4j.configuration>
</systemPropertyVariables>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
index d1761c7..1e111e3 100644
--- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
@@ -19,17 +19,24 @@
package ${package};
import org.apache.tinkerpop.gremlin.driver.Result;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class App {
+ private static final Logger logger = LoggerFactory.getLogger(App.class);
public static void main(String[] args) throws Exception {
Service service = Service.getInstance();
try {
- service.findCreatorsOfSoftware("lop").iterator().forEachRemaining(r -> System.out.println(r));
+ logger.info("Sending request....");
+
+ // should print marko, josh and peter to the logs
+ service.findCreatorsOfSoftware("lop").iterator().forEachRemaining(r -> logger.info(String.format(" - %s", r)));
} catch (Exception ex) {
- ex.printStackTrace();
+ logger.error("Could not execute traversal", ex);
} finally {
service.close();
+ logger.info("Service closed and resources released");
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml
deleted file mode 100644
index 5744d29..0000000
--- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-host: localhost
-port: 8182
-threadPoolWorker: 1
-gremlinPool: 8
-scriptEvaluationTimeout: 30000
-serializedResponseTimeout: 30000
-graphs: {
- graph: conf/tinkergraph-empty.properties}
-plugins:
- - tinkerpop.tinkergraph
-scriptEngines: {
- gremlin-groovy: {
- imports: [java.lang.Math],
- staticImports: [java.lang.Math.PI],
- scripts: [scripts/generate-modern.groovy]}}
-serializers:
- - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
- - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
-metrics: {
- slf4jReporter: {enabled: true, interval: 180000}}
-strictTransactionManagement: false
-threadPoolBoss: 1
-maxInitialLineLength: 4096
-maxHeaderSize: 8192
-maxChunkSize: 8192
-maxContentLength: 65536
-maxAccumulationBufferComponents: 1024
-resultIterationBatchSize: 64
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/68b16eb9/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml
new file mode 100644
index 0000000..5744d29
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+host: localhost
+port: 8182
+threadPoolWorker: 1
+gremlinPool: 8
+scriptEvaluationTimeout: 30000
+serializedResponseTimeout: 30000
+graphs: {
+ graph: conf/tinkergraph-empty.properties}
+plugins:
+ - tinkerpop.tinkergraph
+scriptEngines: {
+ gremlin-groovy: {
+ imports: [java.lang.Math],
+ staticImports: [java.lang.Math.PI],
+ scripts: [scripts/generate-modern.groovy]}}
+serializers:
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
+metrics: {
+ slf4jReporter: {enabled: true, interval: 180000}}
+strictTransactionManagement: false
+threadPoolBoss: 1
+maxInitialLineLength: 4096
+maxHeaderSize: 8192
+maxChunkSize: 8192
+maxContentLength: 65536
+maxAccumulationBufferComponents: 1024
+resultIterationBatchSize: 64
[36/43] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/TINKERPOP-1085' into tp31
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/TINKERPOP-1085' into tp31
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/00e9b20b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/00e9b20b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/00e9b20b
Branch: refs/heads/TINKERPOP-1107
Commit: 00e9b20b8ded07ee872a07ed49755bae13e4b346
Parents: 15909c6 68b16eb
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Feb 24 14:06:28 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Feb 24 14:06:28 2016 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +
.../src/reference/gremlin-applications.asciidoc | 22 +++++
.../upgrade/release-3.1.x-incubating.asciidoc | 15 +++-
.../gremlin-archetype-server/pom.xml | 59 ++++++++++++++
.../META-INF/maven/archetype-metadata.xml | 47 +++++++++++
.../archetype-resources/README.asciidoc | 71 +++++++++++++++++
.../conf/log4j-test.properties | 21 +++++
.../archetype-resources/conf/log4j.properties | 21 +++++
.../conf/tinkergraph-empty.properties | 18 +++++
.../main/resources/archetype-resources/pom.xml | 79 ++++++++++++++++++
.../scripts/generate-modern.groovy | 33 ++++++++
.../archetype-resources/src/main/java/App.java | 42 ++++++++++
.../src/main/java/Service.java | 70 ++++++++++++++++
.../src/test/java/ServiceTest.java | 84 ++++++++++++++++++++
.../src/test/resources/gremlin-server.yaml | 45 +++++++++++
.../projects/standard/archetype.properties | 21 +++++
.../test/resources/projects/standard/goal.txt | 1 +
.../gremlin-archetype-tinkergraph/pom.xml | 59 ++++++++++++++
.../META-INF/maven/archetype-metadata.xml | 38 +++++++++
.../archetype-resources/README.asciidoc | 35 ++++++++
.../main/resources/archetype-resources/pom.xml | 58 ++++++++++++++
.../archetype-resources/src/main/java/App.java | 78 ++++++++++++++++++
.../src/test/java/AppTest.java | 40 ++++++++++
.../projects/standard/archetype.properties | 21 +++++
.../test/resources/projects/standard/goal.txt | 1 +
gremlin-archetype/pom.xml | 70 ++++++++++++++++
pom.xml | 2 +
27 files changed, 1053 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/00e9b20b/pom.xml
----------------------------------------------------------------------
[03/43] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/TINKERPOP-1148' into tp31
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/TINKERPOP-1148' into tp31
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f19311b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f19311b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f19311b7
Branch: refs/heads/TINKERPOP-1107
Commit: f19311b7fbd21ce07f52cfe9fb06e53d507077d2
Parents: 70a3065 b42dcea
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Feb 18 11:28:36 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Feb 18 11:28:36 2016 -0500
----------------------------------------------------------------------
.../server/op/AbstractEvalOpProcessor.java | 51 ++++++++++++--------
.../gremlin/server/op/session/Session.java | 6 ++-
.../server/op/session/SessionOpProcessor.java | 4 +-
.../server/GremlinServerIntegrateTest.java | 27 +++++++++++
4 files changed, 66 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
[04/43] incubator-tinkerpop git commit: splitted
implementations.asciidoc into implementations-hadoop.asciidoc and
implementations-neo4j.asciidoc
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bbf5b3f4/docs/src/reference/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/index.asciidoc b/docs/src/reference/index.asciidoc
index 5a01291..8546063 100644
--- a/docs/src/reference/index.asciidoc
+++ b/docs/src/reference/index.asciidoc
@@ -32,7 +32,8 @@ include::the-graphcomputer.asciidoc[]
include::gremlin-applications.asciidoc[]
-include::implementations.asciidoc[]
+include::implementations-neo4j.asciidoc[]
+include::implementations-hadoop.asciidoc[]
include::conclusion.asciidoc[]
[06/43] incubator-tinkerpop git commit: splitted
implementations.asciidoc into implementations-hadoop.asciidoc and
implementations-neo4j.asciidoc
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bbf5b3f4/docs/src/reference/implementations-neo4j.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-neo4j.asciidoc b/docs/src/reference/implementations-neo4j.asciidoc
new file mode 100644
index 0000000..5602754
--- /dev/null
+++ b/docs/src/reference/implementations-neo4j.asciidoc
@@ -0,0 +1,921 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+[[implementations]]
+Implementations
+===============
+
+image::gremlin-racecar.png[width=325]
+
+[[graph-system-provider-requirements]]
+Graph System Provider Requirements
+----------------------------------
+
+image:tinkerpop-enabled.png[width=140,float=left] At the core of TinkerPop3 is a Java8 API. The implementation of this
+core API and its validation via the `gremlin-test` suite is all that is required of a graph system provider wishing to
+provide a TinkerPop3-enabled graph engine. Once a graph system has a valid implementation, then all the applications
+provided by TinkerPop (e.g. Gremlin Console, Gremlin Server, etc.) and 3rd-party developers (e.g. Gremlin-Scala,
+Gremlin-JS, etc.) will integrate properly. Finally, please feel free to use the logo on the left to promote your
+TinkerPop3 implementation.
+
+Implementing Gremlin-Core
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The classes that a graph system provider should focus on implementing are itemized below. It is a good idea to study
+the <<tinkergraph-gremlin,TinkerGraph>> (in-memory OLTP and OLAP in `tinkergraph-gremlin`), <<neo4j-gremlin,Neo4jGraph>>
+(OTLP w/ transactions in `neo4j-gremlin`) and/or <<hadoop-gremlin,HadoopGraph>> (OLAP in `hadoop-gremlin`)
+implementations for ideas and patterns.
+
+. Online Transactional Processing Graph Systems (*OLTP*)
+ .. Structure API: `Graph`, `Element`, `Vertex`, `Edge`, `Property` and `Transaction` (if transactions are supported).
+ .. Process API: `TraversalStrategy` instances for optimizing Gremlin traversals to the provider's graph system (i.e. `TinkerGraphStepStrategy`).
+. Online Analytics Processing Graph Systems (*OLAP*)
+ .. Everything required of OTLP is required of OLAP (but not vice versa).
+ .. GraphComputer API: `GraphComputer`, `Messenger`, `Memory`.
+
+Please consider the following implementation notes:
+
+* Be sure your `Graph` implementation is named as `XXXGraph` (e.g. TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
+* Use `StringHelper` to ensuring that the `toString()` representation of classes are consistent with other implementations.
+* Ensure that your implementation's `Features` (Graph, Vertex, etc.) are correct so that test cases handle particulars accordingly.
+* Use the numerous static method helper classes such as `ElementHelper`, `GraphComputerHelper`, `VertexProgramHelper`, etc.
+* There are a number of default methods on the provided interfaces that are semantically correct. However, if they are
+not efficient for the implementation, override them.
+* Implement the `structure/` package interfaces first and then, if desired, interfaces in the `process/` package interfaces.
+* `ComputerGraph` is a `Wrapper` system that ensure proper semantics during a GraphComputer computation.
+
+[[oltp-implementations]]
+OLTP Implementations
+^^^^^^^^^^^^^^^^^^^^
+
+image:pipes-character-1.png[width=110,float=right] The most important interfaces to implement are in the `structure/`
+package. These include interfaces like Graph, Vertex, Edge, Property, Transaction, etc. The `StructureStandardSuite`
+will ensure that the semantics of the methods implemented are correct. Moreover, there are numerous `Exceptions`
+classes with static exceptions that should be thrown by the graph system so that all the exceptions and their
+messages are consistent amongst all TinkerPop3 implementations.
+
+[[olap-implementations]]
+OLAP Implementations
+^^^^^^^^^^^^^^^^^^^^
+
+image:furnace-character-1.png[width=110,float=right] Implementing the OLAP interfaces may be a bit more complicated.
+Note that before OLAP interfaces are implemented, it is necessary for the OLTP interfaces to be, at minimal,
+implemented as specified in <<oltp-implementations,OLTP Implementations>>. A summary of each required interface
+implementation is presented below:
+
+. `GraphComputer`: A fluent builder for specifying an isolation level, a VertexProgram, and any number of MapReduce jobs to be submitted.
+. `Memory`: A global blackboard for ANDing, ORing, INCRing, and SETing values for specified keys.
+. `Messenger`: The system that collects and distributes messages being propagated by vertices executing the VertexProgram application.
+. `MapReduce.MapEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications map-phase.
+. `MapReduce.ReduceEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications combine- and reduce-phases.
+
+NOTE: The VertexProgram and MapReduce interfaces in the `process/computer/` package are not required by the graph
+system. Instead, these are interfaces to be implemented by application developers writing VertexPrograms and MapReduce jobs.
+
+IMPORTANT: TinkerPop3 provides three OLAP implementations: <<tinkergraph-gremlin,TinkerGraphComputer>> (TinkerGraph),
+<<giraphgraphcomputer,GiraphGraphComputer>> (HadoopGraph), and <<sparkgraphcomputer,`SparkGraphComputer`>> (Hadoop).
+Given the complexity of the OLAP system, it is good to study and copy many of the patterns used in these reference
+implementations.
+
+Implementing GraphComputer
+++++++++++++++++++++++++++
+
+image:furnace-character-3.png[width=150,float=right] The most complex method in GraphComputer is the `submit()`-method. The method must do the following:
+
+. Ensure the the GraphComputer has not already been executed.
+. Ensure that at least there is a VertexProgram or 1 MapReduce job.
+. If there is a VertexProgram, validate that it can execute on the GraphComputer given the respectively defined features.
+. Create the Memory to be used for the computation.
+. Execute the VertexProgram.setup() method once and only once.
+. Execute the VertexProgram.execute() method for each vertex.
+. Execute the VertexProgram.terminate() method once and if true, repeat VertexProgram.execute().
+. When VertexProgram.terminate() returns true, move to MapReduce job execution.
+. MapReduce jobs are not required to be executed in any specified order.
+. For each Vertex, execute MapReduce.map(). Then (if defined) execute MapReduce.combine() and MapReduce.reduce().
+. Update Memory with runtime information.
+. Construct a new `ComputerResult` containing the compute Graph and Memory.
+
+Implementing Memory
++++++++++++++++++++
+
+image:gremlin-brain.png[width=175,float=left] The Memory object is initially defined by `VertexProgram.setup()`.
+The memory data is available in the first round of the `VertexProgram.execute()` method. Each Vertex, when executing
+the VertexProgram, can update the Memory in its round. However, the update is not seen by the other vertices until
+the next round. At the end of the first round, all the updates are aggregated and the new memory data is available
+on the second round. This process repeats until the VertexProgram terminates.
+
+Implementing Messenger
+++++++++++++++++++++++
+
+The Messenger object is similar to the Memory object in that a vertex can read and write to the Messenger. However,
+the data it reads are the messages sent to the vertex in the previous step and the data it writes are the messages
+that will be readable by the receiving vertices in the subsequent round.
+
+Implementing MapReduce Emitters
++++++++++++++++++++++++++++++++
+
+image:hadoop-logo-notext.png[width=150,float=left] The MapReduce framework in TinkerPop3 is similar to the model
+popularized by link:http://apache.hadoop.org[Hadoop]. The primary difference is that all Mappers process the vertices
+of the graph, not an arbitrary key/value pair. However, the vertices' edges can not be accessed -- only their
+properties. This greatly reduces the amount of data needed to be pushed through the MapReduce engine as any edge
+information required, can be computed in the VertexProgram.execute() method. Moreover, at this stage, vertices can
+not be mutated, only their token and property data read. A Gremlin OLAP system needs to provide implementations for
+to particular classes: `MapReduce.MapEmitter` and `MapReduce.ReduceEmitter`. TinkerGraph's implementation is provided
+below which demonstrates the simplicity of the algorithm (especially when the data is all within the same JVM).
+
+[source,java]
+----
+public class TinkerMapEmitter<K, V> implements MapReduce.MapEmitter<K, V> {
+
+ public Map<K, Queue<V>> reduceMap;
+ public Queue<KeyValue<K, V>> mapQueue;
+ private final boolean doReduce;
+
+ public TinkerMapEmitter(final boolean doReduce) { <1>
+ this.doReduce = doReduce;
+ if (this.doReduce)
+ this.reduceMap = new ConcurrentHashMap<>();
+ else
+ this.mapQueue = new ConcurrentLinkedQueue<>();
+ }
+
+ @Override
+ public void emit(K key, V value) {
+ if (this.doReduce)
+ this.reduceMap.computeIfAbsent(key, k -> new ConcurrentLinkedQueue<>()).add(value); <2>
+ else
+ this.mapQueue.add(new KeyValue<>(key, value)); <3>
+ }
+
+ protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) {
+ if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { <4>
+ final Comparator<K> comparator = mapReduce.getMapKeySort().get();
+ final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue);
+ Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
+ this.mapQueue.clear();
+ this.mapQueue.addAll(list);
+ } else if (mapReduce.getMapKeySort().isPresent()) {
+ final Comparator<K> comparator = mapReduce.getMapKeySort().get();
+ final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>();
+ list.addAll(this.reduceMap.entrySet());
+ Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator));
+ this.reduceMap = new LinkedHashMap<>();
+ list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue()));
+ }
+ }
+}
+----
+
+<1> If the MapReduce job has a reduce, then use one data structure (`reduceMap`), else use another (`mapList`). The
+difference being that a reduction requires a grouping by key and therefore, the `Map<K,Queue<V>>` definition. If no
+reduction/grouping is required, then a simple `Queue<KeyValue<K,V>>` can be leveraged.
+<2> If reduce is to follow, then increment the Map with a new value for the key. `MapHelper` is a TinkerPop3 class
+with static methods for adding data to a Map.
+<3> If no reduce is to follow, then simply append a KeyValue to the queue.
+<4> When the map phase is complete, any map-result sorting required can be executed at this point.
+
+[source,java]
+----
+public class TinkerReduceEmitter<OK, OV> implements MapReduce.ReduceEmitter<OK, OV> {
+
+ protected Queue<KeyValue<OK, OV>> reduceQueue = new ConcurrentLinkedQueue<>();
+
+ @Override
+ public void emit(final OK key, final OV value) {
+ this.reduceQueue.add(new KeyValue<>(key, value));
+ }
+
+ protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) {
+ if (mapReduce.getReduceKeySort().isPresent()) {
+ final Comparator<OK> comparator = mapReduce.getReduceKeySort().get();
+ final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue);
+ Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
+ this.reduceQueue.clear();
+ this.reduceQueue.addAll(list);
+ }
+ }
+}
+----
+
+The method `MapReduce.reduce()` is defined as:
+
+[source,java]
+public void reduce(final OK key, final Iterator<OV> values, final ReduceEmitter<OK, OV> emitter) { ... }
+
+In other words, for the TinkerGraph implementation, iterate through the entrySet of the `reduceMap` and call the
+`reduce()` method on each entry. The `reduce()` method can emit key/value pairs which are simply aggregated into a
+`Queue<KeyValue<OK,OV>>` in an analogous fashion to `TinkerMapEmitter` when no reduce is to follow. These two emitters
+are tied together in `TinkerGraphComputer.submit()`.
+
+[source,java]
+----
+...
+for (final MapReduce mapReduce : mapReducers) {
+ if (mapReduce.doStage(MapReduce.Stage.MAP)) {
+ final TinkerMapEmitter<?, ?> mapEmitter = new TinkerMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
+ final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(this.graph.vertices());
+ workers.setMapReduce(mapReduce);
+ workers.mapReduceWorkerStart(MapReduce.Stage.MAP);
+ workers.executeMapReduce(workerMapReduce -> {
+ while (true) {
+ final Vertex vertex = vertices.next();
+ if (null == vertex) return;
+ workerMapReduce.map(ComputerGraph.mapReduce(vertex), mapEmitter);
+ }
+ });
+ workers.mapReduceWorkerEnd(MapReduce.Stage.MAP);
+
+ // sort results if a map output sort is defined
+ mapEmitter.complete(mapReduce);
+
+ // no need to run combiners as this is single machine
+ if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
+ final TinkerReduceEmitter<?, ?> reduceEmitter = new TinkerReduceEmitter<>();
+ final SynchronizedIterator<Map.Entry<?, Queue<?>>> keyValues = new SynchronizedIterator((Iterator) mapEmitter.reduceMap.entrySet().iterator());
+ workers.mapReduceWorkerStart(MapReduce.Stage.REDUCE);
+ workers.executeMapReduce(workerMapReduce -> {
+ while (true) {
+ final Map.Entry<?, Queue<?>> entry = keyValues.next();
+ if (null == entry) return;
+ workerMapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter);
+ }
+ });
+ workers.mapReduceWorkerEnd(MapReduce.Stage.REDUCE);
+ reduceEmitter.complete(mapReduce); // sort results if a reduce output sort is defined
+ mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator()); <1>
+ } else {
+ mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator()); <2>
+ }
+ }
+}
+...
+----
+
+<1> Note that the final results of the reducer are provided to the Memory as specified by the application developer's
+`MapReduce.addResultToMemory()` implementation.
+<2> If there is no reduce stage, the the map-stage results are inserted into Memory as specified by the application
+developer's `MapReduce.addResultToMemory()` implementation.
+
+[[io-implementations]]
+IO Implementations
+^^^^^^^^^^^^^^^^^^
+
+If a `Graph` requires custom serializers for IO to work properly, implement the `Graph.io` method. A typical example
+of where a `Graph` would require such a custom serializers is if their identifier system uses non-primitive values,
+such as OrientDB's `Rid` class. From basic serialization of a single `Vertex` all the way up the stack to Gremlin
+Server, the need to know how to handle these complex identifiers is an important requirement.
+
+The first step to implementing custom serializers is to first implement the `IoRegistry` interface and register the
+custom classes and serializers to it. Each `Io` implementation has different requirements for what it expects from the
+`IoRegistry`:
+
+* *GraphML* - No custom serializers expected/allowed.
+* *GraphSON* - Register a Jackson `SimpleModule`. The `SimpleModule` encapsulates specific classes to be serialized,
+so it does not need to be registered to a specific class in the `IoRegistry` (use `null`).
+* *Gryo* - Expects registration of one of three objects:
+** Register just the custom class with a `null` Kryo `Serializer` implementation - this class will use default "field-level" Kryo serialization.
+** Register the custom class with a specific Kryo `Serializer' implementation.
+** Register the custom class with a `Function<Kryo, Serializer>` for those cases where the Kryo `Serializer` requires the `Kryo` instance to get constructed.
+
+This implementation should provide a zero-arg constructor as the stack may require instantiation via reflection.
+Consider extending `AbstractIoRegistry` for convenience as follows:
+
+[source,java]
+----
+public class MyGraphIoRegistry extends AbstractIoRegistry {
+ public MyGraphIoRegistry() {
+ register(GraphSONIo.class, null, new MyGraphSimpleModule());
+ register(GryoIo.class, MyGraphIdClass.class, new MyGraphIdSerializer());
+ }
+}
+----
+
+In the `Graph.io` method, provide the `IoRegistry` object to the supplied `Builder` and call the `create` method to
+return that `Io` instance as follows:
+
+[source,java]
+----
+public <I extends Io> I io(final Io.Builder<I> builder) {
+ return (I) builder.graph(this).registry(myGraphIoRegistry).create();
+}}
+----
+
+In this way, `Graph` implementations can pre-configure custom serializers for IO interactions and users will not need
+to know about those details. Following this pattern will ensure proper execution of the test suite as well as
+simplified usage for end-users.
+
+IMPORTANT: Proper implementation of IO is critical to successful `Graph` operations in Gremlin Server. The Test Suite
+does have "serialization" tests that provide some assurance that an implementation is working properly, but those
+tests cannot make assertions against any specifics of a custom serializer. It is the responsibility of the
+implementer to test the specifics of their custom serializers.
+
+TIP: Consider separating serializer code into its own module, if possible, so that clients that use the `Graph`
+implementation remotely don't need a full dependency on the entire `Graph` - just the IO components and related
+classes being serialized.
+
+[[validating-with-gremlin-test]]
+Validating with Gremlin-Test
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+image:gremlin-edumacated.png[width=225]
+
+[source,xml]
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-test</artifactId>
+ <version>x.y.z</version>
+</dependency>
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy-test</artifactId>
+ <version>x.y.z</version>
+</dependency>
+
+The operational semantics of any OLTP or OLAP implementation are validated by `gremlin-test` and functional
+interoperability with the Groovy environment is ensured by `gremlin-groovy-test`. To implement these tests, provide
+test case implementations as shown below, where `XXX` below denotes the name of the graph implementation (e.g.
+TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
+
+[source,java]
+----
+// Structure API tests
+@RunWith(StructureStandardSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
+public class XXXStructureStandardTest {}
+
+// Process API tests
+@RunWith(ProcessComputerSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
+public class XXXProcessComputerTest {}
+
+@RunWith(ProcessStandardSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
+public class XXXProcessStandardTest {}
+
+@RunWith(GroovyEnvironmentSuite.class)
+@GraphProviderClass(provider = XXXProvider.class, graph = TinkerGraph.class)
+public class XXXGroovyEnvironmentTest {}
+
+@RunWith(GroovyProcessStandardSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = TinkerGraph.class)
+public class XXXGroovyProcessStandardTest {}
+
+@RunWith(GroovyProcessComputerSuite.class)
+@GraphProviderClass(provider = XXXGraphComputerProvider.class, graph = TinkerGraph.class)
+public class XXXGroovyProcessComputerTest {}
+----
+
+The above set of tests represent the minimum test suite set to implement. There are other "integration" and
+"performance" tests that should be considered optional. Implementing those tests requires the same pattern as shown above.
+
+IMPORTANT: It is as important to look at "ignored" tests as it is to look at ones that fail. The `gremlin-test`
+suite utilizes the `Feature` implementation exposed by the `Graph` to determine which tests to execute. If a test
+utilizes features that are not supported by the graph, it will ignore them. While that may be fine, implementers
+should validate that the ignored tests are appropriately bypassed and that there are no mistakes in their feature
+definitions. Moreover, implementers should consider filling gaps in their own test suites, especially when
+IO-related tests are being ignored.
+
+The only test-class that requires any code investment is the `GraphProvider` implementation class. This class is a
+used by the test suite to construct `Graph` configurations and instances and provides information about the
+implementation itself. In most cases, it is best to simply extend `AbstractGraphProvider` as it provides many
+default implementations of the `GraphProvider` interface.
+
+Finally, specify the test suites that will be supported by the `Graph` implementation using the `@Graph.OptIn`
+annotation. See the `TinkerGraph` implementation below as an example:
+
+[source,java]
+----
+@Graph.OptIn(Graph.OptIn.SUITE_STRUCTURE_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
+@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_COMPUTER)
+@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT)
+public class TinkerGraph implements Graph {
+----
+
+Only include annotations for the suites the implementation will support. Note that implementing the suite, but
+not specifying the appropriate annotation will prevent the suite from running (an obvious error message will appear
+in this case when running the mis-configured suite).
+
+There are times when there may be a specific test in the suite that the implementation cannot support (despite the
+features it implements) or should not otherwise be executed. It is possible for implementers to "opt-out" of a test
+by using the `@Graph.OptOut` annotation. The following is an example of this annotation usage as taken from
+`HadoopGraph`:
+
+[source, java]
+----
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
+ method = "g_V_matchXa_hasXname_GarciaX__a_inXwrittenByX_b__a_inXsungByX_bX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
+ method = "g_V_matchXa_inXsungByX_b__a_inXsungByX_c__b_outXwrittenByX_d__c_outXwrittenByX_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
+ method = "shouldNotAllowBadMemoryKeys",
+ reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
+ method = "shouldRequireRegisteringMemoryKeys",
+ reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+public class HadoopGraph implements Graph {
+----
+
+The above examples show how to ignore individual tests. It is also possible to:
+
+* Ignore an entire test case (i.e. all the methods within the test) by setting the `method` to "*".
+* Ignore a "base" test class such that test that extend from those classes will all be ignored. This style of
+ignoring is useful for Gremlin "process" tests that have bases classes that are extended by various Gremlin flavors (e.g. groovy).
+* Ignore a `GraphComputer` test based on the type of `GraphComputer` being used. Specify the "computer" attribute on
+the `OptOut` (which is an array specification) which should have a value of the `GraphComputer` implementation class
+that should ignore that test. This attribute should be left empty for "standard" execution and by default all
+`GraphComputer` implementations will be included in the `OptOut` so if there are multiple implementations, explicitly
+specify the ones that should be excluded.
+
+Also note that some of the tests in the Gremlin Test Suite are parameterized tests and require an additional level of
+specificity to be properly ignored. To ignore these types of tests, examine the name template of the parameterized
+tests. It is defined by a Java annotation that looks like this:
+
+[source, java]
+@Parameterized.Parameters(name = "expect({0})")
+
+The annotation above shows that the name of each parameterized test will be prefixed with "expect" and have
+parentheses wrapped around the first parameter (at index 0) value supplied to each test. This information can
+only be garnered by studying the test set up itself. Once the pattern is determined and the specific unique name of
+the parameterized test is identified, add it to the `specific` property on the `OptOut` annotation in addition to
+the other arguments.
+
+These annotations help provide users a level of transparency into test suite compliance (via the
+xref:describe-graph[describeGraph()] utility function). It also allows implementers to have a lot of flexibility in
+terms of how they wish to support TinkerPop. For example, maybe there is a single test case that prevents an
+implementer from claiming support of a `Feature`. The implementer could choose to either not support the `Feature`
+or to support it but "opt-out" of the test with a "reason" as to why so that users understand the limitation.
+
+IMPORTANT: Before using `OptOut` be sure that the reason for using it is sound and it is more of a last resort.
+It is possible that a test from the suite doesn't properly represent the expectations of a feature, is too broad or
+narrow for the semantics it is trying to enforce or simply contains a bug. Please consider raising issues in the
+developer mailing list with such concerns before assuming `OptOut` is the only answer.
+
+IMPORTANT: There are no tests that specifically validate complete compliance with Gremlin Server. Generally speaking,
+a `Graph` that passes the full Test Suite, should be compliant with Gremlin Server. The one area where problems can
+occur is in serialization. Always ensure that IO is properly implemented, that custom serializers are tested fully
+and ultimately integration test the `Graph` with an actual Gremlin Server instance.
+
+CAUTION: Configuring tests to run in parallel might result in errors that are difficult to debug as there is some
+shared state in test execution around graph configuration. It is therefore recommended that parallelism be turned
+off for the test suite (the Maven SureFire Plugin is configured this way by default). It may also be important to
+include this setting, `<reuseForks>false</reuseForks>`, in the SureFire configuration if tests are failing in an
+unexplainable way.
+
+Accessibility via GremlinPlugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+image:gremlin-plugin.png[width=100,float=left] The applications distributed with TinkerPop3 do not distribute with
+any graph system implementations besides TinkerGraph. If your implementation is stored in a Maven repository (e.g.
+Maven Central Repository), then it is best to provide a `GremlinPlugin` implementation so the respective jars can be
+downloaded according and when required by the user. Neo4j's GremlinPlugin is provided below for reference.
+
+[source,java]
+----
+public class Neo4jGremlinPlugin implements GremlinPlugin {
+
+ private static final String IMPORT = "import ";
+ private static final String DOT_STAR = ".*";
+
+ private static final Set<String> IMPORTS = new HashSet<String>() {{
+ add(IMPORT + Neo4jGraph.class.getPackage().getName() + DOT_STAR);
+ }};
+
+ @Override
+ public String getName() {
+ return "neo4j";
+ }
+
+ @Override
+ public void pluginTo(final PluginAcceptor pluginAcceptor) {
+ pluginAcceptor.addImports(IMPORTS);
+ }
+}
+----
+
+With the above plugin implementations, users can now download respective binaries for Gremlin Console, Gremlin Server, etc.
+
+[source,groovy]
+gremlin> g = Neo4jGraph.open('/tmp/neo4j')
+No such property: Neo4jGraph for class: groovysh_evaluate
+Display stack trace? [yN]
+gremlin> :install org.apache.tinkerpop neo4j-gremlin x.y.z
+==>loaded: [org.apache.tinkerpop, neo4j-gremlin, …]
+gremlin> :plugin use tinkerpop.neo4j
+==>tinkerpop.neo4j activated
+gremlin> g = Neo4jGraph.open('/tmp/neo4j')
+==>neo4jgraph[EmbeddedGraphDatabase [/tmp/neo4j]]
+
+In-Depth Implementations
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+image:gremlin-painting.png[width=200,float=right] The graph system implementation details presented thus far are
+minimum requirements necessary to yield a valid TinkerPop3 implementation. However, there are other areas that a
+graph system provider can tweak to provide an implementation more optimized for their underlying graph engine. Typical
+areas of focus include:
+
+* Traversal Strategies: A <<traversalstrategy,TraversalStrategy>> can be used to alter a traversal prior to its
+execution. A typical example is converting a pattern of `g.V().has('name','marko')` into a global index lookup for
+all vertices with name "marko". In this way, a `O(|V|)` lookup becomes an `O(log(|V|))`. Please review
+`TinkerGraphStepStrategy` for ideas.
+* Step Implementations: Every <<graph-traversal-steps,step>> is ultimately referenced by the `GraphTraversal`
+interface. It is possible to extend `GraphTraversal` to use a graph system specific step implementation.
+
+
+[[tinkergraph-gremlin]]
+TinkerGraph-Gremlin
+-------------------
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkergraph-gremlin</artifactId>
+ <version>x.y.z</version>
+</dependency>
+----
+
+image:tinkerpop-character.png[width=100,float=left] TinkerGraph is a single machine, in-memory (with optional
+persistence), non-transactional graph engine that provides both OLTP and OLAP functionality. It is deployed with
+TinkerPop3 and serves as the reference implementation for other providers to study in order to understand the
+semantics of the various methods of the TinkerPop3 API. Constructing a simple graph in Java8 is presented below.
+
+[source,java]
+Graph g = TinkerGraph.open();
+Vertex marko = g.addVertex("name","marko","age",29);
+Vertex lop = g.addVertex("name","lop","lang","java");
+marko.addEdge("created",lop,"weight",0.6d);
+
+The above graph creates two vertices named "marko" and "lop" and connects them via a created-edge with a weight=0.6
+property. Next, the graph can be queried as such.
+
+[source,java]
+g.V().has("name","marko").out("created").values("name")
+
+The `g.V().has("name","marko")` part of the query can be executed in two ways.
+
+ * A linear scan of all vertices filtering out those vertices that don't have the name "marko"
+ * A `O(log(|V|))` index lookup for all vertices with the name "marko"
+
+Given the initial graph construction in the first code block, no index was defined and thus, a linear scan is executed.
+However, if the graph was constructed as such, then an index lookup would be used.
+
+[source,java]
+Graph g = TinkerGraph.open();
+g.createIndex("name",Vertex.class)
+
+The execution times for a vertex lookup by property is provided below for both no-index and indexed version of
+TinkerGraph over the Grateful Dead graph.
+
+[gremlin-groovy]
+----
+graph = TinkerGraph.open()
+g = graph.traversal()
+graph.io(graphml()).readGraph('data/grateful-dead.xml')
+clock(1000) {g.V().has('name','Garcia').iterate()} <1>
+graph = TinkerGraph.open()
+g = graph.traversal()
+graph.createIndex('name',Vertex.class)
+graph.io(graphml()).readGraph('data/grateful-dead.xml')
+clock(1000){g.V().has('name','Garcia').iterate()} <2>
+----
+
+<1> Determine the average runtime of 1000 vertex lookups when no `name`-index is defined.
+<2> Determine the average runtime of 1000 vertex lookups when a `name`-index is defined.
+
+IMPORTANT: Each graph system will have different mechanism by which indices and schemas are defined. TinkerPop3
+does not require any conformance in this area. In TinkerGraph, the only definitions are around indices. With other
+graph systems, property value types, indices, edge labels, etc. may be required to be defined _a priori_ to adding
+data to the graph.
+
+NOTE: TinkerGraph is distributed with Gremlin Server and is therefore automatically available to it for configuration.
+
+Configuration
+~~~~~~~~~~~~~
+
+TinkerGraph has several settings that can be provided on creation via `Configuration` object:
+
+[width="100%",cols="2,10",options="header"]
+|=========================================================
+|Property |Description
+|gremlin.graph |`org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph`
+|gremlin.tinkergraph.vertexIdManager |The `IdManager` implementation to use for vertices.
+|gremlin.tinkergraph.edgeIdManager |The `IdManager` implementation to use for edges.
+|gremlin.tinkergraph.vertexPropertyIdManager |The `IdManager` implementation to use for vertex properties.
+|gremlin.tinkergraph.defaultVertexPropertyCardinality |The default `VertexProperty.Cardinality` to use when `Vertex.property(k,v)` is called.
+|gremlin.tinkergraph.graphLocation |The path and file name for where TinkerGraph should persist the graph data. If a
+value is specified here, the the `gremlin.tinkergraph.graphFormat` should also be specified. If this value is not
+included (default), then the graph will stay in-memory and not be loaded/persisted to disk.
+|gremlin.tinkergraph.graphFormat |The format to use to serialize the graph which may be one of the following:
+`graphml`, `graphson`, `gryo`, or a fully qualified class name that implements Io.Builder interface (which allows for
+external third party graph reader/writer formats to be used for persistence).
+If a value is specified here, then the `gremlin.tinkergraph.graphLocation` should
+also be specified. If this value is not included (default), then the graph will stay in-memory and not be
+loaded/persisted to disk.
+|=========================================================
+
+The `IdManager` settings above refer to how TinkerGraph will control identifiers for vertices, edges and vertex
+properties. There are several options for each of these settings: `ANY`, `LONG`, `INTEGER`, `UUID`, or the fully
+qualified class name of an `IdManager` implementation on the classpath. When not specified, the default values
+for all settings is `ANY`, meaning that the graph will work with any object on the JVM as the identifier and will
+generate new identifiers from `Long` when the identifier is not user supplied. TinkerGraph will also expect the
+user to understand the types used for identifiers when querying, meaning that `g.V(1)` and `g.V(1L)` could return
+two different vertices. `LONG`, `INTEGER` and `UUID` settings will try to coerce identifier values to the expected
+type as well as generate new identifiers with that specified type.
+
+If the TinkerGraph is configured for persistence with `gremlin.tinkergraph.graphLocation` and
+`gremlin.tinkergraph.graphFormat`, then the graph will be written to the specified location with the specified
+format when `Graph.close()` is called. In addition, if these settings are present, TinkerGraph will attempt to
+load the graph from the specified location.
+
+IMPORTANT: If choosing `graphson` as the `gremlin.tinkergraph.graphFormat`, be sure to also establish the various
+`IdManager` settings as well to ensure that identifiers are properly coerced to the appropriate types as GraphSON
+can lose the identifier's type during serialization (i.e. it will assume `Integer` when the default for TinkerGraph
+is `Long`, which could lead to load errors that result in a message like, "Vertex with id already exists").
+
+It is important to consider the data being imported to TinkerGraph with respect to `defaultVertexPropertyCardinality`
+setting. For example, if a `.gryo` file is known to contain multi-property data, be sure to set the default
+cardinality to `list` or else the data will import as `single`. Consider the following:
+
+[gremlin-groovy]
+----
+graph = TinkerGraph.open()
+graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
+g = graph.traversal()
+g.V().properties()
+conf = new BaseConfiguration()
+conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality","list")
+graph = TinkerGraph.open(conf)
+graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
+g = graph.traversal()
+g.V().properties()
+----
+
+[[neo4j-gremlin]]
+Neo4j-Gremlin
+-------------
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>neo4j-gremlin</artifactId>
+ <version>x.y.z</version>
+</dependency>
+<!-- neo4j-tinkerpop-api-impl is NOT Apache 2 licensed - more information below -->
+<dependency>
+ <groupId>org.neo4j</groupId>
+ <artifactId>neo4j-tinkerpop-api-impl</artifactId>
+ <version>0.1-2.2</version>
+</dependency>
+----
+
+link:http://neotechnology.com[Neo Technology] are the developers of the OLTP-based link:http://neo4j.org[Neo4j graph database].
+
+CAUTION: Unless under a commercial agreement with Neo Technology, Neo4j is licensed
+link:http://en.wikipedia.org/wiki/Affero_General_Public_License[AGPL]. The `neo4j-gremlin` module is licensed Apache2
+because it only references the Apache2-licensed Neo4j API (not its implementation). Note that neither the
+<<gremlin-console,Gremlin Console>> nor <<gremlin-server,Gremlin Server>> distribute with the Neo4j implementation
+binaries. To access the binaries, use the `:install` command to download binaries from
+link:http://search.maven.org/[Maven Central Repository].
+
+[source,groovy]
+----
+gremlin> :install org.apache.tinkerpop neo4j-gremlin x.y.z
+==>Loaded: [org.apache.tinkerpop, neo4j-gremlin, x.y.z] - restart the console to use [tinkerpop.neo4j]
+gremlin> :q
+...
+gremlin> :plugin use tinkerpop.neo4j
+==>tinkerpop.neo4j activated
+gremlin> graph = Neo4jGraph.open('/tmp/neo4j')
+==>neo4jgraph[EmbeddedGraphDatabase [/tmp/neo4j]]
+----
+
+NOTE: Neo4j link:http://docs.neo4j.org/chunked/stable/ha.html[High Availability] is currently not supported by
+Neo4j-Gremlin.
+
+TIP: To host Neo4j in <<gremlin-server,Gremlin Server>>, the dependencies must first be "installed" or otherwise
+copied to the Gremlin Server path. The automated method for doing this would be to execute
+`bin/gremlin-server.sh -i org.apache.tinkerpop neo4j-gremlin x.y.z`.
+
+Indices
+~~~~~~~
+
+Neo4j 2.x indices leverage vertex labels to partition the index space. TinkerPop3 does not provide method interfaces
+for defining schemas/indices for the underlying graph system. Thus, in order to create indices, it is important to
+call the Neo4j API directly.
+
+NOTE: `Neo4jGraphStep` will attempt to discern which indices to use when executing a traversal of the form `g.V().has()`.
+
+The Gremlin-Console session below demonstrates Neo4j indices. For more information, please refer to the Neo4j documentation:
+
+* Manipulating indices with link:http://docs.neo4j.org/chunked/stable/query-schema-index.html[Cypher].
+* Manipulating indices with the Neo4j link:http://docs.neo4j.org/chunked/stable/tutorials-java-embedded-new-index.html[Java API].
+
+[gremlin-groovy]
+----
+graph = Neo4jGraph.open('/tmp/neo4j')
+graph.cypher("CREATE INDEX ON :person(name)")
+graph.tx().commit() <1>
+graph.addVertex(label,'person','name','marko')
+graph.addVertex(label,'dog','name','puppy')
+g = graph.traversal()
+g.V().hasLabel('person').has('name','marko').values('name')
+graph.close()
+----
+
+<1> Schema mutations must happen in a different transaction than graph mutations
+
+Below demonstrates the runtime benefits of indices and demonstrates how if there is no defined index (only vertex
+labels), a linear scan of the vertex-label partition is still faster than a linear scan of all vertices.
+
+[gremlin-groovy]
+----
+graph = Neo4jGraph.open('/tmp/neo4j')
+graph.io(graphml()).readGraph('data/grateful-dead.xml')
+g = graph.traversal()
+g.tx().commit()
+clock(1000) {g.V().hasLabel('artist').has('name','Garcia').iterate()} <1>
+graph.cypher("CREATE INDEX ON :artist(name)") <2>
+g.tx().commit()
+Thread.sleep(5000) <3>
+clock(1000) {g.V().hasLabel('artist').has('name','Garcia').iterate()} <4>
+clock(1000) {g.V().has('name','Garcia').iterate()} <5>
+graph.cypher("DROP INDEX ON :artist(name)") <6>
+g.tx().commit()
+graph.close()
+----
+
+<1> Find all artists whose name is Garcia which does a linear scan of the artist vertex-label partition.
+<2> Create an index for all artist vertices on their name property.
+<3> Neo4j indices are eventually consistent so this stalls to give the index time to populate itself.
+<4> Find all artists whose name is Garcia which uses the pre-defined schema index.
+<5> Find all vertices whose name is Garcia which requires a linear scan of all the data in the graph.
+<6> Drop the created index.
+
+Multi/Meta-Properties
+~~~~~~~~~~~~~~~~~~~~~
+
+`Neo4jGraph` supports both multi- and meta-properties (see <<_vertex_properties,vertex properties>>). These features
+are not native to Neo4j and are implemented using "hidden" Neo4j nodes. For example, when a vertex has multiple
+"name" properties, each property is a new node (multi-properties) which can have properties attached to it
+(meta-properties). As such, the native, underlying representation may become difficult to query directly using
+another graph language such as <<_cypher,Cypher>>. The default setting is to disable multi- and meta-properties.
+However, if this feature is desired, then it can be activated via `gremlin.neo4j.metaProperties` and
+`gremlin.neo4j.multiProperties` configurations being set to `true`. Once the configuration is set, it can not be
+changed for the lifetime of the graph.
+
+[gremlin-groovy]
+----
+conf = new BaseConfiguration()
+conf.setProperty('gremlin.neo4j.directory','/tmp/neo4j')
+conf.setProperty('gremlin.neo4j.multiProperties',true)
+conf.setProperty('gremlin.neo4j.metaProperties',true)
+graph = Neo4jGraph.open(conf)
+g = graph.traversal()
+g.addV('name','michael','name','michael hunger','name','mhunger')
+g.V().properties('name').property('acl', 'public')
+g.V(0).valueMap()
+g.V(0).properties()
+g.V(0).properties().valueMap()
+graph.close()
+----
+
+WARNING: `Neo4jGraph` without multi- and meta-properties is in 1-to-1 correspondence with the native, underlying Neo4j
+representation. It is recommended that if the user does not require multi/meta-properties, then they should not
+enable them. Without multi- and meta-properties enabled, Neo4j can be interacted with with other tools and technologies
+that do not leverage TinkerPop.
+
+IMPORTANT: When using a multi-property enabled `Neo4jGraph`, vertices may represent their properties on "hidden
+nodes" adjacent to the vertex. If a vertex property key/value is required for indexing, then two indices are
+required -- e.g. `CREATE INDEX ON :person(name)` and `CREATE INDEX ON :vertexProperty(name)`
+(see <<_indices,Neo4j indices>>).
+
+Cypher
+~~~~~~
+
+image::gremlin-loves-cypher.png[width=400]
+
+NeoTechnology are the creators of the graph pattern-match query language link:http://www.neo4j.org/learn/cypher[Cypher].
+It is possible to leverage Cypher from within Gremlin by using the `Neo4jGraph.cypher()` graph traversal method.
+
+[gremlin-groovy]
+----
+graph = Neo4jGraph.open('/tmp/neo4j')
+graph.io(gryo()).readGraph('data/tinkerpop-modern.kryo')
+graph.cypher('MATCH (a {name:"marko"}) RETURN a')
+graph.cypher('MATCH (a {name:"marko"}) RETURN a').select('a').out('knows').values('name')
+graph.close()
+----
+
+Thus, like <<match-step,`match()`>>-step in Gremlin, it is possible to do a declarative pattern match and then move
+back into imperative Gremlin.
+
+TIP: For those developers using <<gremlin-server,Gremlin Server>> against Neo4j, it is possible to do Cypher queries
+by simply placing the Cypher string in `graph.cypher(...)` before submission to the server.
+
+Multi-Label
+~~~~~~~~~~~
+
+TinkerPop3 requires every `Element` to have a single, immutable string label (i.e. a `Vertex`, `Edge`, and
+`VertexProperty`). In Neo4j, a `Node` (vertex) can have an
+link:http://neo4j.com/docs/stable/graphdb-neo4j-labels.html[arbitrary number of labels] while a `Relationship`
+(edge) can have one and only one. Furthermore, in Neo4j, `Node` labels are mutable while `Relationship` labels are
+not. In order to handle this mismatch, three `Neo4jVertex` specific methods exist in Neo4j-Gremlin.
+
+[source,java]
+public Set<String> labels() // get all the labels of the vertex
+public void addLabel(String label) // add a label to the vertex
+public void removeLabel(String label) // remove a label from the vertex
+
+An example use case is presented below.
+
+[gremlin-groovy]
+----
+graph = Neo4jGraph.open('/tmp/neo4j')
+vertex = (Neo4jVertex) graph.addVertex('human::animal') <1>
+vertex.label() <2>
+vertex.labels() <3>
+vertex.addLabel('organism') <4>
+vertex.label()
+vertex.removeLabel('human') <5>
+vertex.labels()
+vertex.addLabel('organism') <6>
+vertex.labels()
+vertex.removeLabel('human') <7>
+vertex.label()
+g = graph.traversal()
+g.V().has(label,'organism') <8>
+g.V().has(label,of('organism')) <9>
+g.V().has(label,of('organism')).has(label,of('animal'))
+g.V().has(label,of('organism').and(of('animal')))
+graph.close()
+----
+
+<1> Typecasting to a `Neo4jVertex` is only required in Java.
+<2> The standard `Vertex.label()` method returns all the labels in alphabetical order concatenated using `::`.
+<3> `Neo4jVertex.labels()` method returns the individual labels as a set.
+<4> `Neo4jVertex.addLabel()` method adds a single label.
+<5> `Neo4jVertex.removeLabel()` method removes a single label.
+<6> Labels are unique and thus duplicate labels don't exist.
+<7> If a label that does not exist is removed, nothing happens.
+<8> `P.eq()` does a full string match and should only be used if multi-labels are not leveraged.
+<9> `LabelP.of()` is specific to `Neo4jGraph` and used for multi-label matching.
+
+IMPORTANT: `LabelP.of()` is only required if multi-labels are leveraged. `LabelP.of()` is used when
+filtering/looking-up vertices by their label(s) as the standard `P.eq()` does a direct match on the `::`-representation
+of `vertex.label()`
+
+Loading with BulkLoaderVertexProgram
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load
+large amounts of data to and from Neo4j. The following code demonstrates how to load the modern graph from TinkerGraph
+into Neo4j:
+
+[gremlin-groovy]
+----
+wgConf = 'conf/neo4j-standalone.properties'
+modern = TinkerFactory.createModern()
+blvp = BulkLoaderVertexProgram.build().
+ keepOriginalIds(false).
+ writeGraph(wgConf).create(modern)
+modern.compute().workers(1).program(blvp).submit().get()
+graph = GraphFactory.open(wgConf)
+g = graph.traversal()
+g.V().valueMap()
+graph.close()
+----
+
+[source,properties]
+----
+# neo4j-standalone.properties
+
+gremlin.graph=org.apache.tinkerpop.gremlin.neo4j.structure.Neo4jGraph
+gremlin.neo4j.directory=/tmp/neo4j
+gremlin.neo4j.conf.node_auto_indexing=true
+gremlin.neo4j.conf.relationship_auto_indexing=true
+----
[10/43] incubator-tinkerpop git commit: splitted
implementations-neo4j.asciidoc further into implementations-intro.asciidoc,
implementations-tinkergraph.asciidoc and implementations-neo4j.asciidoc
Posted by sp...@apache.org.
splitted implementations-neo4j.asciidoc further into implementations-intro.asciidoc, implementations-tinkergraph.asciidoc and implementations-neo4j.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c9149c23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c9149c23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c9149c23
Branch: refs/heads/TINKERPOP-1107
Commit: c9149c2350fe45df60e0bbed9a6e6257f911a152
Parents: e9c6465
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Fri Feb 19 17:54:30 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Fri Feb 19 17:54:30 2016 +0100
----------------------------------------------------------------------
.../reference/implementations-intro.asciidoc | 545 +++++++++++++++
.../reference/implementations-neo4j.asciidoc | 660 -------------------
.../implementations-tinkergraph.asciidoc | 144 ++++
docs/src/reference/index.asciidoc | 2 +
4 files changed, 691 insertions(+), 660 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c9149c23/docs/src/reference/implementations-intro.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-intro.asciidoc b/docs/src/reference/implementations-intro.asciidoc
new file mode 100644
index 0000000..5357ce3
--- /dev/null
+++ b/docs/src/reference/implementations-intro.asciidoc
@@ -0,0 +1,545 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+[[implementations]]
+Implementations
+===============
+
+image::gremlin-racecar.png[width=325]
+
+[[graph-system-provider-requirements]]
+Graph System Provider Requirements
+----------------------------------
+
+image:tinkerpop-enabled.png[width=140,float=left] At the core of TinkerPop3 is a Java8 API. The implementation of this
+core API and its validation via the `gremlin-test` suite is all that is required of a graph system provider wishing to
+provide a TinkerPop3-enabled graph engine. Once a graph system has a valid implementation, then all the applications
+provided by TinkerPop (e.g. Gremlin Console, Gremlin Server, etc.) and 3rd-party developers (e.g. Gremlin-Scala,
+Gremlin-JS, etc.) will integrate properly. Finally, please feel free to use the logo on the left to promote your
+TinkerPop3 implementation.
+
+Implementing Gremlin-Core
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The classes that a graph system provider should focus on implementing are itemized below. It is a good idea to study
+the <<tinkergraph-gremlin,TinkerGraph>> (in-memory OLTP and OLAP in `tinkergraph-gremlin`), <<neo4j-gremlin,Neo4jGraph>>
+(OTLP w/ transactions in `neo4j-gremlin`) and/or <<hadoop-gremlin,HadoopGraph>> (OLAP in `hadoop-gremlin`)
+implementations for ideas and patterns.
+
+. Online Transactional Processing Graph Systems (*OLTP*)
+ .. Structure API: `Graph`, `Element`, `Vertex`, `Edge`, `Property` and `Transaction` (if transactions are supported).
+ .. Process API: `TraversalStrategy` instances for optimizing Gremlin traversals to the provider's graph system (i.e. `TinkerGraphStepStrategy`).
+. Online Analytics Processing Graph Systems (*OLAP*)
+ .. Everything required of OTLP is required of OLAP (but not vice versa).
+ .. GraphComputer API: `GraphComputer`, `Messenger`, `Memory`.
+
+Please consider the following implementation notes:
+
+* Be sure your `Graph` implementation is named as `XXXGraph` (e.g. TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
+* Use `StringHelper` to ensuring that the `toString()` representation of classes are consistent with other implementations.
+* Ensure that your implementation's `Features` (Graph, Vertex, etc.) are correct so that test cases handle particulars accordingly.
+* Use the numerous static method helper classes such as `ElementHelper`, `GraphComputerHelper`, `VertexProgramHelper`, etc.
+* There are a number of default methods on the provided interfaces that are semantically correct. However, if they are
+not efficient for the implementation, override them.
+* Implement the `structure/` package interfaces first and then, if desired, interfaces in the `process/` package interfaces.
+* `ComputerGraph` is a `Wrapper` system that ensure proper semantics during a GraphComputer computation.
+
+[[oltp-implementations]]
+OLTP Implementations
+^^^^^^^^^^^^^^^^^^^^
+
+image:pipes-character-1.png[width=110,float=right] The most important interfaces to implement are in the `structure/`
+package. These include interfaces like Graph, Vertex, Edge, Property, Transaction, etc. The `StructureStandardSuite`
+will ensure that the semantics of the methods implemented are correct. Moreover, there are numerous `Exceptions`
+classes with static exceptions that should be thrown by the graph system so that all the exceptions and their
+messages are consistent amongst all TinkerPop3 implementations.
+
+[[olap-implementations]]
+OLAP Implementations
+^^^^^^^^^^^^^^^^^^^^
+
+image:furnace-character-1.png[width=110,float=right] Implementing the OLAP interfaces may be a bit more complicated.
+Note that before OLAP interfaces are implemented, it is necessary for the OLTP interfaces to be, at minimal,
+implemented as specified in <<oltp-implementations,OLTP Implementations>>. A summary of each required interface
+implementation is presented below:
+
+. `GraphComputer`: A fluent builder for specifying an isolation level, a VertexProgram, and any number of MapReduce jobs to be submitted.
+. `Memory`: A global blackboard for ANDing, ORing, INCRing, and SETing values for specified keys.
+. `Messenger`: The system that collects and distributes messages being propagated by vertices executing the VertexProgram application.
+. `MapReduce.MapEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications map-phase.
+. `MapReduce.ReduceEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications combine- and reduce-phases.
+
+NOTE: The VertexProgram and MapReduce interfaces in the `process/computer/` package are not required by the graph
+system. Instead, these are interfaces to be implemented by application developers writing VertexPrograms and MapReduce jobs.
+
+IMPORTANT: TinkerPop3 provides three OLAP implementations: <<tinkergraph-gremlin,TinkerGraphComputer>> (TinkerGraph),
+<<giraphgraphcomputer,GiraphGraphComputer>> (HadoopGraph), and <<sparkgraphcomputer,`SparkGraphComputer`>> (Hadoop).
+Given the complexity of the OLAP system, it is good to study and copy many of the patterns used in these reference
+implementations.
+
+Implementing GraphComputer
+++++++++++++++++++++++++++
+
+image:furnace-character-3.png[width=150,float=right] The most complex method in GraphComputer is the `submit()`-method. The method must do the following:
+
+. Ensure the the GraphComputer has not already been executed.
+. Ensure that at least there is a VertexProgram or 1 MapReduce job.
+. If there is a VertexProgram, validate that it can execute on the GraphComputer given the respectively defined features.
+. Create the Memory to be used for the computation.
+. Execute the VertexProgram.setup() method once and only once.
+. Execute the VertexProgram.execute() method for each vertex.
+. Execute the VertexProgram.terminate() method once and if true, repeat VertexProgram.execute().
+. When VertexProgram.terminate() returns true, move to MapReduce job execution.
+. MapReduce jobs are not required to be executed in any specified order.
+. For each Vertex, execute MapReduce.map(). Then (if defined) execute MapReduce.combine() and MapReduce.reduce().
+. Update Memory with runtime information.
+. Construct a new `ComputerResult` containing the compute Graph and Memory.
+
+Implementing Memory
++++++++++++++++++++
+
+image:gremlin-brain.png[width=175,float=left] The Memory object is initially defined by `VertexProgram.setup()`.
+The memory data is available in the first round of the `VertexProgram.execute()` method. Each Vertex, when executing
+the VertexProgram, can update the Memory in its round. However, the update is not seen by the other vertices until
+the next round. At the end of the first round, all the updates are aggregated and the new memory data is available
+on the second round. This process repeats until the VertexProgram terminates.
+
+Implementing Messenger
+++++++++++++++++++++++
+
+The Messenger object is similar to the Memory object in that a vertex can read and write to the Messenger. However,
+the data it reads are the messages sent to the vertex in the previous step and the data it writes are the messages
+that will be readable by the receiving vertices in the subsequent round.
+
+Implementing MapReduce Emitters
++++++++++++++++++++++++++++++++
+
+image:hadoop-logo-notext.png[width=150,float=left] The MapReduce framework in TinkerPop3 is similar to the model
+popularized by link:http://apache.hadoop.org[Hadoop]. The primary difference is that all Mappers process the vertices
+of the graph, not an arbitrary key/value pair. However, the vertices' edges can not be accessed -- only their
+properties. This greatly reduces the amount of data needed to be pushed through the MapReduce engine as any edge
+information required, can be computed in the VertexProgram.execute() method. Moreover, at this stage, vertices can
+not be mutated, only their token and property data read. A Gremlin OLAP system needs to provide implementations for
+to particular classes: `MapReduce.MapEmitter` and `MapReduce.ReduceEmitter`. TinkerGraph's implementation is provided
+below which demonstrates the simplicity of the algorithm (especially when the data is all within the same JVM).
+
+[source,java]
+----
+public class TinkerMapEmitter<K, V> implements MapReduce.MapEmitter<K, V> {
+
+ public Map<K, Queue<V>> reduceMap;
+ public Queue<KeyValue<K, V>> mapQueue;
+ private final boolean doReduce;
+
+ public TinkerMapEmitter(final boolean doReduce) { <1>
+ this.doReduce = doReduce;
+ if (this.doReduce)
+ this.reduceMap = new ConcurrentHashMap<>();
+ else
+ this.mapQueue = new ConcurrentLinkedQueue<>();
+ }
+
+ @Override
+ public void emit(K key, V value) {
+ if (this.doReduce)
+ this.reduceMap.computeIfAbsent(key, k -> new ConcurrentLinkedQueue<>()).add(value); <2>
+ else
+ this.mapQueue.add(new KeyValue<>(key, value)); <3>
+ }
+
+ protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) {
+ if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { <4>
+ final Comparator<K> comparator = mapReduce.getMapKeySort().get();
+ final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue);
+ Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
+ this.mapQueue.clear();
+ this.mapQueue.addAll(list);
+ } else if (mapReduce.getMapKeySort().isPresent()) {
+ final Comparator<K> comparator = mapReduce.getMapKeySort().get();
+ final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>();
+ list.addAll(this.reduceMap.entrySet());
+ Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator));
+ this.reduceMap = new LinkedHashMap<>();
+ list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue()));
+ }
+ }
+}
+----
+
+<1> If the MapReduce job has a reduce, then use one data structure (`reduceMap`), else use another (`mapList`). The
+difference being that a reduction requires a grouping by key and therefore, the `Map<K,Queue<V>>` definition. If no
+reduction/grouping is required, then a simple `Queue<KeyValue<K,V>>` can be leveraged.
+<2> If reduce is to follow, then increment the Map with a new value for the key. `MapHelper` is a TinkerPop3 class
+with static methods for adding data to a Map.
+<3> If no reduce is to follow, then simply append a KeyValue to the queue.
+<4> When the map phase is complete, any map-result sorting required can be executed at this point.
+
+[source,java]
+----
+public class TinkerReduceEmitter<OK, OV> implements MapReduce.ReduceEmitter<OK, OV> {
+
+ protected Queue<KeyValue<OK, OV>> reduceQueue = new ConcurrentLinkedQueue<>();
+
+ @Override
+ public void emit(final OK key, final OV value) {
+ this.reduceQueue.add(new KeyValue<>(key, value));
+ }
+
+ protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) {
+ if (mapReduce.getReduceKeySort().isPresent()) {
+ final Comparator<OK> comparator = mapReduce.getReduceKeySort().get();
+ final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue);
+ Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
+ this.reduceQueue.clear();
+ this.reduceQueue.addAll(list);
+ }
+ }
+}
+----
+
+The method `MapReduce.reduce()` is defined as:
+
+[source,java]
+public void reduce(final OK key, final Iterator<OV> values, final ReduceEmitter<OK, OV> emitter) { ... }
+
+In other words, for the TinkerGraph implementation, iterate through the entrySet of the `reduceMap` and call the
+`reduce()` method on each entry. The `reduce()` method can emit key/value pairs which are simply aggregated into a
+`Queue<KeyValue<OK,OV>>` in an analogous fashion to `TinkerMapEmitter` when no reduce is to follow. These two emitters
+are tied together in `TinkerGraphComputer.submit()`.
+
+[source,java]
+----
+...
+for (final MapReduce mapReduce : mapReducers) {
+ if (mapReduce.doStage(MapReduce.Stage.MAP)) {
+ final TinkerMapEmitter<?, ?> mapEmitter = new TinkerMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
+ final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(this.graph.vertices());
+ workers.setMapReduce(mapReduce);
+ workers.mapReduceWorkerStart(MapReduce.Stage.MAP);
+ workers.executeMapReduce(workerMapReduce -> {
+ while (true) {
+ final Vertex vertex = vertices.next();
+ if (null == vertex) return;
+ workerMapReduce.map(ComputerGraph.mapReduce(vertex), mapEmitter);
+ }
+ });
+ workers.mapReduceWorkerEnd(MapReduce.Stage.MAP);
+
+ // sort results if a map output sort is defined
+ mapEmitter.complete(mapReduce);
+
+ // no need to run combiners as this is single machine
+ if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
+ final TinkerReduceEmitter<?, ?> reduceEmitter = new TinkerReduceEmitter<>();
+ final SynchronizedIterator<Map.Entry<?, Queue<?>>> keyValues = new SynchronizedIterator((Iterator) mapEmitter.reduceMap.entrySet().iterator());
+ workers.mapReduceWorkerStart(MapReduce.Stage.REDUCE);
+ workers.executeMapReduce(workerMapReduce -> {
+ while (true) {
+ final Map.Entry<?, Queue<?>> entry = keyValues.next();
+ if (null == entry) return;
+ workerMapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter);
+ }
+ });
+ workers.mapReduceWorkerEnd(MapReduce.Stage.REDUCE);
+ reduceEmitter.complete(mapReduce); // sort results if a reduce output sort is defined
+ mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator()); <1>
+ } else {
+ mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator()); <2>
+ }
+ }
+}
+...
+----
+
+<1> Note that the final results of the reducer are provided to the Memory as specified by the application developer's
+`MapReduce.addResultToMemory()` implementation.
+<2> If there is no reduce stage, the the map-stage results are inserted into Memory as specified by the application
+developer's `MapReduce.addResultToMemory()` implementation.
+
+[[io-implementations]]
+IO Implementations
+^^^^^^^^^^^^^^^^^^
+
+If a `Graph` requires custom serializers for IO to work properly, implement the `Graph.io` method. A typical example
+of where a `Graph` would require such a custom serializers is if their identifier system uses non-primitive values,
+such as OrientDB's `Rid` class. From basic serialization of a single `Vertex` all the way up the stack to Gremlin
+Server, the need to know how to handle these complex identifiers is an important requirement.
+
+The first step to implementing custom serializers is to first implement the `IoRegistry` interface and register the
+custom classes and serializers to it. Each `Io` implementation has different requirements for what it expects from the
+`IoRegistry`:
+
+* *GraphML* - No custom serializers expected/allowed.
+* *GraphSON* - Register a Jackson `SimpleModule`. The `SimpleModule` encapsulates specific classes to be serialized,
+so it does not need to be registered to a specific class in the `IoRegistry` (use `null`).
+* *Gryo* - Expects registration of one of three objects:
+** Register just the custom class with a `null` Kryo `Serializer` implementation - this class will use default "field-level" Kryo serialization.
+** Register the custom class with a specific Kryo `Serializer' implementation.
+** Register the custom class with a `Function<Kryo, Serializer>` for those cases where the Kryo `Serializer` requires the `Kryo` instance to get constructed.
+
+This implementation should provide a zero-arg constructor as the stack may require instantiation via reflection.
+Consider extending `AbstractIoRegistry` for convenience as follows:
+
+[source,java]
+----
+public class MyGraphIoRegistry extends AbstractIoRegistry {
+ public MyGraphIoRegistry() {
+ register(GraphSONIo.class, null, new MyGraphSimpleModule());
+ register(GryoIo.class, MyGraphIdClass.class, new MyGraphIdSerializer());
+ }
+}
+----
+
+In the `Graph.io` method, provide the `IoRegistry` object to the supplied `Builder` and call the `create` method to
+return that `Io` instance as follows:
+
+[source,java]
+----
+public <I extends Io> I io(final Io.Builder<I> builder) {
+ return (I) builder.graph(this).registry(myGraphIoRegistry).create();
+}}
+----
+
+In this way, `Graph` implementations can pre-configure custom serializers for IO interactions and users will not need
+to know about those details. Following this pattern will ensure proper execution of the test suite as well as
+simplified usage for end-users.
+
+IMPORTANT: Proper implementation of IO is critical to successful `Graph` operations in Gremlin Server. The Test Suite
+does have "serialization" tests that provide some assurance that an implementation is working properly, but those
+tests cannot make assertions against any specifics of a custom serializer. It is the responsibility of the
+implementer to test the specifics of their custom serializers.
+
+TIP: Consider separating serializer code into its own module, if possible, so that clients that use the `Graph`
+implementation remotely don't need a full dependency on the entire `Graph` - just the IO components and related
+classes being serialized.
+
+[[validating-with-gremlin-test]]
+Validating with Gremlin-Test
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+image:gremlin-edumacated.png[width=225]
+
+[source,xml]
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-test</artifactId>
+ <version>x.y.z</version>
+</dependency>
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy-test</artifactId>
+ <version>x.y.z</version>
+</dependency>
+
+The operational semantics of any OLTP or OLAP implementation are validated by `gremlin-test` and functional
+interoperability with the Groovy environment is ensured by `gremlin-groovy-test`. To implement these tests, provide
+test case implementations as shown below, where `XXX` below denotes the name of the graph implementation (e.g.
+TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
+
+[source,java]
+----
+// Structure API tests
+@RunWith(StructureStandardSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
+public class XXXStructureStandardTest {}
+
+// Process API tests
+@RunWith(ProcessComputerSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
+public class XXXProcessComputerTest {}
+
+@RunWith(ProcessStandardSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
+public class XXXProcessStandardTest {}
+
+@RunWith(GroovyEnvironmentSuite.class)
+@GraphProviderClass(provider = XXXProvider.class, graph = TinkerGraph.class)
+public class XXXGroovyEnvironmentTest {}
+
+@RunWith(GroovyProcessStandardSuite.class)
+@GraphProviderClass(provider = XXXGraphProvider.class, graph = TinkerGraph.class)
+public class XXXGroovyProcessStandardTest {}
+
+@RunWith(GroovyProcessComputerSuite.class)
+@GraphProviderClass(provider = XXXGraphComputerProvider.class, graph = TinkerGraph.class)
+public class XXXGroovyProcessComputerTest {}
+----
+
+The above set of tests represent the minimum test suite set to implement. There are other "integration" and
+"performance" tests that should be considered optional. Implementing those tests requires the same pattern as shown above.
+
+IMPORTANT: It is as important to look at "ignored" tests as it is to look at ones that fail. The `gremlin-test`
+suite utilizes the `Feature` implementation exposed by the `Graph` to determine which tests to execute. If a test
+utilizes features that are not supported by the graph, it will ignore them. While that may be fine, implementers
+should validate that the ignored tests are appropriately bypassed and that there are no mistakes in their feature
+definitions. Moreover, implementers should consider filling gaps in their own test suites, especially when
+IO-related tests are being ignored.
+
+The only test-class that requires any code investment is the `GraphProvider` implementation class. This class is a
+used by the test suite to construct `Graph` configurations and instances and provides information about the
+implementation itself. In most cases, it is best to simply extend `AbstractGraphProvider` as it provides many
+default implementations of the `GraphProvider` interface.
+
+Finally, specify the test suites that will be supported by the `Graph` implementation using the `@Graph.OptIn`
+annotation. See the `TinkerGraph` implementation below as an example:
+
+[source,java]
+----
+@Graph.OptIn(Graph.OptIn.SUITE_STRUCTURE_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
+@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_COMPUTER)
+@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT)
+public class TinkerGraph implements Graph {
+----
+
+Only include annotations for the suites the implementation will support. Note that implementing the suite, but
+not specifying the appropriate annotation will prevent the suite from running (an obvious error message will appear
+in this case when running the mis-configured suite).
+
+There are times when there may be a specific test in the suite that the implementation cannot support (despite the
+features it implements) or should not otherwise be executed. It is possible for implementers to "opt-out" of a test
+by using the `@Graph.OptOut` annotation. The following is an example of this annotation usage as taken from
+`HadoopGraph`:
+
+[source, java]
+----
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
+@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
+ method = "g_V_matchXa_hasXname_GarciaX__a_inXwrittenByX_b__a_inXsungByX_bX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
+ method = "g_V_matchXa_inXsungByX_b__a_inXsungByX_c__b_outXwrittenByX_d__c_outXwrittenByX_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
+ method = "shouldNotAllowBadMemoryKeys",
+ reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
+ method = "shouldRequireRegisteringMemoryKeys",
+ reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+public class HadoopGraph implements Graph {
+----
+
+The above examples show how to ignore individual tests. It is also possible to:
+
+* Ignore an entire test case (i.e. all the methods within the test) by setting the `method` to "*".
+* Ignore a "base" test class such that test that extend from those classes will all be ignored. This style of
+ignoring is useful for Gremlin "process" tests that have bases classes that are extended by various Gremlin flavors (e.g. groovy).
+* Ignore a `GraphComputer` test based on the type of `GraphComputer` being used. Specify the "computer" attribute on
+the `OptOut` (which is an array specification) which should have a value of the `GraphComputer` implementation class
+that should ignore that test. This attribute should be left empty for "standard" execution and by default all
+`GraphComputer` implementations will be included in the `OptOut` so if there are multiple implementations, explicitly
+specify the ones that should be excluded.
+
+Also note that some of the tests in the Gremlin Test Suite are parameterized tests and require an additional level of
+specificity to be properly ignored. To ignore these types of tests, examine the name template of the parameterized
+tests. It is defined by a Java annotation that looks like this:
+
+[source, java]
+@Parameterized.Parameters(name = "expect({0})")
+
+The annotation above shows that the name of each parameterized test will be prefixed with "expect" and have
+parentheses wrapped around the first parameter (at index 0) value supplied to each test. This information can
+only be garnered by studying the test set up itself. Once the pattern is determined and the specific unique name of
+the parameterized test is identified, add it to the `specific` property on the `OptOut` annotation in addition to
+the other arguments.
+
+These annotations help provide users a level of transparency into test suite compliance (via the
+xref:describe-graph[describeGraph()] utility function). It also allows implementers to have a lot of flexibility in
+terms of how they wish to support TinkerPop. For example, maybe there is a single test case that prevents an
+implementer from claiming support of a `Feature`. The implementer could choose to either not support the `Feature`
+or to support it but "opt-out" of the test with a "reason" as to why so that users understand the limitation.
+
+IMPORTANT: Before using `OptOut` be sure that the reason for using it is sound and it is more of a last resort.
+It is possible that a test from the suite doesn't properly represent the expectations of a feature, is too broad or
+narrow for the semantics it is trying to enforce or simply contains a bug. Please consider raising issues in the
+developer mailing list with such concerns before assuming `OptOut` is the only answer.
+
+IMPORTANT: There are no tests that specifically validate complete compliance with Gremlin Server. Generally speaking,
+a `Graph` that passes the full Test Suite, should be compliant with Gremlin Server. The one area where problems can
+occur is in serialization. Always ensure that IO is properly implemented, that custom serializers are tested fully
+and ultimately integration test the `Graph` with an actual Gremlin Server instance.
+
+CAUTION: Configuring tests to run in parallel might result in errors that are difficult to debug as there is some
+shared state in test execution around graph configuration. It is therefore recommended that parallelism be turned
+off for the test suite (the Maven SureFire Plugin is configured this way by default). It may also be important to
+include this setting, `<reuseForks>false</reuseForks>`, in the SureFire configuration if tests are failing in an
+unexplainable way.
+
+Accessibility via GremlinPlugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+image:gremlin-plugin.png[width=100,float=left] The applications distributed with TinkerPop3 do not distribute with
+any graph system implementations besides TinkerGraph. If your implementation is stored in a Maven repository (e.g.
+Maven Central Repository), then it is best to provide a `GremlinPlugin` implementation so the respective jars can be
+downloaded according and when required by the user. Neo4j's GremlinPlugin is provided below for reference.
+
+[source,java]
+----
+public class Neo4jGremlinPlugin implements GremlinPlugin {
+
+ private static final String IMPORT = "import ";
+ private static final String DOT_STAR = ".*";
+
+ private static final Set<String> IMPORTS = new HashSet<String>() {{
+ add(IMPORT + Neo4jGraph.class.getPackage().getName() + DOT_STAR);
+ }};
+
+ @Override
+ public String getName() {
+ return "neo4j";
+ }
+
+ @Override
+ public void pluginTo(final PluginAcceptor pluginAcceptor) {
+ pluginAcceptor.addImports(IMPORTS);
+ }
+}
+----
+
+With the above plugin implementations, users can now download respective binaries for Gremlin Console, Gremlin Server, etc.
+
+[source,groovy]
+gremlin> g = Neo4jGraph.open('/tmp/neo4j')
+No such property: Neo4jGraph for class: groovysh_evaluate
+Display stack trace? [yN]
+gremlin> :install org.apache.tinkerpop neo4j-gremlin x.y.z
+==>loaded: [org.apache.tinkerpop, neo4j-gremlin, …]
+gremlin> :plugin use tinkerpop.neo4j
+==>tinkerpop.neo4j activated
+gremlin> g = Neo4jGraph.open('/tmp/neo4j')
+==>neo4jgraph[EmbeddedGraphDatabase [/tmp/neo4j]]
+
+In-Depth Implementations
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+image:gremlin-painting.png[width=200,float=right] The graph system implementation details presented thus far are
+minimum requirements necessary to yield a valid TinkerPop3 implementation. However, there are other areas that a
+graph system provider can tweak to provide an implementation more optimized for their underlying graph engine. Typical
+areas of focus include:
+
+* Traversal Strategies: A <<traversalstrategy,TraversalStrategy>> can be used to alter a traversal prior to its
+execution. A typical example is converting a pattern of `g.V().has('name','marko')` into a global index lookup for
+all vertices with name "marko". In this way, a `O(|V|)` lookup becomes an `O(log(|V|))`. Please review
+`TinkerGraphStepStrategy` for ideas.
+* Step Implementations: Every <<graph-traversal-steps,step>> is ultimately referenced by the `GraphTraversal`
+interface. It is possible to extend `GraphTraversal` to use a graph system specific step implementation.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c9149c23/docs/src/reference/implementations-neo4j.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-neo4j.asciidoc b/docs/src/reference/implementations-neo4j.asciidoc
index 5602754..9accd25 100644
--- a/docs/src/reference/implementations-neo4j.asciidoc
+++ b/docs/src/reference/implementations-neo4j.asciidoc
@@ -14,666 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
////
-[[implementations]]
-Implementations
-===============
-
-image::gremlin-racecar.png[width=325]
-
-[[graph-system-provider-requirements]]
-Graph System Provider Requirements
-----------------------------------
-
-image:tinkerpop-enabled.png[width=140,float=left] At the core of TinkerPop3 is a Java8 API. The implementation of this
-core API and its validation via the `gremlin-test` suite is all that is required of a graph system provider wishing to
-provide a TinkerPop3-enabled graph engine. Once a graph system has a valid implementation, then all the applications
-provided by TinkerPop (e.g. Gremlin Console, Gremlin Server, etc.) and 3rd-party developers (e.g. Gremlin-Scala,
-Gremlin-JS, etc.) will integrate properly. Finally, please feel free to use the logo on the left to promote your
-TinkerPop3 implementation.
-
-Implementing Gremlin-Core
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The classes that a graph system provider should focus on implementing are itemized below. It is a good idea to study
-the <<tinkergraph-gremlin,TinkerGraph>> (in-memory OLTP and OLAP in `tinkergraph-gremlin`), <<neo4j-gremlin,Neo4jGraph>>
-(OTLP w/ transactions in `neo4j-gremlin`) and/or <<hadoop-gremlin,HadoopGraph>> (OLAP in `hadoop-gremlin`)
-implementations for ideas and patterns.
-
-. Online Transactional Processing Graph Systems (*OLTP*)
- .. Structure API: `Graph`, `Element`, `Vertex`, `Edge`, `Property` and `Transaction` (if transactions are supported).
- .. Process API: `TraversalStrategy` instances for optimizing Gremlin traversals to the provider's graph system (i.e. `TinkerGraphStepStrategy`).
-. Online Analytics Processing Graph Systems (*OLAP*)
- .. Everything required of OTLP is required of OLAP (but not vice versa).
- .. GraphComputer API: `GraphComputer`, `Messenger`, `Memory`.
-
-Please consider the following implementation notes:
-
-* Be sure your `Graph` implementation is named as `XXXGraph` (e.g. TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
-* Use `StringHelper` to ensuring that the `toString()` representation of classes are consistent with other implementations.
-* Ensure that your implementation's `Features` (Graph, Vertex, etc.) are correct so that test cases handle particulars accordingly.
-* Use the numerous static method helper classes such as `ElementHelper`, `GraphComputerHelper`, `VertexProgramHelper`, etc.
-* There are a number of default methods on the provided interfaces that are semantically correct. However, if they are
-not efficient for the implementation, override them.
-* Implement the `structure/` package interfaces first and then, if desired, interfaces in the `process/` package interfaces.
-* `ComputerGraph` is a `Wrapper` system that ensure proper semantics during a GraphComputer computation.
-
-[[oltp-implementations]]
-OLTP Implementations
-^^^^^^^^^^^^^^^^^^^^
-
-image:pipes-character-1.png[width=110,float=right] The most important interfaces to implement are in the `structure/`
-package. These include interfaces like Graph, Vertex, Edge, Property, Transaction, etc. The `StructureStandardSuite`
-will ensure that the semantics of the methods implemented are correct. Moreover, there are numerous `Exceptions`
-classes with static exceptions that should be thrown by the graph system so that all the exceptions and their
-messages are consistent amongst all TinkerPop3 implementations.
-
-[[olap-implementations]]
-OLAP Implementations
-^^^^^^^^^^^^^^^^^^^^
-
-image:furnace-character-1.png[width=110,float=right] Implementing the OLAP interfaces may be a bit more complicated.
-Note that before OLAP interfaces are implemented, it is necessary for the OLTP interfaces to be, at minimal,
-implemented as specified in <<oltp-implementations,OLTP Implementations>>. A summary of each required interface
-implementation is presented below:
-
-. `GraphComputer`: A fluent builder for specifying an isolation level, a VertexProgram, and any number of MapReduce jobs to be submitted.
-. `Memory`: A global blackboard for ANDing, ORing, INCRing, and SETing values for specified keys.
-. `Messenger`: The system that collects and distributes messages being propagated by vertices executing the VertexProgram application.
-. `MapReduce.MapEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications map-phase.
-. `MapReduce.ReduceEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications combine- and reduce-phases.
-
-NOTE: The VertexProgram and MapReduce interfaces in the `process/computer/` package are not required by the graph
-system. Instead, these are interfaces to be implemented by application developers writing VertexPrograms and MapReduce jobs.
-
-IMPORTANT: TinkerPop3 provides three OLAP implementations: <<tinkergraph-gremlin,TinkerGraphComputer>> (TinkerGraph),
-<<giraphgraphcomputer,GiraphGraphComputer>> (HadoopGraph), and <<sparkgraphcomputer,`SparkGraphComputer`>> (Hadoop).
-Given the complexity of the OLAP system, it is good to study and copy many of the patterns used in these reference
-implementations.
-
-Implementing GraphComputer
-++++++++++++++++++++++++++
-
-image:furnace-character-3.png[width=150,float=right] The most complex method in GraphComputer is the `submit()`-method. The method must do the following:
-
-. Ensure the the GraphComputer has not already been executed.
-. Ensure that at least there is a VertexProgram or 1 MapReduce job.
-. If there is a VertexProgram, validate that it can execute on the GraphComputer given the respectively defined features.
-. Create the Memory to be used for the computation.
-. Execute the VertexProgram.setup() method once and only once.
-. Execute the VertexProgram.execute() method for each vertex.
-. Execute the VertexProgram.terminate() method once and if true, repeat VertexProgram.execute().
-. When VertexProgram.terminate() returns true, move to MapReduce job execution.
-. MapReduce jobs are not required to be executed in any specified order.
-. For each Vertex, execute MapReduce.map(). Then (if defined) execute MapReduce.combine() and MapReduce.reduce().
-. Update Memory with runtime information.
-. Construct a new `ComputerResult` containing the compute Graph and Memory.
-
-Implementing Memory
-+++++++++++++++++++
-
-image:gremlin-brain.png[width=175,float=left] The Memory object is initially defined by `VertexProgram.setup()`.
-The memory data is available in the first round of the `VertexProgram.execute()` method. Each Vertex, when executing
-the VertexProgram, can update the Memory in its round. However, the update is not seen by the other vertices until
-the next round. At the end of the first round, all the updates are aggregated and the new memory data is available
-on the second round. This process repeats until the VertexProgram terminates.
-
-Implementing Messenger
-++++++++++++++++++++++
-
-The Messenger object is similar to the Memory object in that a vertex can read and write to the Messenger. However,
-the data it reads are the messages sent to the vertex in the previous step and the data it writes are the messages
-that will be readable by the receiving vertices in the subsequent round.
-
-Implementing MapReduce Emitters
-+++++++++++++++++++++++++++++++
-
-image:hadoop-logo-notext.png[width=150,float=left] The MapReduce framework in TinkerPop3 is similar to the model
-popularized by link:http://apache.hadoop.org[Hadoop]. The primary difference is that all Mappers process the vertices
-of the graph, not an arbitrary key/value pair. However, the vertices' edges can not be accessed -- only their
-properties. This greatly reduces the amount of data needed to be pushed through the MapReduce engine as any edge
-information required, can be computed in the VertexProgram.execute() method. Moreover, at this stage, vertices can
-not be mutated, only their token and property data read. A Gremlin OLAP system needs to provide implementations for
-to particular classes: `MapReduce.MapEmitter` and `MapReduce.ReduceEmitter`. TinkerGraph's implementation is provided
-below which demonstrates the simplicity of the algorithm (especially when the data is all within the same JVM).
-
-[source,java]
-----
-public class TinkerMapEmitter<K, V> implements MapReduce.MapEmitter<K, V> {
-
- public Map<K, Queue<V>> reduceMap;
- public Queue<KeyValue<K, V>> mapQueue;
- private final boolean doReduce;
-
- public TinkerMapEmitter(final boolean doReduce) { <1>
- this.doReduce = doReduce;
- if (this.doReduce)
- this.reduceMap = new ConcurrentHashMap<>();
- else
- this.mapQueue = new ConcurrentLinkedQueue<>();
- }
-
- @Override
- public void emit(K key, V value) {
- if (this.doReduce)
- this.reduceMap.computeIfAbsent(key, k -> new ConcurrentLinkedQueue<>()).add(value); <2>
- else
- this.mapQueue.add(new KeyValue<>(key, value)); <3>
- }
-
- protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) {
- if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { <4>
- final Comparator<K> comparator = mapReduce.getMapKeySort().get();
- final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue);
- Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
- this.mapQueue.clear();
- this.mapQueue.addAll(list);
- } else if (mapReduce.getMapKeySort().isPresent()) {
- final Comparator<K> comparator = mapReduce.getMapKeySort().get();
- final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>();
- list.addAll(this.reduceMap.entrySet());
- Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator));
- this.reduceMap = new LinkedHashMap<>();
- list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue()));
- }
- }
-}
-----
-
-<1> If the MapReduce job has a reduce, then use one data structure (`reduceMap`), else use another (`mapList`). The
-difference being that a reduction requires a grouping by key and therefore, the `Map<K,Queue<V>>` definition. If no
-reduction/grouping is required, then a simple `Queue<KeyValue<K,V>>` can be leveraged.
-<2> If reduce is to follow, then increment the Map with a new value for the key. `MapHelper` is a TinkerPop3 class
-with static methods for adding data to a Map.
-<3> If no reduce is to follow, then simply append a KeyValue to the queue.
-<4> When the map phase is complete, any map-result sorting required can be executed at this point.
-
-[source,java]
-----
-public class TinkerReduceEmitter<OK, OV> implements MapReduce.ReduceEmitter<OK, OV> {
-
- protected Queue<KeyValue<OK, OV>> reduceQueue = new ConcurrentLinkedQueue<>();
-
- @Override
- public void emit(final OK key, final OV value) {
- this.reduceQueue.add(new KeyValue<>(key, value));
- }
-
- protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) {
- if (mapReduce.getReduceKeySort().isPresent()) {
- final Comparator<OK> comparator = mapReduce.getReduceKeySort().get();
- final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue);
- Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
- this.reduceQueue.clear();
- this.reduceQueue.addAll(list);
- }
- }
-}
-----
-
-The method `MapReduce.reduce()` is defined as:
-
-[source,java]
-public void reduce(final OK key, final Iterator<OV> values, final ReduceEmitter<OK, OV> emitter) { ... }
-
-In other words, for the TinkerGraph implementation, iterate through the entrySet of the `reduceMap` and call the
-`reduce()` method on each entry. The `reduce()` method can emit key/value pairs which are simply aggregated into a
-`Queue<KeyValue<OK,OV>>` in an analogous fashion to `TinkerMapEmitter` when no reduce is to follow. These two emitters
-are tied together in `TinkerGraphComputer.submit()`.
-
-[source,java]
-----
-...
-for (final MapReduce mapReduce : mapReducers) {
- if (mapReduce.doStage(MapReduce.Stage.MAP)) {
- final TinkerMapEmitter<?, ?> mapEmitter = new TinkerMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
- final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(this.graph.vertices());
- workers.setMapReduce(mapReduce);
- workers.mapReduceWorkerStart(MapReduce.Stage.MAP);
- workers.executeMapReduce(workerMapReduce -> {
- while (true) {
- final Vertex vertex = vertices.next();
- if (null == vertex) return;
- workerMapReduce.map(ComputerGraph.mapReduce(vertex), mapEmitter);
- }
- });
- workers.mapReduceWorkerEnd(MapReduce.Stage.MAP);
-
- // sort results if a map output sort is defined
- mapEmitter.complete(mapReduce);
-
- // no need to run combiners as this is single machine
- if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
- final TinkerReduceEmitter<?, ?> reduceEmitter = new TinkerReduceEmitter<>();
- final SynchronizedIterator<Map.Entry<?, Queue<?>>> keyValues = new SynchronizedIterator((Iterator) mapEmitter.reduceMap.entrySet().iterator());
- workers.mapReduceWorkerStart(MapReduce.Stage.REDUCE);
- workers.executeMapReduce(workerMapReduce -> {
- while (true) {
- final Map.Entry<?, Queue<?>> entry = keyValues.next();
- if (null == entry) return;
- workerMapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter);
- }
- });
- workers.mapReduceWorkerEnd(MapReduce.Stage.REDUCE);
- reduceEmitter.complete(mapReduce); // sort results if a reduce output sort is defined
- mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator()); <1>
- } else {
- mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator()); <2>
- }
- }
-}
-...
-----
-
-<1> Note that the final results of the reducer are provided to the Memory as specified by the application developer's
-`MapReduce.addResultToMemory()` implementation.
-<2> If there is no reduce stage, the the map-stage results are inserted into Memory as specified by the application
-developer's `MapReduce.addResultToMemory()` implementation.
-
-[[io-implementations]]
-IO Implementations
-^^^^^^^^^^^^^^^^^^
-
-If a `Graph` requires custom serializers for IO to work properly, implement the `Graph.io` method. A typical example
-of where a `Graph` would require such a custom serializers is if their identifier system uses non-primitive values,
-such as OrientDB's `Rid` class. From basic serialization of a single `Vertex` all the way up the stack to Gremlin
-Server, the need to know how to handle these complex identifiers is an important requirement.
-
-The first step to implementing custom serializers is to first implement the `IoRegistry` interface and register the
-custom classes and serializers to it. Each `Io` implementation has different requirements for what it expects from the
-`IoRegistry`:
-
-* *GraphML* - No custom serializers expected/allowed.
-* *GraphSON* - Register a Jackson `SimpleModule`. The `SimpleModule` encapsulates specific classes to be serialized,
-so it does not need to be registered to a specific class in the `IoRegistry` (use `null`).
-* *Gryo* - Expects registration of one of three objects:
-** Register just the custom class with a `null` Kryo `Serializer` implementation - this class will use default "field-level" Kryo serialization.
-** Register the custom class with a specific Kryo `Serializer' implementation.
-** Register the custom class with a `Function<Kryo, Serializer>` for those cases where the Kryo `Serializer` requires the `Kryo` instance to get constructed.
-
-This implementation should provide a zero-arg constructor as the stack may require instantiation via reflection.
-Consider extending `AbstractIoRegistry` for convenience as follows:
-
-[source,java]
-----
-public class MyGraphIoRegistry extends AbstractIoRegistry {
- public MyGraphIoRegistry() {
- register(GraphSONIo.class, null, new MyGraphSimpleModule());
- register(GryoIo.class, MyGraphIdClass.class, new MyGraphIdSerializer());
- }
-}
-----
-
-In the `Graph.io` method, provide the `IoRegistry` object to the supplied `Builder` and call the `create` method to
-return that `Io` instance as follows:
-
-[source,java]
-----
-public <I extends Io> I io(final Io.Builder<I> builder) {
- return (I) builder.graph(this).registry(myGraphIoRegistry).create();
-}}
-----
-
-In this way, `Graph` implementations can pre-configure custom serializers for IO interactions and users will not need
-to know about those details. Following this pattern will ensure proper execution of the test suite as well as
-simplified usage for end-users.
-
-IMPORTANT: Proper implementation of IO is critical to successful `Graph` operations in Gremlin Server. The Test Suite
-does have "serialization" tests that provide some assurance that an implementation is working properly, but those
-tests cannot make assertions against any specifics of a custom serializer. It is the responsibility of the
-implementer to test the specifics of their custom serializers.
-
-TIP: Consider separating serializer code into its own module, if possible, so that clients that use the `Graph`
-implementation remotely don't need a full dependency on the entire `Graph` - just the IO components and related
-classes being serialized.
-
-[[validating-with-gremlin-test]]
-Validating with Gremlin-Test
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-image:gremlin-edumacated.png[width=225]
-
-[source,xml]
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-test</artifactId>
- <version>x.y.z</version>
-</dependency>
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-groovy-test</artifactId>
- <version>x.y.z</version>
-</dependency>
-
-The operational semantics of any OLTP or OLAP implementation are validated by `gremlin-test` and functional
-interoperability with the Groovy environment is ensured by `gremlin-groovy-test`. To implement these tests, provide
-test case implementations as shown below, where `XXX` below denotes the name of the graph implementation (e.g.
-TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
-
-[source,java]
-----
-// Structure API tests
-@RunWith(StructureStandardSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
-public class XXXStructureStandardTest {}
-
-// Process API tests
-@RunWith(ProcessComputerSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
-public class XXXProcessComputerTest {}
-
-@RunWith(ProcessStandardSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
-public class XXXProcessStandardTest {}
-
-@RunWith(GroovyEnvironmentSuite.class)
-@GraphProviderClass(provider = XXXProvider.class, graph = TinkerGraph.class)
-public class XXXGroovyEnvironmentTest {}
-
-@RunWith(GroovyProcessStandardSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = TinkerGraph.class)
-public class XXXGroovyProcessStandardTest {}
-
-@RunWith(GroovyProcessComputerSuite.class)
-@GraphProviderClass(provider = XXXGraphComputerProvider.class, graph = TinkerGraph.class)
-public class XXXGroovyProcessComputerTest {}
-----
-
-The above set of tests represent the minimum test suite set to implement. There are other "integration" and
-"performance" tests that should be considered optional. Implementing those tests requires the same pattern as shown above.
-
-IMPORTANT: It is as important to look at "ignored" tests as it is to look at ones that fail. The `gremlin-test`
-suite utilizes the `Feature` implementation exposed by the `Graph` to determine which tests to execute. If a test
-utilizes features that are not supported by the graph, it will ignore them. While that may be fine, implementers
-should validate that the ignored tests are appropriately bypassed and that there are no mistakes in their feature
-definitions. Moreover, implementers should consider filling gaps in their own test suites, especially when
-IO-related tests are being ignored.
-
-The only test-class that requires any code investment is the `GraphProvider` implementation class. This class is a
-used by the test suite to construct `Graph` configurations and instances and provides information about the
-implementation itself. In most cases, it is best to simply extend `AbstractGraphProvider` as it provides many
-default implementations of the `GraphProvider` interface.
-
-Finally, specify the test suites that will be supported by the `Graph` implementation using the `@Graph.OptIn`
-annotation. See the `TinkerGraph` implementation below as an example:
-
-[source,java]
-----
-@Graph.OptIn(Graph.OptIn.SUITE_STRUCTURE_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
-@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_COMPUTER)
-@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT)
-public class TinkerGraph implements Graph {
-----
-
-Only include annotations for the suites the implementation will support. Note that implementing the suite, but
-not specifying the appropriate annotation will prevent the suite from running (an obvious error message will appear
-in this case when running the mis-configured suite).
-
-There are times when there may be a specific test in the suite that the implementation cannot support (despite the
-features it implements) or should not otherwise be executed. It is possible for implementers to "opt-out" of a test
-by using the `@Graph.OptOut` annotation. The following is an example of this annotation usage as taken from
-`HadoopGraph`:
-
-[source, java]
-----
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
- method = "g_V_matchXa_hasXname_GarciaX__a_inXwrittenByX_b__a_inXsungByX_bX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
- method = "g_V_matchXa_inXsungByX_b__a_inXsungByX_c__b_outXwrittenByX_d__c_outXwrittenByX_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldNotAllowBadMemoryKeys",
- reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldRequireRegisteringMemoryKeys",
- reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-public class HadoopGraph implements Graph {
-----
-
-The above examples show how to ignore individual tests. It is also possible to:
-
-* Ignore an entire test case (i.e. all the methods within the test) by setting the `method` to "*".
-* Ignore a "base" test class such that test that extend from those classes will all be ignored. This style of
-ignoring is useful for Gremlin "process" tests that have bases classes that are extended by various Gremlin flavors (e.g. groovy).
-* Ignore a `GraphComputer` test based on the type of `GraphComputer` being used. Specify the "computer" attribute on
-the `OptOut` (which is an array specification) which should have a value of the `GraphComputer` implementation class
-that should ignore that test. This attribute should be left empty for "standard" execution and by default all
-`GraphComputer` implementations will be included in the `OptOut` so if there are multiple implementations, explicitly
-specify the ones that should be excluded.
-
-Also note that some of the tests in the Gremlin Test Suite are parameterized tests and require an additional level of
-specificity to be properly ignored. To ignore these types of tests, examine the name template of the parameterized
-tests. It is defined by a Java annotation that looks like this:
-
-[source, java]
-@Parameterized.Parameters(name = "expect({0})")
-
-The annotation above shows that the name of each parameterized test will be prefixed with "expect" and have
-parentheses wrapped around the first parameter (at index 0) value supplied to each test. This information can
-only be garnered by studying the test set up itself. Once the pattern is determined and the specific unique name of
-the parameterized test is identified, add it to the `specific` property on the `OptOut` annotation in addition to
-the other arguments.
-
-These annotations help provide users a level of transparency into test suite compliance (via the
-xref:describe-graph[describeGraph()] utility function). It also allows implementers to have a lot of flexibility in
-terms of how they wish to support TinkerPop. For example, maybe there is a single test case that prevents an
-implementer from claiming support of a `Feature`. The implementer could choose to either not support the `Feature`
-or to support it but "opt-out" of the test with a "reason" as to why so that users understand the limitation.
-
-IMPORTANT: Before using `OptOut` be sure that the reason for using it is sound and it is more of a last resort.
-It is possible that a test from the suite doesn't properly represent the expectations of a feature, is too broad or
-narrow for the semantics it is trying to enforce or simply contains a bug. Please consider raising issues in the
-developer mailing list with such concerns before assuming `OptOut` is the only answer.
-
-IMPORTANT: There are no tests that specifically validate complete compliance with Gremlin Server. Generally speaking,
-a `Graph` that passes the full Test Suite, should be compliant with Gremlin Server. The one area where problems can
-occur is in serialization. Always ensure that IO is properly implemented, that custom serializers are tested fully
-and ultimately integration test the `Graph` with an actual Gremlin Server instance.
-
-CAUTION: Configuring tests to run in parallel might result in errors that are difficult to debug as there is some
-shared state in test execution around graph configuration. It is therefore recommended that parallelism be turned
-off for the test suite (the Maven SureFire Plugin is configured this way by default). It may also be important to
-include this setting, `<reuseForks>false</reuseForks>`, in the SureFire configuration if tests are failing in an
-unexplainable way.
-
-Accessibility via GremlinPlugin
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-image:gremlin-plugin.png[width=100,float=left] The applications distributed with TinkerPop3 do not distribute with
-any graph system implementations besides TinkerGraph. If your implementation is stored in a Maven repository (e.g.
-Maven Central Repository), then it is best to provide a `GremlinPlugin` implementation so the respective jars can be
-downloaded according and when required by the user. Neo4j's GremlinPlugin is provided below for reference.
-
-[source,java]
-----
-public class Neo4jGremlinPlugin implements GremlinPlugin {
-
- private static final String IMPORT = "import ";
- private static final String DOT_STAR = ".*";
-
- private static final Set<String> IMPORTS = new HashSet<String>() {{
- add(IMPORT + Neo4jGraph.class.getPackage().getName() + DOT_STAR);
- }};
-
- @Override
- public String getName() {
- return "neo4j";
- }
-
- @Override
- public void pluginTo(final PluginAcceptor pluginAcceptor) {
- pluginAcceptor.addImports(IMPORTS);
- }
-}
-----
-
-With the above plugin implementations, users can now download respective binaries for Gremlin Console, Gremlin Server, etc.
-
-[source,groovy]
-gremlin> g = Neo4jGraph.open('/tmp/neo4j')
-No such property: Neo4jGraph for class: groovysh_evaluate
-Display stack trace? [yN]
-gremlin> :install org.apache.tinkerpop neo4j-gremlin x.y.z
-==>loaded: [org.apache.tinkerpop, neo4j-gremlin, …]
-gremlin> :plugin use tinkerpop.neo4j
-==>tinkerpop.neo4j activated
-gremlin> g = Neo4jGraph.open('/tmp/neo4j')
-==>neo4jgraph[EmbeddedGraphDatabase [/tmp/neo4j]]
-
-In-Depth Implementations
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-image:gremlin-painting.png[width=200,float=right] The graph system implementation details presented thus far are
-minimum requirements necessary to yield a valid TinkerPop3 implementation. However, there are other areas that a
-graph system provider can tweak to provide an implementation more optimized for their underlying graph engine. Typical
-areas of focus include:
-
-* Traversal Strategies: A <<traversalstrategy,TraversalStrategy>> can be used to alter a traversal prior to its
-execution. A typical example is converting a pattern of `g.V().has('name','marko')` into a global index lookup for
-all vertices with name "marko". In this way, a `O(|V|)` lookup becomes an `O(log(|V|))`. Please review
-`TinkerGraphStepStrategy` for ideas.
-* Step Implementations: Every <<graph-traversal-steps,step>> is ultimately referenced by the `GraphTraversal`
-interface. It is possible to extend `GraphTraversal` to use a graph system specific step implementation.
-
-
-[[tinkergraph-gremlin]]
-TinkerGraph-Gremlin
--------------------
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>tinkergraph-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-----
-
-image:tinkerpop-character.png[width=100,float=left] TinkerGraph is a single machine, in-memory (with optional
-persistence), non-transactional graph engine that provides both OLTP and OLAP functionality. It is deployed with
-TinkerPop3 and serves as the reference implementation for other providers to study in order to understand the
-semantics of the various methods of the TinkerPop3 API. Constructing a simple graph in Java8 is presented below.
-
-[source,java]
-Graph g = TinkerGraph.open();
-Vertex marko = g.addVertex("name","marko","age",29);
-Vertex lop = g.addVertex("name","lop","lang","java");
-marko.addEdge("created",lop,"weight",0.6d);
-
-The above graph creates two vertices named "marko" and "lop" and connects them via a created-edge with a weight=0.6
-property. Next, the graph can be queried as such.
-
-[source,java]
-g.V().has("name","marko").out("created").values("name")
-
-The `g.V().has("name","marko")` part of the query can be executed in two ways.
-
- * A linear scan of all vertices filtering out those vertices that don't have the name "marko"
- * A `O(log(|V|))` index lookup for all vertices with the name "marko"
-
-Given the initial graph construction in the first code block, no index was defined and thus, a linear scan is executed.
-However, if the graph was constructed as such, then an index lookup would be used.
-
-[source,java]
-Graph g = TinkerGraph.open();
-g.createIndex("name",Vertex.class)
-
-The execution times for a vertex lookup by property is provided below for both no-index and indexed version of
-TinkerGraph over the Grateful Dead graph.
-
-[gremlin-groovy]
-----
-graph = TinkerGraph.open()
-g = graph.traversal()
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
-clock(1000) {g.V().has('name','Garcia').iterate()} <1>
-graph = TinkerGraph.open()
-g = graph.traversal()
-graph.createIndex('name',Vertex.class)
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
-clock(1000){g.V().has('name','Garcia').iterate()} <2>
-----
-
-<1> Determine the average runtime of 1000 vertex lookups when no `name`-index is defined.
-<2> Determine the average runtime of 1000 vertex lookups when a `name`-index is defined.
-
-IMPORTANT: Each graph system will have different mechanism by which indices and schemas are defined. TinkerPop3
-does not require any conformance in this area. In TinkerGraph, the only definitions are around indices. With other
-graph systems, property value types, indices, edge labels, etc. may be required to be defined _a priori_ to adding
-data to the graph.
-
-NOTE: TinkerGraph is distributed with Gremlin Server and is therefore automatically available to it for configuration.
-
-Configuration
-~~~~~~~~~~~~~
-
-TinkerGraph has several settings that can be provided on creation via `Configuration` object:
-
-[width="100%",cols="2,10",options="header"]
-|=========================================================
-|Property |Description
-|gremlin.graph |`org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph`
-|gremlin.tinkergraph.vertexIdManager |The `IdManager` implementation to use for vertices.
-|gremlin.tinkergraph.edgeIdManager |The `IdManager` implementation to use for edges.
-|gremlin.tinkergraph.vertexPropertyIdManager |The `IdManager` implementation to use for vertex properties.
-|gremlin.tinkergraph.defaultVertexPropertyCardinality |The default `VertexProperty.Cardinality` to use when `Vertex.property(k,v)` is called.
-|gremlin.tinkergraph.graphLocation |The path and file name for where TinkerGraph should persist the graph data. If a
-value is specified here, the the `gremlin.tinkergraph.graphFormat` should also be specified. If this value is not
-included (default), then the graph will stay in-memory and not be loaded/persisted to disk.
-|gremlin.tinkergraph.graphFormat |The format to use to serialize the graph which may be one of the following:
-`graphml`, `graphson`, `gryo`, or a fully qualified class name that implements Io.Builder interface (which allows for
-external third party graph reader/writer formats to be used for persistence).
-If a value is specified here, then the `gremlin.tinkergraph.graphLocation` should
-also be specified. If this value is not included (default), then the graph will stay in-memory and not be
-loaded/persisted to disk.
-|=========================================================
-
-The `IdManager` settings above refer to how TinkerGraph will control identifiers for vertices, edges and vertex
-properties. There are several options for each of these settings: `ANY`, `LONG`, `INTEGER`, `UUID`, or the fully
-qualified class name of an `IdManager` implementation on the classpath. When not specified, the default values
-for all settings is `ANY`, meaning that the graph will work with any object on the JVM as the identifier and will
-generate new identifiers from `Long` when the identifier is not user supplied. TinkerGraph will also expect the
-user to understand the types used for identifiers when querying, meaning that `g.V(1)` and `g.V(1L)` could return
-two different vertices. `LONG`, `INTEGER` and `UUID` settings will try to coerce identifier values to the expected
-type as well as generate new identifiers with that specified type.
-
-If the TinkerGraph is configured for persistence with `gremlin.tinkergraph.graphLocation` and
-`gremlin.tinkergraph.graphFormat`, then the graph will be written to the specified location with the specified
-format when `Graph.close()` is called. In addition, if these settings are present, TinkerGraph will attempt to
-load the graph from the specified location.
-
-IMPORTANT: If choosing `graphson` as the `gremlin.tinkergraph.graphFormat`, be sure to also establish the various
-`IdManager` settings as well to ensure that identifiers are properly coerced to the appropriate types as GraphSON
-can lose the identifier's type during serialization (i.e. it will assume `Integer` when the default for TinkerGraph
-is `Long`, which could lead to load errors that result in a message like, "Vertex with id already exists").
-
-It is important to consider the data being imported to TinkerGraph with respect to `defaultVertexPropertyCardinality`
-setting. For example, if a `.gryo` file is known to contain multi-property data, be sure to set the default
-cardinality to `list` or else the data will import as `single`. Consider the following:
-
-[gremlin-groovy]
-----
-graph = TinkerGraph.open()
-graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
-g = graph.traversal()
-g.V().properties()
-conf = new BaseConfiguration()
-conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality","list")
-graph = TinkerGraph.open(conf)
-graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
-g = graph.traversal()
-g.V().properties()
-----
-
[[neo4j-gremlin]]
Neo4j-Gremlin
-------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c9149c23/docs/src/reference/implementations-tinkergraph.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
new file mode 100644
index 0000000..aadec06
--- /dev/null
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -0,0 +1,144 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+[[tinkergraph-gremlin]]
+TinkerGraph-Gremlin
+-------------------
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkergraph-gremlin</artifactId>
+ <version>x.y.z</version>
+</dependency>
+----
+
+image:tinkerpop-character.png[width=100,float=left] TinkerGraph is a single machine, in-memory (with optional
+persistence), non-transactional graph engine that provides both OLTP and OLAP functionality. It is deployed with
+TinkerPop3 and serves as the reference implementation for other providers to study in order to understand the
+semantics of the various methods of the TinkerPop3 API. Constructing a simple graph in Java8 is presented below.
+
+[source,java]
+Graph g = TinkerGraph.open();
+Vertex marko = g.addVertex("name","marko","age",29);
+Vertex lop = g.addVertex("name","lop","lang","java");
+marko.addEdge("created",lop,"weight",0.6d);
+
+The above graph creates two vertices named "marko" and "lop" and connects them via a created-edge with a weight=0.6
+property. Next, the graph can be queried as such.
+
+[source,java]
+g.V().has("name","marko").out("created").values("name")
+
+The `g.V().has("name","marko")` part of the query can be executed in two ways.
+
+ * A linear scan of all vertices filtering out those vertices that don't have the name "marko"
+ * A `O(log(|V|))` index lookup for all vertices with the name "marko"
+
+Given the initial graph construction in the first code block, no index was defined and thus, a linear scan is executed.
+However, if the graph was constructed as such, then an index lookup would be used.
+
+[source,java]
+Graph g = TinkerGraph.open();
+g.createIndex("name",Vertex.class)
+
+The execution times for a vertex lookup by property is provided below for both no-index and indexed version of
+TinkerGraph over the Grateful Dead graph.
+
+[gremlin-groovy]
+----
+graph = TinkerGraph.open()
+g = graph.traversal()
+graph.io(graphml()).readGraph('data/grateful-dead.xml')
+clock(1000) {g.V().has('name','Garcia').iterate()} <1>
+graph = TinkerGraph.open()
+g = graph.traversal()
+graph.createIndex('name',Vertex.class)
+graph.io(graphml()).readGraph('data/grateful-dead.xml')
+clock(1000){g.V().has('name','Garcia').iterate()} <2>
+----
+
+<1> Determine the average runtime of 1000 vertex lookups when no `name`-index is defined.
+<2> Determine the average runtime of 1000 vertex lookups when a `name`-index is defined.
+
+IMPORTANT: Each graph system will have different mechanism by which indices and schemas are defined. TinkerPop3
+does not require any conformance in this area. In TinkerGraph, the only definitions are around indices. With other
+graph systems, property value types, indices, edge labels, etc. may be required to be defined _a priori_ to adding
+data to the graph.
+
+NOTE: TinkerGraph is distributed with Gremlin Server and is therefore automatically available to it for configuration.
+
+Configuration
+~~~~~~~~~~~~~
+
+TinkerGraph has several settings that can be provided on creation via `Configuration` object:
+
+[width="100%",cols="2,10",options="header"]
+|=========================================================
+|Property |Description
+|gremlin.graph |`org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph`
+|gremlin.tinkergraph.vertexIdManager |The `IdManager` implementation to use for vertices.
+|gremlin.tinkergraph.edgeIdManager |The `IdManager` implementation to use for edges.
+|gremlin.tinkergraph.vertexPropertyIdManager |The `IdManager` implementation to use for vertex properties.
+|gremlin.tinkergraph.defaultVertexPropertyCardinality |The default `VertexProperty.Cardinality` to use when `Vertex.property(k,v)` is called.
+|gremlin.tinkergraph.graphLocation |The path and file name for where TinkerGraph should persist the graph data. If a
+value is specified here, the the `gremlin.tinkergraph.graphFormat` should also be specified. If this value is not
+included (default), then the graph will stay in-memory and not be loaded/persisted to disk.
+|gremlin.tinkergraph.graphFormat |The format to use to serialize the graph which may be one of the following:
+`graphml`, `graphson`, `gryo`, or a fully qualified class name that implements Io.Builder interface (which allows for
+external third party graph reader/writer formats to be used for persistence).
+If a value is specified here, then the `gremlin.tinkergraph.graphLocation` should
+also be specified. If this value is not included (default), then the graph will stay in-memory and not be
+loaded/persisted to disk.
+|=========================================================
+
+The `IdManager` settings above refer to how TinkerGraph will control identifiers for vertices, edges and vertex
+properties. There are several options for each of these settings: `ANY`, `LONG`, `INTEGER`, `UUID`, or the fully
+qualified class name of an `IdManager` implementation on the classpath. When not specified, the default values
+for all settings is `ANY`, meaning that the graph will work with any object on the JVM as the identifier and will
+generate new identifiers from `Long` when the identifier is not user supplied. TinkerGraph will also expect the
+user to understand the types used for identifiers when querying, meaning that `g.V(1)` and `g.V(1L)` could return
+two different vertices. `LONG`, `INTEGER` and `UUID` settings will try to coerce identifier values to the expected
+type as well as generate new identifiers with that specified type.
+
+If the TinkerGraph is configured for persistence with `gremlin.tinkergraph.graphLocation` and
+`gremlin.tinkergraph.graphFormat`, then the graph will be written to the specified location with the specified
+format when `Graph.close()` is called. In addition, if these settings are present, TinkerGraph will attempt to
+load the graph from the specified location.
+
+IMPORTANT: If choosing `graphson` as the `gremlin.tinkergraph.graphFormat`, be sure to also establish the various
+`IdManager` settings as well to ensure that identifiers are properly coerced to the appropriate types as GraphSON
+can lose the identifier's type during serialization (i.e. it will assume `Integer` when the default for TinkerGraph
+is `Long`, which could lead to load errors that result in a message like, "Vertex with id already exists").
+
+It is important to consider the data being imported to TinkerGraph with respect to `defaultVertexPropertyCardinality`
+setting. For example, if a `.gryo` file is known to contain multi-property data, be sure to set the default
+cardinality to `list` or else the data will import as `single`. Consider the following:
+
+[gremlin-groovy]
+----
+graph = TinkerGraph.open()
+graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
+g = graph.traversal()
+g.V().properties()
+conf = new BaseConfiguration()
+conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality","list")
+graph = TinkerGraph.open(conf)
+graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
+g = graph.traversal()
+g.V().properties()
+----
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c9149c23/docs/src/reference/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/index.asciidoc b/docs/src/reference/index.asciidoc
index 8546063..0006403 100644
--- a/docs/src/reference/index.asciidoc
+++ b/docs/src/reference/index.asciidoc
@@ -32,6 +32,8 @@ include::the-graphcomputer.asciidoc[]
include::gremlin-applications.asciidoc[]
+include::implementations-intro.asciidoc[]
+include::implementations-tinkergraph.asciidoc[]
include::implementations-neo4j.asciidoc[]
include::implementations-hadoop.asciidoc[]
[17/43] incubator-tinkerpop git commit: Enabled build on windows
Posted by sp...@apache.org.
Enabled build on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c193d538
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c193d538
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c193d538
Branch: refs/heads/TINKERPOP-1107
Commit: c193d538fd8ac9d7b70ca2d64e151a076f6e5138
Parents: 10c978d
Author: Marvin Froeder <ve...@gmail.com>
Authored: Thu Jan 21 13:33:56 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
appveyor.yml | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c193d538/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 0000000..92eae9f
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,23 @@
+version: '{build}'
+
+environment:
+ matrix:
+ - JAVA_HOME: C:\Program Files\Java\jdk1.8.0
+
+os: Windows Server 2012
+
+install:
+ - ps: |
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
+ if (!(Test-Path -Path "C:\maven" )) {
+ (new-object System.Net.WebClient).DownloadFile('http://www.us.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.zip', 'C:\maven-bin.zip')
+ [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven")
+ }
+ - cmd: SET PATH=C:\maven\apache-maven-3.2.5\bin;%JAVA_HOME%\bin;%PATH%
+ - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g
+ - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g
+ - cmd: mvn --version
+ - cmd: java -version
+
+build_script:
+ - mvn clean install -B -Dci
[32/43] incubator-tinkerpop git commit: Moved assertion and increased
timeout for test.
Posted by sp...@apache.org.
Moved assertion and increased timeout for test.
Assertion needed to come after the executor was shutdown and the test seemingly needs more time on appveryor. CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c5447c29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c5447c29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c5447c29
Branch: refs/heads/TINKERPOP-1107
Commit: c5447c29cd6abceef4168ebd3ccc65d7ea8e27a2
Parents: 13f8005
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Feb 24 07:44:37 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Feb 24 07:44:37 2016 -0500
----------------------------------------------------------------------
.../gremlin/groovy/engine/GremlinExecutorTest.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c5447c29/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
index 251d8cf..aad1f77 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
@@ -631,11 +631,12 @@ public class GremlinExecutorTest {
}
});
+ service.shutdown();
+ assertThat(service.awaitTermination(60000, TimeUnit.MILLISECONDS), is(true));
+
// likely a concurrency exception if it occurs - and if it does then we've messed up because that's what this
// test is partially designed to protected against.
assertThat(failed.get(), is(false));
- service.shutdown();
- assertThat(service.awaitTermination(30000, TimeUnit.MILLISECONDS), is(true));
assertEquals(max, futures.size());
futures.forEach(t -> {
@@ -685,11 +686,12 @@ public class GremlinExecutorTest {
}
});
+ service.shutdown();
+ assertThat(service.awaitTermination(60000, TimeUnit.MILLISECONDS), is(true));
+
// likely a concurrency exception if it occurs - and if it does then we've messed up because that's what this
// test is partially designed to protected against.
assertThat(failed.get(), is(false));
- service.shutdown();
- assertThat(service.awaitTermination(30000, TimeUnit.MILLISECONDS), is(true));
assertEquals(max, futures.size());
futures.forEach(t -> {
[26/43] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1041'
of https://github.com/velo/incubator-tinkerpop into tp31
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-1041' of https://github.com/velo/incubator-tinkerpop into tp31
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f8cbe8fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f8cbe8fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f8cbe8fb
Branch: refs/heads/TINKERPOP-1107
Commit: f8cbe8fbbff682dd9e6281329511ba39bc1920ae
Parents: f2972e8 1ca9549
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Feb 22 08:17:01 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Feb 22 08:17:01 2016 -0500
----------------------------------------------------------------------
appveyor.yml | 14 +++++++
.../groovy/engine/ScriptEnginesTest.java | 5 ++-
.../apache/tinkerpop/gremlin/TestHelper.java | 12 +++++-
.../gremlin/hadoop/HadoopGraphProvider.java | 6 +--
pom.xml | 44 ++++++++++++++++++++
spark-gremlin/pom.xml | 21 ++++++++++
.../spark/structure/io/SparkContextStorage.java | 8 ++--
7 files changed, 99 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[22/43] incubator-tinkerpop git commit: Fixed regex construction
Posted by sp...@apache.org.
Fixed regex construction
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/164fdace
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/164fdace
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/164fdace
Branch: refs/heads/TINKERPOP-1107
Commit: 164fdace2e58e9632d8de4bba0fe8bea4c2bd5b2
Parents: 317d5c8
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Wed Feb 3 09:17:11 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:31:56 2016 +1300
----------------------------------------------------------------------
.../gremlin/spark/structure/io/SparkContextStorage.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/164fdace/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
index 4b113a3..799c215 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
@@ -75,7 +75,7 @@ public final class SparkContextStorage implements Storage {
@Override
public List<String> ls(final String location) {
final List<String> rdds = new ArrayList<>();
- final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/');
+ final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace('\\', '/').replace(".", "\\.").replace("*", ".*");
for (final RDD<?> rdd : Spark.getRDDs()) {
if (rdd.name().replace('\\', '/').matches(wildCardLocation))
rdds.add(rdd.name() + " [" + rdd.getStorageLevel().description() + "]");
@@ -103,7 +103,7 @@ public final class SparkContextStorage implements Storage {
@Override
public boolean rm(final String location) {
final List<String> rdds = new ArrayList<>();
- final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/');
+ final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace('\\', '/').replace(".", "\\.").replace("*", ".*");
for (final RDD<?> rdd : Spark.getRDDs()) {
if (rdd.name().replace('\\', '/').matches(wildCardLocation))
rdds.add(rdd.name());
[43/43] incubator-tinkerpop git commit: Added some more asserts
around variable scoping with interpreter mode.
Posted by sp...@apache.org.
Added some more asserts around variable scoping with interpreter mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/73489b3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/73489b3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/73489b3a
Branch: refs/heads/TINKERPOP-1107
Commit: 73489b3a89bab18d59fc5e1dfc9b23a10c5222ab
Parents: 566c488
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Feb 19 10:05:41 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Feb 25 07:40:47 2016 -0500
----------------------------------------------------------------------
.../jsr223/GremlinGroovyScriptEngineTest.java | 25 ++++++++++++++++++++
1 file changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/73489b3a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
index 17cd809..19ced88 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
@@ -19,6 +19,8 @@
package org.apache.tinkerpop.gremlin.groovy.jsr223;
import groovy.lang.Closure;
+import groovy.lang.MissingPropertyException;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.NoImportCustomizerProvider;
@@ -55,6 +57,7 @@ import java.util.stream.IntStream;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -99,6 +102,17 @@ public class GremlinGroovyScriptEngineTest {
assertEquals(4, engine.eval("yyy = xxx + 1"));
assertEquals(7, engine.eval("def zzz = yyy + xxx"));
assertEquals(4, engine.eval("zzz - xxx"));
+ assertEquals("accessible-globally", engine.eval("if (yyy > 0) { def inner = 'should-stay-local'; outer = 'accessible-globally' }\n outer"));
+ assertEquals("accessible-globally", engine.eval("outer"));
+
+ try {
+ engine.eval("inner");
+ fail("Should not have been able to access 'inner'");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(MissingPropertyException.class));
+ }
+
assertEquals(10, engine.eval("addItUp(zzz,xxx)"));
}
@@ -112,6 +126,17 @@ public class GremlinGroovyScriptEngineTest {
assertEquals(4, engine.eval("yyy = xxx + 1", b));
assertEquals(7, engine.eval("def zzz = yyy + xxx", b));
assertEquals(4, engine.eval("zzz - xxx", b));
+ assertEquals("accessible-globally", engine.eval("if (yyy > 0) { def inner = 'should-stay-local'; outer = 'accessible-globally' }\n outer", b));
+ assertEquals("accessible-globally", engine.eval("outer", b));
+
+ try {
+ engine.eval("inner", b);
+ fail("Should not have been able to access 'inner'");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(MissingPropertyException.class));
+ }
+
assertEquals(10, engine.eval("addItUp(zzz,xxx)", b));
}
[41/43] incubator-tinkerpop git commit: Removed some redundant text
in NOTICE - CTR
Posted by sp...@apache.org.
Removed some redundant text in NOTICE - CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d0005b79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d0005b79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d0005b79
Branch: refs/heads/TINKERPOP-1107
Commit: d0005b797b55ab151a2bf683488c86a7cf2e4886
Parents: da22bec
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Feb 25 07:32:50 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Feb 25 07:33:09 2016 -0500
----------------------------------------------------------------------
gremlin-console/src/main/static/NOTICE | 3 ---
gremlin-server/src/main/static/NOTICE | 3 ---
2 files changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d0005b79/gremlin-console/src/main/static/NOTICE
----------------------------------------------------------------------
diff --git a/gremlin-console/src/main/static/NOTICE b/gremlin-console/src/main/static/NOTICE
index 51d06c8..b9fa6eb 100644
--- a/gremlin-console/src/main/static/NOTICE
+++ b/gremlin-console/src/main/static/NOTICE
@@ -20,9 +20,6 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
------------------------------------------------------------------------
Apache Groovy (AL ASF)
------------------------------------------------------------------------
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
This product includes/uses ANTLR (http://www.antlr2.org/)
developed by Terence Parr 1989-2006
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d0005b79/gremlin-server/src/main/static/NOTICE
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/static/NOTICE b/gremlin-server/src/main/static/NOTICE
index b9b9a2e..a767d39 100644
--- a/gremlin-server/src/main/static/NOTICE
+++ b/gremlin-server/src/main/static/NOTICE
@@ -13,9 +13,6 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
------------------------------------------------------------------------
Apache Groovy (AL ASF)
------------------------------------------------------------------------
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
This product includes/uses ANTLR (http://www.antlr2.org/)
developed by Terence Parr 1989-2006
[24/43] incubator-tinkerpop git commit: Created an appveyor profile
that ignored knowly broken tests
Posted by sp...@apache.org.
Created an appveyor profile that ignored knowly broken tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1ca95490
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1ca95490
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1ca95490
Branch: refs/heads/TINKERPOP-1107
Commit: 1ca95490d42892c4adffc867f8900e5e327e8812
Parents: bc58b8b
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Sat Feb 13 12:17:02 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 11:35:14 2016 +1300
----------------------------------------------------------------------
appveyor.yml | 2 +-
spark-gremlin/pom.xml | 21 +++++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1ca95490/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index a99752a..fe00740 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -11,4 +11,4 @@ install:
- cmd: refreshenv
build_script:
- - mvn clean install -B -Dci
+ - mvn clean install -B -Dci -fae -Pappveyor
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1ca95490/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 7d6c666..fb7b2f7 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -393,4 +393,25 @@
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>appveyor</id>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/SparkHadoopGremlinTest*</exclude>
+ <exclude>**/SparkGraphComputerGroovyProcessIntegrateTest*</exclude>
+ <exclude>**/SparkGraphComputerProcessIntegrateTest*</exclude>
+ <exclude>**/GryoSerializerIntegrateTest*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
\ No newline at end of file
[40/43] incubator-tinkerpop git commit: Bump to groovy 2.4.6.
Posted by sp...@apache.org.
Bump to groovy 2.4.6.
There were no changes to dependencies in the distributions and no new NOTICEs in the groovy jars. CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/da22becb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/da22becb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/da22becb
Branch: refs/heads/TINKERPOP-1107
Commit: da22becb9cdaa67d141eb854da8a538a06201988
Parents: 1d54f00
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Feb 25 07:31:56 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Feb 25 07:31:56 2016 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
pom.xml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/da22becb/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 910d9b6..6a57a60 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.2 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Bumped to Apache Groovy 2.4.6.
* Added the `gremlin-archetype-server` archetype that demonstrates
* Added the `gremlin-archetype-tinkergraph` archetype that demonstrates a basic project that uses TinkerGraph.
* Added `gremlin-archetype` module to house TinkerPop "examples".
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/da22becb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 206df9f..1a4ac02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@ limitations under the License.
<url>https://git-wip-us.apache.org/repos/asf?p=incubator-tinkerpop.git</url>
</scm>
<properties>
- <groovy.version>2.4.5</groovy.version>
+ <groovy.version>2.4.6</groovy.version>
<junit.version>4.12</junit.version>
<metrics.version>3.0.2</metrics.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
[31/43] incubator-tinkerpop git commit: Add exception to logging on
ScriptEngines init.
Posted by sp...@apache.org.
Add exception to logging on ScriptEngines init.
Specifically applies to logging around the application of CompilerCustomizers. CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/13f80056
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/13f80056
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/13f80056
Branch: refs/heads/TINKERPOP-1107
Commit: 13f800563417cd479944e85df963658f3624435a
Parents: cf19449
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Feb 23 13:05:00 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Feb 23 13:05:29 2016 -0500
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/13f80056/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
index 84b0e9b..1113eb5 100644
--- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
@@ -388,7 +388,7 @@ public class ScriptEngines implements AutoCloseable {
providers.add((CompilerCustomizerProvider) providerClass.newInstance());
}
} catch(Exception ex) {
- logger.warn("Could not instantiate CompilerCustomizerProvider implementation [{}]. It will not be applied.", k);
+ logger.warn(String.format("Could not instantiate CompilerCustomizerProvider implementation [%s]. It will not be applied.", k), ex);
}
});
[18/43] incubator-tinkerpop git commit: Ignoring rat on eclipse and
appveyor files
Posted by sp...@apache.org.
Ignoring rat on eclipse and appveyor files
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/92edead8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/92edead8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/92edead8
Branch: refs/heads/TINKERPOP-1107
Commit: 92edead8ba8c34516e4ecc3febcf60a8be8a6024
Parents: c193d53
Author: Marvin H Froeder <ma...@vizexplorer.com>
Authored: Thu Jan 21 13:46:05 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
pom.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/92edead8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8f0bb2b..62eb81c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -264,6 +264,10 @@ limitations under the License.
<excludeSubProjects>false</excludeSubProjects>
<excludes>
<exclude>.travis.yml</exclude>
+ <exclude>appveyor.yml</exclude>
+ <exclude>**/.classpath</exclude>
+ <exclude>**/.project</exclude>
+ <exclude>**/.settings/**</exclude>
<exclude>.repository/**</exclude>
<exclude>docs/static/**</exclude>
<exclude>**/target/**</exclude>
[21/43] incubator-tinkerpop git commit: Attemp to download maven
using https
Posted by sp...@apache.org.
Attemp to download maven using https
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/884e45d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/884e45d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/884e45d1
Branch: refs/heads/TINKERPOP-1107
Commit: 884e45d136d74144aa0d51c4ada3822bfc786ee9
Parents: 164fdac
Author: Marvin Froeder <ve...@gmail.com>
Authored: Thu Feb 4 08:08:12 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:31:56 2016 +1300
----------------------------------------------------------------------
appveyor.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/884e45d1/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index 92eae9f..e31d2ea 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -10,10 +10,10 @@ install:
- ps: |
Add-Type -AssemblyName System.IO.Compression.FileSystem
if (!(Test-Path -Path "C:\maven" )) {
- (new-object System.Net.WebClient).DownloadFile('http://www.us.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.zip', 'C:\maven-bin.zip')
+ (new-object System.Net.WebClient).DownloadFile('https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip', 'C:\maven-bin.zip')
[System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven")
}
- - cmd: SET PATH=C:\maven\apache-maven-3.2.5\bin;%JAVA_HOME%\bin;%PATH%
+ - cmd: SET PATH=C:\maven\apache-maven-3.3.9\bin;%JAVA_HOME%\bin;%PATH%
- cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g
- cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g
- cmd: mvn --version
[27/43] incubator-tinkerpop git commit: Add notes about the build
servers in the contributing guide.
Posted by sp...@apache.org.
Add notes about the build servers in the contributing guide.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6f279954
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6f279954
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6f279954
Branch: refs/heads/TINKERPOP-1107
Commit: 6f279954027856763c4b074b1bbf046c9acdd98c
Parents: f8cbe8f
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Feb 22 08:44:12 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Feb 22 08:44:12 2016 -0500
----------------------------------------------------------------------
docs/src/dev/developer/contributing.asciidoc | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6f279954/docs/src/dev/developer/contributing.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/developer/contributing.asciidoc b/docs/src/dev/developer/contributing.asciidoc
index 88d600c..90687a8 100644
--- a/docs/src/dev/developer/contributing.asciidoc
+++ b/docs/src/dev/developer/contributing.asciidoc
@@ -217,6 +217,19 @@ Contributors should examine the current code base to determine what the code sty
style to what is already present. Of specific note however, TinkerPop does not use "import wildcards" - IDEs should
be adjusted accordingly to not auto-wildcard the imports.
+Build Server
+~~~~~~~~~~~~
+
+TinkerPop uses both link:https://travis-ci.com/[Travis] and link:https://www.appveyor.com/[AppVeyor] for
+link:https://en.wikipedia.org/wiki/Continuous_integration[CI] services. Travis validates builds on Ubuntu, while
+AppVeyor validates builds on Windows. The build statuses can be found here:
+
+* link:https://travis-ci.org/apache/incubator-tinkerpop[Travis Build Status]
+* link:https://ci.appveyor.com/project/ApacheSoftwareFoundation/incubator-tinkerpop[AppVeyor Build Status]
+
+Note that the CI process does not run integration tests or include Neo4j-related tests as those tests would likely
+exceed the allowable times for builds on these servers.
+
Deprecation
~~~~~~~~~~~
[30/43] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1165'
of https://github.com/rjbriody/incubator-tinkerpop into tp31
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-1165' of https://github.com/rjbriody/incubator-tinkerpop into tp31
CTR - http://mail-archives.apache.org/mod_mbox/incubator-tinkerpop-dev/201602.mbox/%3CCAA-H4390Nppe%3DDANvDuvkuJ6pS4p97EuDjb8ySvXDsHXHF%2BiRA%40mail.gmail.com%3E
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/cf194499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/cf194499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/cf194499
Branch: refs/heads/TINKERPOP-1107
Commit: cf194499a7996aa048089a03e45d3e2ffb3e2f38
Parents: 6f27995 5d0ac3e
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Feb 23 10:18:42 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Feb 23 10:18:42 2016 -0500
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cf194499/pom.xml
----------------------------------------------------------------------
[19/43] incubator-tinkerpop git commit: Changed back slashes into
regular ones
Posted by sp...@apache.org.
Changed back slashes into regular ones
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/daa6379b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/daa6379b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/daa6379b
Branch: refs/heads/TINKERPOP-1107
Commit: daa6379bef196acdbdc769c2a516f1750010f4cc
Parents: 9d0c877
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Mon Feb 1 17:19:33 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
.../apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/daa6379b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index 9600dad..4a6f06c 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -80,7 +80,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
"tinkerpop-classic.kryo",
"tinkerpop-crew.kryo");
for (final String fileName : kryoResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
}
final List<String> graphsonResources = Arrays.asList(
@@ -89,7 +89,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
"tinkerpop-classic.json",
"tinkerpop-crew.json");
for (final String fileName : graphsonResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
}
final List<String> scriptResources = Arrays.asList(
@@ -100,7 +100,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
"script-input-grateful-dead.groovy",
"script-output-grateful-dead.groovy");
for (final String fileName : scriptResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
}
} catch (Exception e) {
e.printStackTrace();
[33/43] incubator-tinkerpop git commit: Ensured that
TraversalExplaination works for anonymous traversals (or any traversal that
has no strategies). Added TraversalExplanationTest that verifies correct
behavior. Fixed TINKERPOP-1175
Posted by sp...@apache.org.
Ensured that TraversalExplaination works for anonymous traversals (or any traversal that has no strategies). Added TraversalExplanationTest that verifies correct behavior. Fixed TINKERPOP-1175
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/15909c66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/15909c66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/15909c66
Branch: refs/heads/TINKERPOP-1107
Commit: 15909c669b40d08dde16c225ee818796252d6a79
Parents: c5447c2
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Feb 24 07:30:24 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Feb 24 07:30:24 2016 -0700
----------------------------------------------------------------------
.../traversal/util/TraversalExplanation.java | 4 +--
.../util/TraversalExplanationTest.java | 38 ++++++++++++++++++++
2 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/15909c66/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanation.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanation.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanation.java
index 97c7ab8..786b994 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanation.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanation.java
@@ -84,7 +84,7 @@ public class TraversalExplanation implements Serializable {
public String toString() {
final String originalTraversal = "Original Traversal";
final String finalTraversal = "Final Traversal";
- final int maxStrategyColumnLength = this.strategyTraversals.stream().map(Pair::getValue0).map(Object::toString).map(String::length).max(Comparator.<Integer>naturalOrder()).get();
+ final int maxStrategyColumnLength = this.strategyTraversals.stream().map(Pair::getValue0).map(Object::toString).map(String::length).max(Comparator.<Integer>naturalOrder()).orElse(15);
final int maxTraversalColumnLength = Stream.concat(Stream.of(Pair.with(null, this.traversal)), this.strategyTraversals.stream()).map(Pair::getValue1).map(Object::toString).map(String::length).max(Comparator.<Integer>naturalOrder()).get();
final StringBuilder builder = new StringBuilder("Traversal Explanation\n");
@@ -115,7 +115,7 @@ public class TraversalExplanation implements Serializable {
for (int i = 0; i < maxStrategyColumnLength - finalTraversal.length() + 7; i++) {
builder.append(" ");
}
- builder.append(this.strategyTraversals.get(this.strategyTraversals.size() - 1).getValue1());
+ builder.append(this.strategyTraversals.size() > 0 ? this.strategyTraversals.get(this.strategyTraversals.size() - 1).getValue1() : this.traversal);
return builder.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/15909c66/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanationTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanationTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanationTest.java
new file mode 100644
index 0000000..5b154a0
--- /dev/null
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalExplanationTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.traversal.util;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class TraversalExplanationTest {
+
+ @Test
+ public void shouldSupportAnonymousTraversals() {
+ final String toString = __.out("knows").in("created").explain().toString();
+ assertTrue(toString.contains("Traversal Explanation"));
+ assertTrue(toString.contains("Original Traversal"));
+ assertTrue(toString.contains("Final Traversal"));
+ }
+}
[16/43] incubator-tinkerpop git commit: TINKERPOP-1041 - give import
thread a chance to run
Posted by sp...@apache.org.
TINKERPOP-1041 - give import thread a chance to run
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/0715c4ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/0715c4ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/0715c4ef
Branch: refs/heads/TINKERPOP-1107
Commit: 0715c4ef247351cdd994378273f7d1b3aba282a8
Parents: 92edead
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Mon Feb 1 09:47:15 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
.../tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/0715c4ef/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java
index eb45c0b..9a7e1bf 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java
@@ -185,6 +185,7 @@ public class ScriptEnginesTest {
successes.incrementAndGet();
} catch (Exception ex) {
if (failures.incrementAndGet() == 500) threadImport.start();
+ Thread.yield();
}
})
);
@@ -194,8 +195,8 @@ public class ScriptEnginesTest {
threadEvalAndTriggerImport.join();
threadImport.join();
- assertTrue(successes.intValue() > 0);
- assertTrue(failures.intValue() >= 500);
+ assertTrue("Success: " + successes.intValue() + " - Failures: " + failures.intValue(), successes.intValue() > 0);
+ assertTrue("Success: " + successes.intValue() + " - Failures: " + failures.intValue(), failures.intValue() >= 500);
engines.close();
}
[11/43] incubator-tinkerpop git commit: minor parser fix
Posted by sp...@apache.org.
minor parser fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/606c68bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/606c68bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/606c68bf
Branch: refs/heads/TINKERPOP-1107
Commit: 606c68bfad95e6cc9407803a071e734a01489ade
Parents: c9149c2
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Fri Feb 19 18:26:17 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Fri Feb 19 18:26:17 2016 +0100
----------------------------------------------------------------------
docs/preprocessor/preprocess.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/606c68bf/docs/preprocessor/preprocess.sh
----------------------------------------------------------------------
diff --git a/docs/preprocessor/preprocess.sh b/docs/preprocessor/preprocess.sh
index 00d887b..cb7ab22 100755
--- a/docs/preprocessor/preprocess.sh
+++ b/docs/preprocessor/preprocess.sh
@@ -124,7 +124,7 @@ find "${TP_HOME}/docs/src/" -name index.asciidoc | xargs -n1 dirname | while rea
if [ ${process_subdirs} -eq 1 ]; then
find "${subdir}" -name "*.asciidoc" |
xargs -n1 basename |
- xargs -n1 -I {} echo "echo -ne {}' '; (grep -n {} ${subdir}/index.asciidoc || echo 0) | cut -d ':' -f1" | /bin/bash | sort -nk2 | cut -d ' ' -f1 |
+ xargs -n1 -I {} echo "echo -ne {}' '; (grep -n {} ${subdir}/index.asciidoc || echo 0) | head -n1 | cut -d ':' -f1" | /bin/bash | sort -nk2 | cut -d ' ' -f1 |
xargs -n1 -I {} echo "${subdir}/{}" |
xargs -n1 ${TP_HOME}/docs/preprocessor/preprocess-file.sh "${CONSOLE_HOME}"
[29/43] incubator-tinkerpop git commit: Added gremlin-archetypes and
related submodules.
Posted by sp...@apache.org.
Added gremlin-archetypes and related submodules.
Established the model for "examples" in TinkerPop using Maven archetypes. Provided two archetypes: gremlin-archetype-server and gremlin-archetype-tinkergraph.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bb3bf09f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bb3bf09f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bb3bf09f
Branch: refs/heads/TINKERPOP-1107
Commit: bb3bf09f57ba2ceb40565f2f9cefa886718d241d
Parents: 6f27995
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Feb 23 10:01:15 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Feb 23 10:01:15 2016 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +
.../src/reference/gremlin-applications.asciidoc | 22 +++++
.../upgrade/release-3.1.x-incubating.asciidoc | 15 +++-
.../gremlin-archetype-server/pom.xml | 59 ++++++++++++++
.../META-INF/maven/archetype-metadata.xml | 47 +++++++++++
.../archetype-resources/README.asciidoc | 67 ++++++++++++++++
.../archetype-resources/conf/log4j.properties | 21 +++++
.../conf/tinkergraph-empty.properties | 18 +++++
.../main/resources/archetype-resources/pom.xml | 80 +++++++++++++++++++
.../scripts/generate-modern.groovy | 33 ++++++++
.../archetype-resources/src/main/java/App.java | 35 ++++++++
.../src/main/java/Service.java | 70 ++++++++++++++++
.../src/main/resources/gremlin-server.yaml | 45 +++++++++++
.../src/test/java/ServiceTest.java | 84 ++++++++++++++++++++
.../projects/standard/archetype.properties | 21 +++++
.../test/resources/projects/standard/goal.txt | 1 +
.../gremlin-archetype-tinkergraph/pom.xml | 59 ++++++++++++++
.../META-INF/maven/archetype-metadata.xml | 38 +++++++++
.../archetype-resources/README.asciidoc | 35 ++++++++
.../main/resources/archetype-resources/pom.xml | 58 ++++++++++++++
.../archetype-resources/src/main/java/App.java | 78 ++++++++++++++++++
.../src/test/java/AppTest.java | 40 ++++++++++
.../projects/standard/archetype.properties | 21 +++++
.../test/resources/projects/standard/goal.txt | 1 +
gremlin-archetype/pom.xml | 70 ++++++++++++++++
pom.xml | 2 +
26 files changed, 1022 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index b35efcb..67965cf 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,9 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.2 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added the `gremlin-archetype-driver` archetype that demonstrates
+* Added the `gremlin-archetype-tinkergraph` archetype that demonstrates a basic project that uses TinkerGraph.
+* Added `gremlin-archetype` module to house TinkerPop "examples".
* Fixed a bug where multiple "close" requests were being sent by the driver on `Client.close()`.
* Fixed an `Property` attach bug that shows up in serialization-based `GraphComputer` implementations.
* Fixed a pom.xml bug where Gremlin Console/Server were not pulling the latest Neo4j 2.3.2.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/docs/src/reference/gremlin-applications.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/gremlin-applications.asciidoc b/docs/src/reference/gremlin-applications.asciidoc
index 43608b1..02bf1f0 100644
--- a/docs/src/reference/gremlin-applications.asciidoc
+++ b/docs/src/reference/gremlin-applications.asciidoc
@@ -1880,3 +1880,25 @@ The following shows the output for `HadoopGraph`:
----
describeGraph(HadoopGraph)
----
+
+[[gremlin-archetypes]]
+Gremlin Archetypes
+==================
+
+TinkerPop has a number of link:https://maven.apache.org/guides/introduction/introduction-to-archetypes.html[Maven archetypes],
+which provide example project templates to quickly get started with TinkerPop. The available archetypes are as follows:
+
+* `gremlin-archetype-server` - An example project that demonstrates the basic structure of a
+<<gremlin-server,Gremlin Server>> project, how to connect with the Gremlin Driver, and how to embed Gremlin Server in
+a testing framework.
+* `gremlin-archetype-tinkergraph` - A basic example of how to structure a TinkerPop project with Maven.
+
+You can use Maven to generate these example projects with a command like:
+
+[source,shell]
+$ mvn archetype:generate -DarchetypeGroupId=org.apache.tinkerpop -DarchetypeArtifactId=gremlin-archetype-server
+ -DarchetypeVersion=3.1.2-incubating -DgroupId=com.my -DartifactId=app
+
+This command will generate a new Maven project in a directory called "app" with a `pom.xml` specifying a `groupId` of
+`com.my`. Please see the `README.asciidoc` in the root of each generated project for information on how to build and
+execute it.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/docs/src/upgrade/release-3.1.x-incubating.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.1.x-incubating.asciidoc b/docs/src/upgrade/release-3.1.x-incubating.asciidoc
index 1b7425c..9776058 100644
--- a/docs/src/upgrade/release-3.1.x-incubating.asciidoc
+++ b/docs/src/upgrade/release-3.1.x-incubating.asciidoc
@@ -27,11 +27,24 @@ TinkerPop 3.1.2
*Release Date: NOT OFFICIALLY RELEASED YET*
-Please see the link:https://github.com/apache/incubator-tinkerpop/blob/3.1.1-incubating/CHANGELOG.asciidoc#tinkerpop-312-release-date-XXXXXXXXXXXXXXXXXXXXXXXXXX[changelog] for a complete list of all the modifications that are part of this release.
+Please see the link:https://github.com/apache/incubator-tinkerpop/blob/3.1.2-incubating/CHANGELOG.asciidoc#tinkerpop-312-release-date-XXXXXXXXXXXXXXXXXXXXXXXXXX[changelog] for a complete list of all the modifications that are part of this release.
Upgrading for Users
~~~~~~~~~~~~~~~~~~~
+TinkerPop Archetypes
+^^^^^^^^^^^^^^^^^^^^
+
+TinkerPop now offers link:https://maven.apache.org/guides/introduction/introduction-to-archetypes.html[Maven archetypes],
+which provide example project templates to quickly get started with TinkerPop. The available archetypes are as follows:
+
+* `gremlin-archetype-server` - An example project that demonstrates the basic structure of a Gremlin Server project,
+how to connect with the Gremlin Driver, and how to embed Gremlin Server in a testing framework.
+* `gremlin-archetype-tinkergraph` - A basic example of how to structure a TinkerPop project with Maven.
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-1085[TINKERPOP-1085],
+link:http://tinkerpop.apache.org/docs/3.1.2-incubating/reference/#gremlin-archetypes[Reference Documentation - Archetypes]
+
Session Transaction Management
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/pom.xml b/gremlin-archetype/gremlin-archetype-server/pom.xml
new file mode 100644
index 0000000..c5efbf7
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/pom.xml
@@ -0,0 +1,59 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-archetype</artifactId>
+ <version>3.1.2-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>gremlin-archetype-server</artifactId>
+ <name>Apache TinkerPop :: Archetype - Server</name>
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-archetype-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ </plugins>
+
+ <!-- apply variable substitution on the following files using variables from this pom -->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>archetype-resources/pom.xml</include>
+ <include>archetype-resources/README.asciidoc</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>false</filtering>
+ <excludes>
+ <exclude>archetype-resources/pom.xml</exclude>
+ <exclude>archetype-resources/README.asciidoc</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
new file mode 100644
index 0000000..a6c9d46
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/META-INF/maven/archetype-metadata.xml
@@ -0,0 +1,47 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<archetype-descriptor xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd" name="gremlin-archetype-tinkergraph"
+ xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <fileSets>
+ <fileSet filtered="true">
+ <directory></directory>
+ <includes>
+ <include>README.asciidoc</include>
+ </includes>
+ <excludes>
+ <exclude>**/*.xml</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet filtered="true">
+ <directory>conf/</directory>
+ </fileSet>
+ <fileSet filtered="true">
+ <directory>scripts/</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="true">
+ <directory>src/main/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="true">
+ <directory>src/main/resources</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="true">
+ <directory>src/test/java</directory>
+ </fileSet>
+ </fileSets>
+
+</archetype-descriptor>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
new file mode 100644
index 0000000..42f369d
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/README.asciidoc
@@ -0,0 +1,67 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+Gremlin Server Application
+==========================
+
+This is a starter project that demonstrates how a basic
+link:http://tinkerpop.apache.org/docs/${project.version}/reference/#gremlin-server[Gremlin Server] project is structured
+with Maven. This project demonstrates how to connect to Gremlin Server through Java using the
+link:http://tinkerpop.apache.org/docs/${project.version}/reference/#connecting-via-java[Gremlin Driver] that is
+distributed by TinkerPop.
+
+Prerequisites
+-------------
+
+* Java 8 Update 40+
+* link:https://maven.apache.org/[Maven 3.x]
+* Gremlin Server is link:http://archive.apache.org/dist/incubator/tinkerpop/[downloaded] and unpackaged
+
+Building and Running
+--------------------
+
+Start Gremlin Server in the directory it was unpackaged and specify usage of the "modern" graph configuration:
+
+[source,text]
+$ bin/gremlin-server.sh conf/gremlin-server-modern.yaml
+[INFO] GremlinServer -
+ \,,,/
+ (o o)
+-----oOOo-(3)-oOOo-----
+
+[INFO] GremlinServer - Configuring Gremlin Server from conf/gremlin-server-modern.yaml
+[INFO] MetricManager - Configured Metrics Slf4jReporter configured with interval=180000ms and loggerName=org.apache.tinkerpop.gremlin.server.Settings$Slf4jReporterMetrics
+[INFO] Graphs - Graph [graph] was successfully configured via [conf/tinkergraph-empty.properties].
+[INFO] ServerGremlinExecutor - Initialized Gremlin thread pool. Threads in pool named with pattern gremlin-*
+[INFO] ScriptEngines - Loaded gremlin-groovy ScriptEngine
+[INFO] GremlinExecutor - Initialized gremlin-groovy ScriptEngine with scripts/generate-modern.groovy
+[INFO] ServerGremlinExecutor - Initialized GremlinExecutor and configured ScriptEngines.
+[INFO] ServerGremlinExecutor - A GraphTraversalSource is now bound to [g] with graphtraversalsource[tinkergraph[vertices:0 edges:0], standard]
+[INFO] OpLoader - Adding the standard OpProcessor.
+[INFO] OpLoader - Adding the control OpProcessor.
+[INFO] OpLoader - Adding the session OpProcessor.
+[INFO] GremlinServer - Executing start up LifeCycleHook
+[INFO] Logger$info - Loading 'modern' graph data.
+[INFO] AbstractChannelizer - Configured application/vnd.gremlin-v1.0+gryo with org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0
+[INFO] AbstractChannelizer - Configured application/vnd.gremlin-v1.0+gryo-stringd with org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0
+[INFO] GremlinServer$1 - Gremlin Server configured with worker thread pool of 1, gremlin pool of 8 and boss thread pool of 1.
+[INFO] GremlinServer$1 - Channel started at port 8182.
+
+Build and run this project as follows:
+
+[source,text]
+mvn clean package
+mvn exec:java -Dexec.mainClass="${package}.App"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/tinkergraph-empty.properties
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/tinkergraph-empty.properties b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/tinkergraph-empty.properties
new file mode 100644
index 0000000..e09a13d
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/conf/tinkergraph-empty.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
+gremlin.tinkergraph.vertexIdManager=LONG
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
new file mode 100644
index 0000000..5c47aea
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/pom.xml
@@ -0,0 +1,80 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>\${groupId}</groupId>
+ <artifactId>\${artifactId}</artifactId>
+ <version>\${version}</version>
+
+ <name>Getting started with Gremlin Server</name>
+
+ <packaging>jar</packaging>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-driver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-server</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <!-- TinkerPop3 requires Java 8 -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.17</version>
+ <configuration>
+ <systemPropertyVariables>
+ <log4j.configuration>file:conf/log4j.properties</log4j.configuration>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/scripts/generate-modern.groovy
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/scripts/generate-modern.groovy b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/scripts/generate-modern.groovy
new file mode 100644
index 0000000..23b4a61
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/scripts/generate-modern.groovy
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// an init script that returns a Map allows explicit setting of global bindings.
+def globals = [:]
+
+// Generates the modern graph into an "empty" TinkerGraph via LifeCycleHook.
+// Note that the name of the key in the "global" map is unimportant.
+globals << [hook : [
+ onStartUp: { ctx ->
+ ctx.logger.info("Loading 'modern' graph data.")
+ TinkerFactory.generateModern(graph)
+ }
+] as LifeCycleHook]
+
+// define the default TraversalSource to bind queries to - this one will be named "g".
+globals << [g : graph.traversal()]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
new file mode 100644
index 0000000..d1761c7
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/App.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package ${package};
+
+import org.apache.tinkerpop.gremlin.driver.Result;
+
+public class App {
+
+ public static void main(String[] args) throws Exception {
+ Service service = Service.getInstance();
+ try {
+ service.findCreatorsOfSoftware("lop").iterator().forEachRemaining(r -> System.out.println(r));
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ } finally {
+ service.close();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/Service.java
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/Service.java b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/Service.java
new file mode 100644
index 0000000..b2e9de6
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/java/Service.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package ${package};
+
+import org.apache.tinkerpop.gremlin.driver.Cluster;
+import org.apache.tinkerpop.gremlin.driver.Client;
+import org.apache.tinkerpop.gremlin.driver.Result;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class Service implements AutoCloseable {
+
+ /**
+ * There typically needs to be only one Cluster instance in an application.
+ */
+ private final Cluster cluster = Cluster.open();
+
+ /**
+ * Use the Cluster instance to construct different Client instances (e.g. one for sessionless communication
+ * and one or more sessions). A sessionless Client should be thread-safe and typically no more than one is
+ * needed unless there is some need to divide connection pools across multiple Client instances. In this case
+ * there is just a single sessionless Client instance used for the entire App.
+ */
+ private final Client client = cluster.connect();
+
+ /**
+ * Create Service as a singleton given the simplicity of App.
+ */
+ private static final Service INSTANCE = new Service();
+
+ private Service() {}
+
+ public static Service getInstance() {
+ return INSTANCE;
+ }
+
+ public List<String> findCreatorsOfSoftware(String softwareName) throws Exception {
+ // it is very important from a performance perspective to parameterize queries
+ Map params = new HashMap();
+ params.put("n", softwareName);
+
+ return client.submit("g.V().hasLabel('software').has('name',n).in('created').values('name')", params)
+ .all().get().stream().map(r -> r.getString()).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() throws Exception {
+ client.close();
+ cluster.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml
new file mode 100644
index 0000000..5744d29
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/main/resources/gremlin-server.yaml
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+host: localhost
+port: 8182
+threadPoolWorker: 1
+gremlinPool: 8
+scriptEvaluationTimeout: 30000
+serializedResponseTimeout: 30000
+graphs: {
+ graph: conf/tinkergraph-empty.properties}
+plugins:
+ - tinkerpop.tinkergraph
+scriptEngines: {
+ gremlin-groovy: {
+ imports: [java.lang.Math],
+ staticImports: [java.lang.Math.PI],
+ scripts: [scripts/generate-modern.groovy]}}
+serializers:
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
+metrics: {
+ slf4jReporter: {enabled: true, interval: 180000}}
+strictTransactionManagement: false
+threadPoolBoss: 1
+maxInitialLineLength: 4096
+maxHeaderSize: 8192
+maxChunkSize: 8192
+maxContentLength: 65536
+maxAccumulationBufferComponents: 1024
+resultIterationBatchSize: 64
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/java/ServiceTest.java
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/java/ServiceTest.java b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/java/ServiceTest.java
new file mode 100644
index 0000000..6f7d5c3
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/java/ServiceTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package ${package};
+
+import org.apache.tinkerpop.gremlin.server.GremlinServer;
+import org.apache.tinkerpop.gremlin.server.Settings;
+
+import java.io.InputStream;
+import java.util.List;
+import java.util.Arrays;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.CoreMatchers.is;
+
+/**
+ * Basic test class that demonstrates how to start and stop an embedded Gremlin Server instance in a test. Note that
+ * the server instance is not started or stopped in a thread-safe manner, but typically this is acceptable for most
+ * testing use cases.
+ */
+public class ServiceTest {
+ private GremlinServer server;
+
+ private static Service service = Service.getInstance();
+
+ @Before
+ public void setUp() throws Exception {
+ startServer();
+ }
+
+ /**
+ * Starts a new instance of Gremlin Server.
+ */
+ public void startServer() throws Exception {
+ final InputStream stream = ServiceTest.class.getResourceAsStream("gremlin-server.yaml");
+ this.server = new GremlinServer(Settings.read(stream));
+
+ server.start().join();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stopServer();
+ }
+
+ /**
+ * Stops a current instance of Gremlin Server.
+ */
+ public void stopServer() throws Exception {
+ server.stop().join();
+ }
+
+ @AfterClass
+ public static void tearDownCase() throws Exception {
+ service.close();
+ }
+
+ @Test
+ public void shouldCreateGraph() throws Exception {
+ List<String> result = service.findCreatorsOfSoftware("lop");
+ assertThat(result, is(Arrays.asList("marko", "josh", "peter")));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/archetype.properties
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/archetype.properties b/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/archetype.properties
new file mode 100644
index 0000000..26067b7
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/archetype.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+groupId=org.apache.tinkerpop
+artifactId=gremlin-archetype-server
+package=com.test.example
+version=1.0.0-SNAPSHOT
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/goal.txt
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/goal.txt b/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/goal.txt
new file mode 100644
index 0000000..4a1a71d
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-server/src/test/resources/projects/standard/goal.txt
@@ -0,0 +1 @@
+verify
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/pom.xml b/gremlin-archetype/gremlin-archetype-tinkergraph/pom.xml
new file mode 100644
index 0000000..078071d
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/pom.xml
@@ -0,0 +1,59 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-archetype</artifactId>
+ <version>3.1.2-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>gremlin-archetype-tinkergraph</artifactId>
+ <name>Apache TinkerPop :: Archetype - TinkerGraph</name>
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-archetype-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ </plugins>
+
+ <!-- apply variable substitution on the following files using variables from this pom -->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>archetype-resources/pom.xml</include>
+ <include>archetype-resources/README.asciidoc</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>false</filtering>
+ <excludes>
+ <exclude>archetype-resources/pom.xml</exclude>
+ <exclude>archetype-resources/README.asciidoc</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/META-INF/maven/archetype-metadata.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/META-INF/maven/archetype-metadata.xml b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/META-INF/maven/archetype-metadata.xml
new file mode 100644
index 0000000..a159ee3
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/META-INF/maven/archetype-metadata.xml
@@ -0,0 +1,38 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<archetype-descriptor xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd" name="gremlin-archetype-tinkergraph"
+ xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <fileSets>
+ <fileSet filtered="true">
+ <directory></directory>
+ <includes>
+ <include>README.asciidoc</include>
+ </includes>
+ <excludes>
+ <exclude>**/*.xml</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet filtered="true" packaged="true">
+ <directory>src/main/java</directory>
+ </fileSet>
+ <fileSet filtered="true" packaged="true">
+ <directory>src/test/java</directory>
+ </fileSet>
+ </fileSets>
+
+</archetype-descriptor>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/README.asciidoc
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/README.asciidoc b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/README.asciidoc
new file mode 100644
index 0000000..59d5aff
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/README.asciidoc
@@ -0,0 +1,35 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+Simple TinkerPop Application
+============================
+
+This is a simple starter project that demonstrates how a basic TinkerPop project is structured with Maven. This project
+uses link:http://tinkerpop.apache.org/docs/${project.version}/reference/#tinkergraph-gremlin[TinkerGraph] as an
+embedded graph database, loads it with some data and then executes a shortest path calculation, printing the results.
+
+Prerequisites
+-------------
+
+* Java 8 Update 40+
+* link:https://maven.apache.org/[Maven 3.x]
+
+Building and Running
+--------------------
+
+[source,text]
+mvn clean package
+mvn exec:java -Dexec.mainClass="${package}.App"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/pom.xml b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/pom.xml
new file mode 100644
index 0000000..271f9dd
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/pom.xml
@@ -0,0 +1,58 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>\${groupId}</groupId>
+ <artifactId>\${artifactId}</artifactId>
+ <version>\${version}</version>
+
+ <name>Getting started with TinkerGraph</name>
+
+ <packaging>jar</packaging>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkergraph-gremlin</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <!-- TinkerPop3 requires Java 8 -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/main/java/App.java
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/main/java/App.java b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/main/java/App.java
new file mode 100644
index 0000000..dd97224
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/main/java/App.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package ${package};
+
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.*;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+public class App {
+
+ public static void main(String[] args) {
+ // Create a new TinkerGraph and load some test data. The Graph instance is typically named "graph" as a
+ // variable name. You will see this pattern consistently in TinkerPop documentation, the mailing list, etc.
+ Graph graph = TinkerGraph.open();
+ loadData(graph);
+
+ // Create a GraphTraversalSource instance that is used to query the data in the Graph instance. This variable
+ // is typically denoted as "g". In TinkerPop documentation you can always count on references to "g" as
+ // being a object of this type.
+ GraphTraversalSource g = graph.traversal();
+
+ Vertex fromNode = findByName(g, "marko");
+ Vertex toNode = findByName(g, "peter");
+
+ List list = calculateShortestPathBetween(g, fromNode, toNode);
+ System.out.println(list.toString());
+ System.exit(0);
+ }
+
+ public static Vertex findByName(GraphTraversalSource g, String name) {
+ return g.V().has("name", name).next();
+ }
+
+ public static List calculateShortestPathBetween(GraphTraversalSource g, Vertex fromNode, Vertex toNode) {
+ ArrayList list = new ArrayList();
+ g.V(fromNode).repeat(both().simplePath()).until(is(toNode)).limit(1).path().fill(list);
+ return list;
+ }
+
+ public static void loadData(Graph graph) {
+ // see org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory.generateModern()
+ final Vertex marko = graph.addVertex(T.id, 1, T.label, "person", "name", "marko", "age", 29);
+ final Vertex vadas = graph.addVertex(T.id, 2, T.label, "person", "name", "vadas", "age", 27);
+ final Vertex lop = graph.addVertex(T.id, 3, T.label, "software", "name", "lop", "lang", "java");
+ final Vertex josh = graph.addVertex(T.id, 4, T.label, "person", "name", "josh", "age", 32);
+ final Vertex ripple = graph.addVertex(T.id, 5, T.label, "software", "name", "ripple", "lang", "java");
+ final Vertex peter = graph.addVertex(T.id, 6, T.label, "person", "name", "peter", "age", 35);
+ marko.addEdge("knows", vadas, T.id, 7, "weight", 0.5d);
+ marko.addEdge("knows", josh, T.id, 8, "weight", 1.0d);
+ marko.addEdge("created", lop, T.id, 9, "weight", 0.4d);
+ josh.addEdge("created", ripple, T.id, 10, "weight", 1.0d);
+ josh.addEdge("created", lop, T.id, 11, "weight", 0.4d);
+ peter.addEdge("created", lop, T.id, 12, "weight", 0.2d);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/test/java/AppTest.java
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/test/java/AppTest.java b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/test/java/AppTest.java
new file mode 100644
index 0000000..7b400b7
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/main/resources/archetype-resources/src/test/java/AppTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package ${package};
+
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class AppTest {
+
+ @Test
+ public void shouldCreateGraph() {
+ TinkerGraph graph = TinkerGraph.open();
+ App.loadData(graph);
+
+ GraphTraversalSource g = graph.traversal();
+
+ assertEquals(6, g.V().count().next().intValue());
+ assertEquals(6, g.E().count().next().intValue());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/archetype.properties
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/archetype.properties b/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/archetype.properties
new file mode 100644
index 0000000..9c3e51c
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/archetype.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+groupId=org.apache.tinkerpop
+artifactId=gremlin-archetype-tinkergraph
+package=com.test.example
+version=1.0.0-SNAPSHOT
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/goal.txt
----------------------------------------------------------------------
diff --git a/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/goal.txt b/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/goal.txt
new file mode 100644
index 0000000..4a1a71d
--- /dev/null
+++ b/gremlin-archetype/gremlin-archetype-tinkergraph/src/test/resources/projects/standard/goal.txt
@@ -0,0 +1 @@
+verify
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/gremlin-archetype/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-archetype/pom.xml b/gremlin-archetype/pom.xml
new file mode 100644
index 0000000..fdfe35f
--- /dev/null
+++ b/gremlin-archetype/pom.xml
@@ -0,0 +1,70 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>tinkerpop</artifactId>
+ <groupId>org.apache.tinkerpop</groupId>
+ <version>3.1.2-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>gremlin-archetype</artifactId>
+ <name>Apache TinkerPop :: Gremlin Archetype</name>
+ <packaging>pom</packaging>
+ <modules>
+ <module>gremlin-archetype-tinkergraph</module>
+ <module>gremlin-archetype-server</module>
+ </modules>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-archetype-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <!-- without this configuration -DskipTests is useless -->
+ <skip>${skipTests}</skip>
+ </configuration>
+ </plugin>
+
+ <!--
+ Variables are applied in two separate rounds. The first round applies variables from this pom.xml
+ and the second round removes escaping from variables so that they can become variables during
+ archetype generation
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <configuration>
+ <escapeString>\</escapeString>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bb3bf09f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 311ddad..a8f1cfa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,6 +118,7 @@ limitations under the License.
<module>gremlin-driver</module>
<module>gremlin-console</module>
<module>gremlin-server</module>
+ <module>gremlin-archetype</module>
</modules>
<scm>
<connection>scm:git:git@git-wip-us.apache.org:repos/asf/incubator-tinkerpop.git</connection>
@@ -277,6 +278,7 @@ limitations under the License.
<exclude>**/*.json</exclude>
<exclude>**/*.xml</exclude>
<exclude>**/*.ldjson</exclude>
+ <exclude>**/goal.txt</exclude>
<exclude>**/src/main/resources/org/apache/tinkerpop/gremlin/structure/io/script/*.txt</exclude>
<exclude>**/src/main/resources/META-INF/services/**</exclude>
<exclude>**/src/main/ext/**</exclude>
[20/43] incubator-tinkerpop git commit: Url separator is always /
even on windows
Posted by sp...@apache.org.
Url separator is always / even on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/9d0c8770
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/9d0c8770
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/9d0c8770
Branch: refs/heads/TINKERPOP-1107
Commit: 9d0c87707e828b2ff5dbf8ef3d60051deebb055d
Parents: 0715c4e
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Mon Feb 1 13:15:33 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
.../java/org/apache/tinkerpop/gremlin/TestHelper.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9d0c8770/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
index 303b1e3..a3c252f 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
@@ -32,6 +32,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
@@ -50,6 +52,7 @@ import static org.junit.Assert.assertFalse;
public final class TestHelper {
private static final String SEP = File.separator;
+ private static final char URL_SEP = '/';
public static final String TEST_DATA_RELATIVE_DIR = "test-case-data";
private TestHelper() {
@@ -105,9 +108,14 @@ public final class TestHelper {
}
private static String computePath(final Class clazz) {
- final String clsUri = clazz.getName().replace('.', SEP.charAt(0)) + ".class";
+ final String clsUri = clazz.getName().replace('.', URL_SEP) + ".class";
final URL url = clazz.getClassLoader().getResource(clsUri);
- final String clsPath = url.getPath();
+ String clsPath;
+ try {
+ clsPath = new File(url.toURI()).getAbsolutePath();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Unable to computePath for " + clazz, e);
+ }
return clsPath.substring(0, clsPath.length() - clsUri.length());
}
[28/43] incubator-tinkerpop git commit: Add -parameters to compile
args.
Posted by sp...@apache.org.
Add -parameters to compile args.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5d0ac3ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5d0ac3ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5d0ac3ea
Branch: refs/heads/TINKERPOP-1107
Commit: 5d0ac3ea13ae5de0c52d08e4879998fef1fda4c2
Parents: f19311b
Author: rjbriody <bo...@datastax.com>
Authored: Tue Feb 23 08:36:33 2016 -0500
Committer: rjbriody <bo...@datastax.com>
Committed: Tue Feb 23 08:36:33 2016 -0500
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5d0ac3ea/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8f0bb2b..bb6b9cf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -308,6 +308,9 @@ limitations under the License.
<configuration>
<source>1.8</source>
<target>1.8</target>
+ <compilerArgs>
+ <arg>-parameters</arg>
+ </compilerArgs>
</configuration>
</plugin>
<plugin>
[39/43] incubator-tinkerpop git commit: Fixed asciidoc formatting
warning.
Posted by sp...@apache.org.
Fixed asciidoc formatting warning.
Cant' have a level 0 heading format on non-book doctypes CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1d54f002
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1d54f002
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1d54f002
Branch: refs/heads/TINKERPOP-1107
Commit: 1d54f0026e3ae18b76c572d36a1eb9dae03a2555
Parents: eaf7539
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Feb 25 06:55:12 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Feb 25 06:55:12 2016 -0500
----------------------------------------------------------------------
docs/src/tutorials/getting-started/index.asciidoc | 2 +-
docs/src/tutorials/the-gremlin-console/index.asciidoc | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d54f002/docs/src/tutorials/getting-started/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/tutorials/getting-started/index.asciidoc b/docs/src/tutorials/getting-started/index.asciidoc
index dcdb8fa..e9987df 100644
--- a/docs/src/tutorials/getting-started/index.asciidoc
+++ b/docs/src/tutorials/getting-started/index.asciidoc
@@ -18,7 +18,7 @@ limitations under the License.
image::apache-tinkerpop-logo.png[width=500]
Getting Started
-===============
+---------------
link:http://tinkerpop.apache.org[Apache TinkerPop] is an open source Graph Computing Framework. Within itself, TinkerPop
represents a large collection of capabilities and technologies and, in its wider ecosystem, an additionally extended
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d54f002/docs/src/tutorials/the-gremlin-console/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/tutorials/the-gremlin-console/index.asciidoc b/docs/src/tutorials/the-gremlin-console/index.asciidoc
index f477877..8159e92 100644
--- a/docs/src/tutorials/the-gremlin-console/index.asciidoc
+++ b/docs/src/tutorials/the-gremlin-console/index.asciidoc
@@ -18,7 +18,7 @@ limitations under the License.
image::apache-tinkerpop-logo.png[width=500]
The Gremlin Console
-===================
+-------------------
In link:http://tinkerpop.apache.org/docs/x.y.z/tutorials/getting-started/#_the_first_five_minutes["The First Five Minutes"]
of the link:http://tinkerpop.apache.org[Apache TinkerPop] tutorial on how to
[42/43] incubator-tinkerpop git commit: Allow ScriptEngine to accept
scripts in interpreter mode.
Posted by sp...@apache.org.
Allow ScriptEngine to accept scripts in interpreter mode.
Scripts sent with variables normally considered "local" by use of the def keyword or the inclusion of a type will be treated as global variables when the InterpreterModeCustomizerProvider configuration is included in the configuration of the ScriptEngine.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/566c4889
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/566c4889
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/566c4889
Branch: refs/heads/TINKERPOP-1107
Commit: 566c4889d05175057ecfa51ebf34cc9aa651b123
Parents: d0005b7
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Feb 18 10:47:28 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Feb 25 07:40:47 2016 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../jsr223/GremlinVariableAnalyzer.groovy | 104 +++++++++++++++++
.../groovy/jsr223/ast/InterpreterMode.groovy | 30 +++++
.../ast/InterpreterModeASTTransformation.groovy | 115 +++++++++++++++++++
.../jsr223/GremlinGroovyScriptEngine.java | 41 ++++++-
.../InterpreterModeCustomizerProvider.java | 37 ++++++
.../jsr223/GremlinGroovyScriptEngineTest.java | 27 +++++
.../server/GremlinServerIntegrateTest.java | 57 +++++++++
8 files changed, 408 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 6a57a60..b3a05fe 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.2 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added "interpreter mode" for the `ScriptEngine` and Gremlin Server which allows variables defined with `def` or a type to be recognized as "global".
* Bumped to Apache Groovy 2.4.6.
* Added the `gremlin-archetype-server` archetype that demonstrates
* Added the `gremlin-archetype-tinkergraph` archetype that demonstrates a basic project that uses TinkerGraph.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinVariableAnalyzer.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinVariableAnalyzer.groovy b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinVariableAnalyzer.groovy
new file mode 100644
index 0000000..1d1bcbe
--- /dev/null
+++ b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinVariableAnalyzer.groovy
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.groovy.jsr223
+
+import org.codehaus.groovy.ast.DynamicVariable
+import org.codehaus.groovy.ast.GroovyClassVisitor
+import org.codehaus.groovy.ast.expr.VariableExpression
+import org.codehaus.groovy.control.CompilationUnit
+import org.codehaus.groovy.control.CompilerConfiguration
+import org.codehaus.groovy.control.Phases
+import org.codehaus.groovy.tools.shell.util.ScriptVariableAnalyzer
+
+import java.security.CodeSource
+
+/**
+ * An extension of Groovy's {@code VariableVisitor} that exposes the bound and unbound variables publicly. This
+ * class can likely be removed with the next update of Groovy (after 2.4.5) as the {code ScriptVariableAnalyzer} ends
+ * up exposing {@code getBoundVars() in such a way as to allow for the {@code ClassLoader} to be supplied.
+ */
+class GremlinVariableAnalyzer {
+ public static class GremlinVariableVisitor extends ScriptVariableAnalyzer.VariableVisitor {
+ String lastBound
+
+ @Override
+ void visitVariableExpression(VariableExpression expression) {
+ if (!(expression.variable in ['args', 'context', 'this', 'super'])) {
+ if (expression.accessedVariable instanceof DynamicVariable) {
+ unbound << expression.variable
+ } else {
+ bound << expression.variable
+ lastBound = bound
+ }
+ }
+ super.visitVariableExpression(expression)
+ }
+
+ @Override
+ public Set<String> getBound() {
+ return super.getBound()
+ }
+
+ @Override
+ public Set<String> getUnbound() {
+ return super.getUnbound()
+ }
+ }
+
+ public static class GremlinVisitorClassLoader extends GroovyClassLoader {
+ private final GroovyClassVisitor visitor
+
+ public GremlinVisitorClassLoader(final GroovyClassVisitor visitor, ClassLoader parent) {
+ super(parent == null ? Thread.currentThread().getContextClassLoader() : parent)
+ this.visitor = visitor
+ }
+
+ @Override
+ protected CompilationUnit createCompilationUnit(final CompilerConfiguration config, final CodeSource source) {
+ CompilationUnit cu = super.createCompilationUnit(config, source)
+ cu.addPhaseOperation(new ScriptVariableAnalyzer.VisitorSourceOperation(visitor), Phases.CLASS_GENERATION)
+ return cu
+ }
+ }
+
+ public static BoundVars getBoundVars(final String scriptText, ClassLoader parent) {
+ assert scriptText != null
+ final GroovyClassVisitor visitor = new GremlinVariableVisitor()
+ new GremlinVisitorClassLoader(visitor, parent).parseClass(scriptText)
+ return new BoundVars(visitor.getLastBound(), visitor.getBound())
+ }
+
+ public static class BoundVars {
+ private String lastBound;
+ private Set<String> bound;
+
+ BoundVars(String lastBound, Set<String> bound) {
+ this.lastBound = lastBound
+ this.bound = bound
+ }
+
+ String getLastBound() {
+ return lastBound
+ }
+
+ Set<String> getBound() {
+ return bound
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterMode.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterMode.groovy b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterMode.groovy
new file mode 100644
index 0000000..a8da72f
--- /dev/null
+++ b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterMode.groovy
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.groovy.jsr223.ast
+
+import org.codehaus.groovy.transform.GroovyASTTransformationClass
+
+import java.lang.annotation.Retention
+import java.lang.annotation.RetentionPolicy
+
+@Retention(RetentionPolicy.SOURCE)
+@GroovyASTTransformationClass(["org.apache.tinkerpop.gremlin.groovy.jsr223.ast.InterpreterModeASTTransformation"])
+public @interface InterpreterMode {
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterModeASTTransformation.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterModeASTTransformation.groovy b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterModeASTTransformation.groovy
new file mode 100644
index 0000000..66638b8
--- /dev/null
+++ b/gremlin-groovy/src/main/groovy/org/apache/tinkerpop/gremlin/groovy/jsr223/ast/InterpreterModeASTTransformation.groovy
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.groovy.jsr223.ast
+
+import groovy.transform.CompileStatic
+import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine
+import org.codehaus.groovy.ast.ASTNode
+import org.codehaus.groovy.ast.ClassHelper
+import org.codehaus.groovy.ast.ClassNode
+import org.codehaus.groovy.ast.MethodNode
+import org.codehaus.groovy.ast.Parameter
+import org.codehaus.groovy.ast.expr.ArgumentListExpression
+import org.codehaus.groovy.ast.expr.BinaryExpression
+import org.codehaus.groovy.ast.expr.ConstantExpression
+import org.codehaus.groovy.ast.expr.DeclarationExpression
+import org.codehaus.groovy.ast.expr.MapExpression
+import org.codehaus.groovy.ast.expr.MethodCallExpression
+import org.codehaus.groovy.ast.expr.VariableExpression
+import org.codehaus.groovy.ast.stmt.BlockStatement
+import org.codehaus.groovy.ast.stmt.CatchStatement
+import org.codehaus.groovy.ast.stmt.EmptyStatement
+import org.codehaus.groovy.ast.stmt.ExpressionStatement
+import org.codehaus.groovy.ast.stmt.Statement
+import org.codehaus.groovy.ast.stmt.TryCatchStatement
+import org.codehaus.groovy.control.CompilePhase
+import org.codehaus.groovy.control.SourceUnit
+import org.codehaus.groovy.syntax.Token
+import org.codehaus.groovy.syntax.Types
+import org.codehaus.groovy.transform.ASTTransformation
+import org.codehaus.groovy.transform.GroovyASTTransformation
+
+/**
+ * An {@code ASTTransformation} that promotes "local" variables to global ones. In this case, "local" refers to those
+ * variables that are defined in a script with "def" at the root of the script. These would typically be interpreted
+ * as local to the script, but this transform changes that, by wrapping the entire script in a try/catch where such
+ * variables are written to a "hidden" {@link Map} so that the {@code ScriptEngine} can later access them to place
+ * them into the global context.
+ */
+@CompileStatic
+@GroovyASTTransformation(phase=CompilePhase.SEMANTIC_ANALYSIS)
+class InterpreterModeASTTransformation implements ASTTransformation {
+
+ @Override
+ void visit(ASTNode[] astNodes, SourceUnit sourceUnit) {
+ ClassNode scriptNode = (ClassNode) astNodes[1]
+ def runMethodOfScript = scriptNode.declaredMethodsMap["java.lang.Object run()"]
+ runMethodOfScript.code = wrap(runMethodOfScript)
+ }
+
+ private static BlockStatement wrap(MethodNode method) {
+ BlockStatement wrappedBlock = new BlockStatement()
+ BlockStatement existingBlock = ((BlockStatement) method.code)
+
+ // the variable names that will be written back to the global context
+ def variableNames = [] as Set<String>
+ variableNames.addAll(findTopLevelVariableDeclarations(existingBlock.statements))
+ method.variableScope.referencedClassVariablesIterator.each{variableNames << it.name}
+
+ // the map to hold the variables and values
+ wrappedBlock.addStatement(createGlobalMapAST())
+
+ // the finally block will capture all the vars in the "globals" map
+ BlockStatement finallyBlock = new BlockStatement()
+ variableNames.each {
+ finallyBlock.addStatement(createAssignToGlobalMapAST(it))
+ }
+
+ wrappedBlock.addStatement(new TryCatchStatement(existingBlock, finallyBlock))
+
+ return wrappedBlock
+ }
+
+ private static List<String> findTopLevelVariableDeclarations(def existingStatements) {
+ existingStatements.findAll{ it instanceof ExpressionStatement }
+ .collect{ ((ExpressionStatement) it).expression }
+ .findAll{ it instanceof DeclarationExpression}
+ .collect{ ((DeclarationExpression) it).leftExpression }
+ .collect{ ((VariableExpression) it).name }
+ }
+
+ private static Statement createAssignToGlobalMapAST(String varName) {
+ def tryCatch = new TryCatchStatement(new ExpressionStatement(
+ new MethodCallExpression(
+ new VariableExpression(GremlinGroovyScriptEngine.COLLECTED_BOUND_VARS_MAP_VARNAME),
+ "put",
+ new ArgumentListExpression(new ConstantExpression(varName), new VariableExpression(varName)))), EmptyStatement.INSTANCE)
+
+ tryCatch.addCatch(new CatchStatement(new Parameter(ClassHelper.make(MissingPropertyException), "ex"), EmptyStatement.INSTANCE))
+ return tryCatch
+ }
+
+ private static Statement createGlobalMapAST() {
+ new ExpressionStatement(
+ new BinaryExpression(
+ new VariableExpression(GremlinGroovyScriptEngine.COLLECTED_BOUND_VARS_MAP_VARNAME),
+ Token.newSymbol(Types.EQUAL, 0, 0),
+ new MapExpression()))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java
index 72e38c3..23240cb 100644
--- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.EmptyImportCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.ImportCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.NoImportCustomizerProvider;
+import org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.InterpreterModeCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.loaders.GremlinLoader;
import org.apache.tinkerpop.gremlin.groovy.plugin.Artifact;
import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin;
@@ -125,6 +126,12 @@ public class GremlinGroovyScriptEngine extends GroovyScriptEngineImpl implements
*/
public static final String REFERENCE_TYPE_HARD = "hard";
+ /**
+ * Name of variable that holds local variables to be globally bound if "interpreter mode" is enabled with
+ * {@link InterpreterModeCustomizerProvider}.
+ */
+ public static final String COLLECTED_BOUND_VARS_MAP_VARNAME = "gremlin_script_engine_collected_boundvars";
+
private static final Pattern patternImportStatic = Pattern.compile("\\Aimport\\sstatic.*");
public static final ThreadLocal<Map<String, Object>> COMPILE_OPTIONS = new ThreadLocal<Map<String, Object>>(){
@@ -146,7 +153,7 @@ public class GremlinGroovyScriptEngine extends GroovyScriptEngineImpl implements
private GremlinGroovyClassLoader loader;
- private AtomicLong counter = new AtomicLong(0l);
+ private AtomicLong counter = new AtomicLong(0L);
/**
* The list of loaded plugins for the console.
@@ -164,6 +171,7 @@ public class GremlinGroovyScriptEngine extends GroovyScriptEngineImpl implements
private final List<CompilerCustomizerProvider> customizerProviders;
private final Set<Artifact> artifactsToUse = new HashSet<>();
+ private final boolean interpreterModeEnabled;
/**
* Creates a new instance using the {@link DefaultImportCustomizerProvider}.
@@ -193,9 +201,14 @@ public class GremlinGroovyScriptEngine extends GroovyScriptEngineImpl implements
.map(p -> (ImportCustomizerProvider) p)
.findFirst().orElse(NoImportCustomizerProvider.INSTANCE);
+ // determine if interpreter mode should be enabled
+ interpreterModeEnabled = providers.stream()
+ .anyMatch(p -> p.getClass().equals(InterpreterModeCustomizerProvider.class));
+
// remove used providers as the rest will be applied directly
customizerProviders = providers.stream()
- .filter(p -> p != null && !(p instanceof ImportCustomizerProvider))
+ .filter(p -> p != null &&
+ !((p instanceof ImportCustomizerProvider)))
.collect(Collectors.toList());
createClassLoader();
@@ -531,7 +544,28 @@ public class GremlinGroovyScriptEngine extends GroovyScriptEngineImpl implements
}
}
});
- return scriptObject.run();
+
+ final Object o = scriptObject.run();
+
+ // if interpreter mode is enable then local vars of the script are promoted to engine scope bindings.
+ if (interpreterModeEnabled) {
+ final Map<String, Object> localVars = (Map<String, Object>) context.getAttribute(COLLECTED_BOUND_VARS_MAP_VARNAME);
+ if (localVars != null) {
+ localVars.entrySet().forEach(e -> {
+ // closures need to be cached for later use
+ if (e.getValue() instanceof Closure)
+ globalClosures.put(e.getKey(), (Closure) e.getValue());
+
+ context.setAttribute(e.getKey(), e.getValue(), ScriptContext.ENGINE_SCOPE);
+ });
+
+ // get rid of the temporary collected vars
+ context.removeAttribute(COLLECTED_BOUND_VARS_MAP_VARNAME, ScriptContext.ENGINE_SCOPE);
+ localVars.clear();
+ }
+ }
+
+ return o;
} catch (Exception e) {
throw new ScriptException(e);
}
@@ -642,5 +676,4 @@ public class GremlinGroovyScriptEngine extends GroovyScriptEngineImpl implements
}
return buf.toString();
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/customizer/InterpreterModeCustomizerProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/customizer/InterpreterModeCustomizerProvider.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/customizer/InterpreterModeCustomizerProvider.java
new file mode 100644
index 0000000..e3f95f4
--- /dev/null
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/customizer/InterpreterModeCustomizerProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.groovy.jsr223.customizer;
+
+import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider;
+import org.apache.tinkerpop.gremlin.groovy.jsr223.ast.InterpreterMode;
+import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer;
+import org.codehaus.groovy.control.customizers.CompilationCustomizer;
+
+/**
+ * Places the {@code ScriptEngine} in "interpreter mode" where local variables of a script are treated as global
+ * bindings. This implementation is technically not a true {@link CompilerCustomizerProvider} instance as the
+ * "interpreter mode" feature does not require a {@code CompilerCustomizer}. This class merely acts as a flag that
+ * tells the {@link org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine} to turn this feature on.
+ */
+public class InterpreterModeCustomizerProvider implements CompilerCustomizerProvider {
+ @Override
+ public CompilationCustomizer create() {
+ return new ASTTransformationCustomizer(InterpreterMode.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
index 4740cd2..17cd809 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
@@ -22,6 +22,7 @@ import groovy.lang.Closure;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.NoImportCustomizerProvider;
+import org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.InterpreterModeCustomizerProvider;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.javatuples.Pair;
@@ -85,7 +86,33 @@ public class GremlinGroovyScriptEngineTest {
@Test
public void shouldEvalWithNoBindings() throws Exception {
final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine();
+ engine.eval("def addItUp(x,y){x+y}");
assertEquals(3, engine.eval("1+2"));
+ assertEquals(3, engine.eval("addItUp(1,2)"));
+ }
+
+ @Test
+ public void shouldPromoteDefinedVarsInInterpreterModeWithNoBindings() throws Exception {
+ final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(new InterpreterModeCustomizerProvider());
+ engine.eval("def addItUp = { x, y -> x + y }");
+ assertEquals(3, engine.eval("int xxx = 1 + 2"));
+ assertEquals(4, engine.eval("yyy = xxx + 1"));
+ assertEquals(7, engine.eval("def zzz = yyy + xxx"));
+ assertEquals(4, engine.eval("zzz - xxx"));
+ assertEquals(10, engine.eval("addItUp(zzz,xxx)"));
+ }
+
+ @Test
+ public void shouldPromoteDefinedVarsInInterpreterModeWithBindings() throws Exception {
+ final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(new InterpreterModeCustomizerProvider());
+ final Bindings b = new SimpleBindings();
+ b.put("x", 2);
+ engine.eval("def addItUp = { x, y -> x + y }", b);
+ assertEquals(3, engine.eval("int xxx = 1 + x", b));
+ assertEquals(4, engine.eval("yyy = xxx + 1", b));
+ assertEquals(7, engine.eval("def zzz = yyy + xxx", b));
+ assertEquals(4, engine.eval("zzz - xxx", b));
+ assertEquals(10, engine.eval("addItUp(zzz,xxx)", b));
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/566c4889/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
index 7714a93..61c1a98 100644
--- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
+++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerIntegrateTest.java
@@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.driver.simple.SimpleClient;
import org.apache.tinkerpop.gremlin.driver.simple.WebSocketClient;
import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine;
import org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.CompileStaticCustomizerProvider;
+import org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.InterpreterModeCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.SimpleSandboxExtension;
import org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.TimedInterruptCustomizerProvider;
import org.apache.tinkerpop.gremlin.structure.T;
@@ -45,6 +46,7 @@ import org.junit.Test;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -136,6 +138,9 @@ public class GremlinServerIntegrateTest extends AbstractGremlinServerIntegration
case "shouldUseSimpleSandbox":
settings.scriptEngines.get("gremlin-groovy").config = getScriptEngineConfForSimpleSandbox();
break;
+ case "shouldUseInterpreterMode":
+ settings.scriptEngines.get("gremlin-groovy").config = getScriptEngineConfForInterpreterMode();
+ break;
case "shouldReceiveFailureTimeOutOnScriptEvalOfOutOfControlLoop":
settings.scriptEngines.get("gremlin-groovy").config = getScriptEngineConfForTimedInterrupt();
break;
@@ -164,6 +169,58 @@ public class GremlinServerIntegrateTest extends AbstractGremlinServerIntegration
return scriptEngineConf;
}
+ private static Map<String, Object> getScriptEngineConfForInterpreterMode() {
+ final Map<String,Object> scriptEngineConf = new HashMap<>();
+ final Map<String,Object> interpreterProviderConf = new HashMap<>();
+ interpreterProviderConf.put(InterpreterModeCustomizerProvider.class.getName(), Collections.EMPTY_LIST);
+ scriptEngineConf.put("compilerCustomizerProviders", interpreterProviderConf);
+ return scriptEngineConf;
+ }
+
+ @Test
+ public void shouldUseInterpreterMode() throws Exception {
+ final Cluster cluster = Cluster.open();
+ final Client client = cluster.connect(name.getMethodName());
+
+ client.submit("def subtractAway(x,y){x-y};[]").all().get();
+ client.submit("multiplyIt = { x,y -> x * y};[]").all().get();
+
+ assertEquals(2, client.submit("x = 1 + 1").all().get().get(0).getInt());
+ assertEquals(3, client.submit("int y = x + 1").all().get().get(0).getInt());
+ assertEquals(5, client.submit("def z = x + y").all().get().get(0).getInt());
+
+ final Map<String,Object> m = new HashMap<>();
+ m.put("x", 10);
+ assertEquals(-5, client.submit("z - x", m).all().get().get(0).getInt());
+ assertEquals(15, client.submit("addItUp(x,z)", m).all().get().get(0).getInt());
+ assertEquals(5, client.submit("subtractAway(x,z)", m).all().get().get(0).getInt());
+ assertEquals(50, client.submit("multiplyIt(x,z)", m).all().get().get(0).getInt());
+
+ cluster.close();
+ }
+
+ @Test
+ public void shouldNotUseInterpreterMode() throws Exception {
+ final Cluster cluster = Cluster.open();
+ final Client client = cluster.connect(name.getMethodName());
+
+ client.submit("def subtractAway(x,y){x-y};[]").all().get();
+ client.submit("multiplyIt = { x,y -> x * y};[]").all().get();
+
+ assertEquals(2, client.submit("x = 1 + 1").all().get().get(0).getInt());
+ assertEquals(3, client.submit("y = x + 1").all().get().get(0).getInt());
+ assertEquals(5, client.submit("z = x + y").all().get().get(0).getInt());
+
+ final Map<String,Object> m = new HashMap<>();
+ m.put("x", 10);
+ assertEquals(-5, client.submit("z - x", m).all().get().get(0).getInt());
+ assertEquals(15, client.submit("addItUp(x,z)", m).all().get().get(0).getInt());
+ assertEquals(5, client.submit("subtractAway(x,z)", m).all().get().get(0).getInt());
+ assertEquals(50, client.submit("multiplyIt(x,z)", m).all().get().get(0).getInt());
+
+ cluster.close();
+ }
+
@Test
public void shouldUseSimpleSandbox() throws Exception {
final Cluster cluster = Cluster.open();
[34/43] incubator-tinkerpop git commit: Fix HTTP keepalive support
Posted by sp...@apache.org.
Fix HTTP keepalive support
This commit[1] broke HTTP keepalive support by introducing a
double-negation on the call to Netty's isKeepAlive.
[1]: https://github.com/apache/incubator-tinkerpop/commit/c0121aae8266e9ff2c70c1517e809b5ecf1dd07a
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/33cd0039
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/33cd0039
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/33cd0039
Branch: refs/heads/TINKERPOP-1107
Commit: 33cd00390a014d6ce54c7627b00a3d1b3114bddc
Parents: 13f8005
Author: Benjamin Anderson <b...@banjiewen.net>
Authored: Tue Feb 23 17:24:15 2016 -0800
Committer: Benjamin Anderson <b...@banjiewen.net>
Committed: Wed Feb 24 10:05:20 2016 -0800
----------------------------------------------------------------------
.../gremlin/server/handler/HttpGremlinEndpointHandler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/33cd0039/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
index d37a1bc..7d2222f 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
@@ -180,7 +180,7 @@ public class HttpGremlinEndpointHandler extends ChannelInboundHandlerAdapter {
}
final String origin = req.headers().get(ORIGIN);
- final boolean keepAlive = !isKeepAlive(req);
+ final boolean keepAlive = isKeepAlive(req);
// not using the req any where below here - assume it is safe to release at this point.
ReferenceCountUtil.release(msg);
[02/43] incubator-tinkerpop git commit: Properly handle session
bindings to avoid concurrency errors.
Posted by sp...@apache.org.
Properly handle session bindings to avoid concurrency errors.
Basically, the bindings needed to be constructed within the session's executor to avoid two threads accessing it at the same time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b42dcea2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b42dcea2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b42dcea2
Branch: refs/heads/TINKERPOP-1107
Commit: b42dcea255cdb5c8df91a35561d6845ac1f41f9e
Parents: a5a00bf
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Feb 16 16:08:43 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Feb 16 16:08:43 2016 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b42dcea2/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index b8f48eb..cc4b4ad 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -200,7 +200,9 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
try {
b.putAll(bindingsSupplier.get());
} catch (OpProcessorException ope) {
- ope.printStackTrace();
+ // this should bubble up in the GremlinExecutor properly as the RuntimeException will be
+ // unwrapped and the root cause thrown
+ throw new RuntimeException(ope);
}
})
.withResult(o -> {
[14/43] incubator-tinkerpop git commit: Changed back slashes into
regular ones
Posted by sp...@apache.org.
Changed back slashes into regular ones
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/317d5c8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/317d5c8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/317d5c8a
Branch: refs/heads/TINKERPOP-1107
Commit: 317d5c8a2ca1677d634fc9a983eae234e6f2f323
Parents: a6cfc3b
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Mon Feb 1 17:39:13 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
.../gremlin/spark/structure/io/SparkContextStorage.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/317d5c8a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
index 76a3299..4b113a3 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/SparkContextStorage.java
@@ -75,9 +75,9 @@ public final class SparkContextStorage implements Storage {
@Override
public List<String> ls(final String location) {
final List<String> rdds = new ArrayList<>();
- final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*");
+ final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/');
for (final RDD<?> rdd : Spark.getRDDs()) {
- if (rdd.name().matches(wildCardLocation))
+ if (rdd.name().replace('\\', '/').matches(wildCardLocation))
rdds.add(rdd.name() + " [" + rdd.getStorageLevel().description() + "]");
}
return rdds;
@@ -103,9 +103,9 @@ public final class SparkContextStorage implements Storage {
@Override
public boolean rm(final String location) {
final List<String> rdds = new ArrayList<>();
- final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*");
+ final String wildCardLocation = (location.endsWith("*") ? location : location + "*").replace(".", "\\.").replace("*", ".*").replace('\\', '/');
for (final RDD<?> rdd : Spark.getRDDs()) {
- if (rdd.name().matches(wildCardLocation))
+ if (rdd.name().replace('\\', '/').matches(wildCardLocation))
rdds.add(rdd.name());
}
rdds.forEach(Spark::removeRDD);
[37/43] incubator-tinkerpop git commit: Merge branch
'fix-http-keepalive' of https://github.com/banjiewen/incubator-tinkerpop into
tp31
Posted by sp...@apache.org.
Merge branch 'fix-http-keepalive' of https://github.com/banjiewen/incubator-tinkerpop into tp31
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/df95d781
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/df95d781
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/df95d781
Branch: refs/heads/TINKERPOP-1107
Commit: df95d78139e4c52cd4750afb31d7a6fd7ea1cd79
Parents: 00e9b20 33cd003
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Feb 24 15:01:55 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Feb 24 15:01:55 2016 -0500
----------------------------------------------------------------------
.../gremlin/server/handler/HttpGremlinEndpointHandler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[08/43] incubator-tinkerpop git commit: deactivate Spark plugin for
implementations-neo4j.asciidoc and Neo4j plugin for
implementations-hadoop.asciidoc
Posted by sp...@apache.org.
deactivate Spark plugin for implementations-neo4j.asciidoc and Neo4j plugin for implementations-hadoop.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e9c6465c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e9c6465c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e9c6465c
Branch: refs/heads/TINKERPOP-1107
Commit: e9c6465c9060b2ad4a537bd75ab71c5bae25cf22
Parents: bbf5b3f
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Thu Feb 18 23:53:44 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Thu Feb 18 23:53:44 2016 +0100
----------------------------------------------------------------------
docs/preprocessor/preprocess-file.sh | 29 ++++++++++++++++++++++++++++-
1 file changed, 28 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e9c6465c/docs/preprocessor/preprocess-file.sh
----------------------------------------------------------------------
diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh
index a8bd3e3..fdc4799 100755
--- a/docs/preprocessor/preprocess-file.sh
+++ b/docs/preprocessor/preprocess-file.sh
@@ -37,7 +37,7 @@ fi
trap cleanup INT
function cleanup {
- rm -rf ${output}
+ rm -rf ${output} ${CONSOLE_HOME}/.ext
exit 255
}
@@ -57,6 +57,27 @@ if [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then
fi
pushd "${CONSOLE_HOME}" > /dev/null
+ doc=`basename ${input} .asciidoc`
+
+ case "${doc}" in
+ "implementations-neo4j")
+ # deactivate Spark plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars
+ mkdir .ext
+ mv ext/spark-gremlin .ext/
+ cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'SparkGremlinPlugin' > .ext/plugins.txt
+ ;;
+ "implementations-hadoop")
+ # deactivate Neo4j plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars
+ mkdir .ext
+ mv ext/neo4j-gremlin .ext/
+ cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'Neo4jGremlinPlugin' > .ext/plugins.txt
+ ;;
+ esac
+
+ if [ -d ".ext" ]; then
+ mv .ext/plugins.txt ext/
+ fi
+
awk -f ${AWK_SCRIPTS}/prepare.awk ${input} |
awk -f ${AWK_SCRIPTS}/init-code-blocks.awk |
awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template | HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/giraph-gremlin/lib:${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh |
@@ -70,6 +91,12 @@ if [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then
[ ${ec} -eq 0 ] || break
done
+ if [ -d ".ext" ]; then
+ mv .ext/plugins.all ext/plugins.txt
+ mv .ext/* ext/
+ rm -r .ext/
+ fi
+
if [ ${ec} -eq 0 ]; then
ec=`grep -c '\bpb([0-9][0-9]*);' ${output}`
fi
[25/43] incubator-tinkerpop git commit: Added Conductor Gremlin
Posted by sp...@apache.org.
Added Conductor Gremlin
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f2972e87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f2972e87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f2972e87
Branch: refs/heads/TINKERPOP-1107
Commit: f2972e877339f7a5252efe9a8a81096e60be6093
Parents: 10c978d
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Feb 22 08:13:35 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Feb 22 08:13:35 2016 -0500
----------------------------------------------------------------------
docs/static/images/gremlin-conductor.png | Bin 0 -> 38958 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f2972e87/docs/static/images/gremlin-conductor.png
----------------------------------------------------------------------
diff --git a/docs/static/images/gremlin-conductor.png b/docs/static/images/gremlin-conductor.png
new file mode 100644
index 0000000..d392ba5
Binary files /dev/null and b/docs/static/images/gremlin-conductor.png differ
[38/43] incubator-tinkerpop git commit: Very minor grammatical fix in
reference docs CTR
Posted by sp...@apache.org.
Very minor grammatical fix in reference docs CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/eaf7539c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/eaf7539c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/eaf7539c
Branch: refs/heads/TINKERPOP-1107
Commit: eaf7539c50ebc8441a2293ea2ef592659f01af9a
Parents: df95d78
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Feb 24 16:03:21 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Feb 24 16:03:21 2016 -0500
----------------------------------------------------------------------
docs/src/reference/gremlin-applications.asciidoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eaf7539c/docs/src/reference/gremlin-applications.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/gremlin-applications.asciidoc b/docs/src/reference/gremlin-applications.asciidoc
index 06fe0cc..39d212b 100644
--- a/docs/src/reference/gremlin-applications.asciidoc
+++ b/docs/src/reference/gremlin-applications.asciidoc
@@ -1186,7 +1186,7 @@ Considering Sessions
The preferred approach for issuing requests to Gremlin Server is to do so in a sessionless manner. The concept of
"sessionless" refers to a request that is completely encapsulated within a single transaction, such that the script
-in the request starts with a new transaction and ends with closed transaction. Sessionless requests have automatic
+in the request starts with a new transaction and ends with a closed transaction. Sessionless requests have automatic
transaction management handled by Gremlin Server, thus automatically opening and closing transactions as previously
described. The downside to the sessionless approach is that the entire script to be executed must be known at the
time of submission so that it can all be executed at once. This requirement makes it difficult for some use cases
[09/43] incubator-tinkerpop git commit: Fixed a broken test from a
previous commit.
Posted by sp...@apache.org.
Fixed a broken test from a previous commit.
Not sure how the test in gremlin server was failing as I thought I'd run the integration test suite prior to commit, but I suppose I somehow did not. CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/23e90f81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/23e90f81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/23e90f81
Branch: refs/heads/TINKERPOP-1107
Commit: 23e90f81939d8e875ee67654de419b5b928e778f
Parents: f19311b
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Feb 19 07:17:46 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Feb 19 07:18:42 2016 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/23e90f81/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
index cc4b4ad..e168857 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java
@@ -230,7 +230,9 @@ public abstract class AbstractEvalOpProcessor implements OpProcessor {
timerContext.stop();
if (t != null) {
- if (t instanceof TimedInterruptTimeoutException) {
+ if (t instanceof OpProcessorException) {
+ ctx.writeAndFlush(((OpProcessorException) t).getResponseMessage());
+ } else if (t instanceof TimedInterruptTimeoutException) {
// occurs when the TimedInterruptCustomizerProvider is in play
final String errorMessage = String.format("A timeout occurred within the script during evaluation of [%s] - consider increasing the limit given to TimedInterruptCustomizerProvider", msg);
logger.warn(errorMessage);
[13/43] incubator-tinkerpop git commit: Added an extra assert to
harden GremlinExecutor test
Posted by sp...@apache.org.
Added an extra assert to harden GremlinExecutor test
Helps validate timeouts weren't occurring in the test as part of the error.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/10c978db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/10c978db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/10c978db
Branch: refs/heads/TINKERPOP-1107
Commit: 10c978dbb5a62c4a5cd5ea9adc769e9ba6a8e596
Parents: 26f81b9
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Sat Feb 20 07:17:17 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Sat Feb 20 07:18:57 2016 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/10c978db/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
index 6d6abb4..251d8cf 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
@@ -635,7 +635,7 @@ public class GremlinExecutorTest {
// test is partially designed to protected against.
assertThat(failed.get(), is(false));
service.shutdown();
- service.awaitTermination(30000, TimeUnit.MILLISECONDS);
+ assertThat(service.awaitTermination(30000, TimeUnit.MILLISECONDS), is(true));
assertEquals(max, futures.size());
futures.forEach(t -> {
@@ -685,12 +685,12 @@ public class GremlinExecutorTest {
}
});
- service.shutdown();
- service.awaitTermination(30000, TimeUnit.MILLISECONDS);
-
// likely a concurrency exception if it occurs - and if it does then we've messed up because that's what this
// test is partially designed to protected against.
assertThat(failed.get(), is(false));
+ service.shutdown();
+ assertThat(service.awaitTermination(30000, TimeUnit.MILLISECONDS), is(true));
+
assertEquals(max, futures.size());
futures.forEach(t -> {
assertEquals(t.getValue0(), t.getValue1().get(0));
[05/43] incubator-tinkerpop git commit: splitted
implementations.asciidoc into implementations-hadoop.asciidoc and
implementations-neo4j.asciidoc
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bbf5b3f4/docs/src/reference/implementations.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations.asciidoc b/docs/src/reference/implementations.asciidoc
deleted file mode 100644
index 38c758c..0000000
--- a/docs/src/reference/implementations.asciidoc
+++ /dev/null
@@ -1,1835 +0,0 @@
-////
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-////
-[[implementations]]
-Implementations
-===============
-
-image::gremlin-racecar.png[width=325]
-
-[[graph-system-provider-requirements]]
-Graph System Provider Requirements
-----------------------------------
-
-image:tinkerpop-enabled.png[width=140,float=left] At the core of TinkerPop3 is a Java8 API. The implementation of this
-core API and its validation via the `gremlin-test` suite is all that is required of a graph system provider wishing to
-provide a TinkerPop3-enabled graph engine. Once a graph system has a valid implementation, then all the applications
-provided by TinkerPop (e.g. Gremlin Console, Gremlin Server, etc.) and 3rd-party developers (e.g. Gremlin-Scala,
-Gremlin-JS, etc.) will integrate properly. Finally, please feel free to use the logo on the left to promote your
-TinkerPop3 implementation.
-
-Implementing Gremlin-Core
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The classes that a graph system provider should focus on implementing are itemized below. It is a good idea to study
-the <<tinkergraph-gremlin,TinkerGraph>> (in-memory OLTP and OLAP in `tinkergraph-gremlin`), <<neo4j-gremlin,Neo4jGraph>>
-(OTLP w/ transactions in `neo4j-gremlin`) and/or <<hadoop-gremlin,HadoopGraph>> (OLAP in `hadoop-gremlin`)
-implementations for ideas and patterns.
-
-. Online Transactional Processing Graph Systems (*OLTP*)
- .. Structure API: `Graph`, `Element`, `Vertex`, `Edge`, `Property` and `Transaction` (if transactions are supported).
- .. Process API: `TraversalStrategy` instances for optimizing Gremlin traversals to the provider's graph system (i.e. `TinkerGraphStepStrategy`).
-. Online Analytics Processing Graph Systems (*OLAP*)
- .. Everything required of OTLP is required of OLAP (but not vice versa).
- .. GraphComputer API: `GraphComputer`, `Messenger`, `Memory`.
-
-Please consider the following implementation notes:
-
-* Be sure your `Graph` implementation is named as `XXXGraph` (e.g. TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
-* Use `StringHelper` to ensuring that the `toString()` representation of classes are consistent with other implementations.
-* Ensure that your implementation's `Features` (Graph, Vertex, etc.) are correct so that test cases handle particulars accordingly.
-* Use the numerous static method helper classes such as `ElementHelper`, `GraphComputerHelper`, `VertexProgramHelper`, etc.
-* There are a number of default methods on the provided interfaces that are semantically correct. However, if they are
-not efficient for the implementation, override them.
-* Implement the `structure/` package interfaces first and then, if desired, interfaces in the `process/` package interfaces.
-* `ComputerGraph` is a `Wrapper` system that ensure proper semantics during a GraphComputer computation.
-
-[[oltp-implementations]]
-OLTP Implementations
-^^^^^^^^^^^^^^^^^^^^
-
-image:pipes-character-1.png[width=110,float=right] The most important interfaces to implement are in the `structure/`
-package. These include interfaces like Graph, Vertex, Edge, Property, Transaction, etc. The `StructureStandardSuite`
-will ensure that the semantics of the methods implemented are correct. Moreover, there are numerous `Exceptions`
-classes with static exceptions that should be thrown by the graph system so that all the exceptions and their
-messages are consistent amongst all TinkerPop3 implementations.
-
-[[olap-implementations]]
-OLAP Implementations
-^^^^^^^^^^^^^^^^^^^^
-
-image:furnace-character-1.png[width=110,float=right] Implementing the OLAP interfaces may be a bit more complicated.
-Note that before OLAP interfaces are implemented, it is necessary for the OLTP interfaces to be, at minimal,
-implemented as specified in <<oltp-implementations,OLTP Implementations>>. A summary of each required interface
-implementation is presented below:
-
-. `GraphComputer`: A fluent builder for specifying an isolation level, a VertexProgram, and any number of MapReduce jobs to be submitted.
-. `Memory`: A global blackboard for ANDing, ORing, INCRing, and SETing values for specified keys.
-. `Messenger`: The system that collects and distributes messages being propagated by vertices executing the VertexProgram application.
-. `MapReduce.MapEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications map-phase.
-. `MapReduce.ReduceEmitter`: The system that collects key/value pairs being emitted by the MapReduce applications combine- and reduce-phases.
-
-NOTE: The VertexProgram and MapReduce interfaces in the `process/computer/` package are not required by the graph
-system. Instead, these are interfaces to be implemented by application developers writing VertexPrograms and MapReduce jobs.
-
-IMPORTANT: TinkerPop3 provides three OLAP implementations: <<tinkergraph-gremlin,TinkerGraphComputer>> (TinkerGraph),
-<<giraphgraphcomputer,GiraphGraphComputer>> (HadoopGraph), and <<sparkgraphcomputer,`SparkGraphComputer`>> (Hadoop).
-Given the complexity of the OLAP system, it is good to study and copy many of the patterns used in these reference
-implementations.
-
-Implementing GraphComputer
-++++++++++++++++++++++++++
-
-image:furnace-character-3.png[width=150,float=right] The most complex method in GraphComputer is the `submit()`-method. The method must do the following:
-
-. Ensure the the GraphComputer has not already been executed.
-. Ensure that at least there is a VertexProgram or 1 MapReduce job.
-. If there is a VertexProgram, validate that it can execute on the GraphComputer given the respectively defined features.
-. Create the Memory to be used for the computation.
-. Execute the VertexProgram.setup() method once and only once.
-. Execute the VertexProgram.execute() method for each vertex.
-. Execute the VertexProgram.terminate() method once and if true, repeat VertexProgram.execute().
-. When VertexProgram.terminate() returns true, move to MapReduce job execution.
-. MapReduce jobs are not required to be executed in any specified order.
-. For each Vertex, execute MapReduce.map(). Then (if defined) execute MapReduce.combine() and MapReduce.reduce().
-. Update Memory with runtime information.
-. Construct a new `ComputerResult` containing the compute Graph and Memory.
-
-Implementing Memory
-+++++++++++++++++++
-
-image:gremlin-brain.png[width=175,float=left] The Memory object is initially defined by `VertexProgram.setup()`.
-The memory data is available in the first round of the `VertexProgram.execute()` method. Each Vertex, when executing
-the VertexProgram, can update the Memory in its round. However, the update is not seen by the other vertices until
-the next round. At the end of the first round, all the updates are aggregated and the new memory data is available
-on the second round. This process repeats until the VertexProgram terminates.
-
-Implementing Messenger
-++++++++++++++++++++++
-
-The Messenger object is similar to the Memory object in that a vertex can read and write to the Messenger. However,
-the data it reads are the messages sent to the vertex in the previous step and the data it writes are the messages
-that will be readable by the receiving vertices in the subsequent round.
-
-Implementing MapReduce Emitters
-+++++++++++++++++++++++++++++++
-
-image:hadoop-logo-notext.png[width=150,float=left] The MapReduce framework in TinkerPop3 is similar to the model
-popularized by link:http://apache.hadoop.org[Hadoop]. The primary difference is that all Mappers process the vertices
-of the graph, not an arbitrary key/value pair. However, the vertices' edges can not be accessed -- only their
-properties. This greatly reduces the amount of data needed to be pushed through the MapReduce engine as any edge
-information required, can be computed in the VertexProgram.execute() method. Moreover, at this stage, vertices can
-not be mutated, only their token and property data read. A Gremlin OLAP system needs to provide implementations for
-to particular classes: `MapReduce.MapEmitter` and `MapReduce.ReduceEmitter`. TinkerGraph's implementation is provided
-below which demonstrates the simplicity of the algorithm (especially when the data is all within the same JVM).
-
-[source,java]
-----
-public class TinkerMapEmitter<K, V> implements MapReduce.MapEmitter<K, V> {
-
- public Map<K, Queue<V>> reduceMap;
- public Queue<KeyValue<K, V>> mapQueue;
- private final boolean doReduce;
-
- public TinkerMapEmitter(final boolean doReduce) { <1>
- this.doReduce = doReduce;
- if (this.doReduce)
- this.reduceMap = new ConcurrentHashMap<>();
- else
- this.mapQueue = new ConcurrentLinkedQueue<>();
- }
-
- @Override
- public void emit(K key, V value) {
- if (this.doReduce)
- this.reduceMap.computeIfAbsent(key, k -> new ConcurrentLinkedQueue<>()).add(value); <2>
- else
- this.mapQueue.add(new KeyValue<>(key, value)); <3>
- }
-
- protected void complete(final MapReduce<K, V, ?, ?, ?> mapReduce) {
- if (!this.doReduce && mapReduce.getMapKeySort().isPresent()) { <4>
- final Comparator<K> comparator = mapReduce.getMapKeySort().get();
- final List<KeyValue<K, V>> list = new ArrayList<>(this.mapQueue);
- Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
- this.mapQueue.clear();
- this.mapQueue.addAll(list);
- } else if (mapReduce.getMapKeySort().isPresent()) {
- final Comparator<K> comparator = mapReduce.getMapKeySort().get();
- final List<Map.Entry<K, Queue<V>>> list = new ArrayList<>();
- list.addAll(this.reduceMap.entrySet());
- Collections.sort(list, Comparator.comparing(Map.Entry::getKey, comparator));
- this.reduceMap = new LinkedHashMap<>();
- list.forEach(entry -> this.reduceMap.put(entry.getKey(), entry.getValue()));
- }
- }
-}
-----
-
-<1> If the MapReduce job has a reduce, then use one data structure (`reduceMap`), else use another (`mapList`). The
-difference being that a reduction requires a grouping by key and therefore, the `Map<K,Queue<V>>` definition. If no
-reduction/grouping is required, then a simple `Queue<KeyValue<K,V>>` can be leveraged.
-<2> If reduce is to follow, then increment the Map with a new value for the key. `MapHelper` is a TinkerPop3 class
-with static methods for adding data to a Map.
-<3> If no reduce is to follow, then simply append a KeyValue to the queue.
-<4> When the map phase is complete, any map-result sorting required can be executed at this point.
-
-[source,java]
-----
-public class TinkerReduceEmitter<OK, OV> implements MapReduce.ReduceEmitter<OK, OV> {
-
- protected Queue<KeyValue<OK, OV>> reduceQueue = new ConcurrentLinkedQueue<>();
-
- @Override
- public void emit(final OK key, final OV value) {
- this.reduceQueue.add(new KeyValue<>(key, value));
- }
-
- protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) {
- if (mapReduce.getReduceKeySort().isPresent()) {
- final Comparator<OK> comparator = mapReduce.getReduceKeySort().get();
- final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue);
- Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator));
- this.reduceQueue.clear();
- this.reduceQueue.addAll(list);
- }
- }
-}
-----
-
-The method `MapReduce.reduce()` is defined as:
-
-[source,java]
-public void reduce(final OK key, final Iterator<OV> values, final ReduceEmitter<OK, OV> emitter) { ... }
-
-In other words, for the TinkerGraph implementation, iterate through the entrySet of the `reduceMap` and call the
-`reduce()` method on each entry. The `reduce()` method can emit key/value pairs which are simply aggregated into a
-`Queue<KeyValue<OK,OV>>` in an analogous fashion to `TinkerMapEmitter` when no reduce is to follow. These two emitters
-are tied together in `TinkerGraphComputer.submit()`.
-
-[source,java]
-----
-...
-for (final MapReduce mapReduce : mapReducers) {
- if (mapReduce.doStage(MapReduce.Stage.MAP)) {
- final TinkerMapEmitter<?, ?> mapEmitter = new TinkerMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
- final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(this.graph.vertices());
- workers.setMapReduce(mapReduce);
- workers.mapReduceWorkerStart(MapReduce.Stage.MAP);
- workers.executeMapReduce(workerMapReduce -> {
- while (true) {
- final Vertex vertex = vertices.next();
- if (null == vertex) return;
- workerMapReduce.map(ComputerGraph.mapReduce(vertex), mapEmitter);
- }
- });
- workers.mapReduceWorkerEnd(MapReduce.Stage.MAP);
-
- // sort results if a map output sort is defined
- mapEmitter.complete(mapReduce);
-
- // no need to run combiners as this is single machine
- if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
- final TinkerReduceEmitter<?, ?> reduceEmitter = new TinkerReduceEmitter<>();
- final SynchronizedIterator<Map.Entry<?, Queue<?>>> keyValues = new SynchronizedIterator((Iterator) mapEmitter.reduceMap.entrySet().iterator());
- workers.mapReduceWorkerStart(MapReduce.Stage.REDUCE);
- workers.executeMapReduce(workerMapReduce -> {
- while (true) {
- final Map.Entry<?, Queue<?>> entry = keyValues.next();
- if (null == entry) return;
- workerMapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter);
- }
- });
- workers.mapReduceWorkerEnd(MapReduce.Stage.REDUCE);
- reduceEmitter.complete(mapReduce); // sort results if a reduce output sort is defined
- mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator()); <1>
- } else {
- mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator()); <2>
- }
- }
-}
-...
-----
-
-<1> Note that the final results of the reducer are provided to the Memory as specified by the application developer's
-`MapReduce.addResultToMemory()` implementation.
-<2> If there is no reduce stage, the the map-stage results are inserted into Memory as specified by the application
-developer's `MapReduce.addResultToMemory()` implementation.
-
-[[io-implementations]]
-IO Implementations
-^^^^^^^^^^^^^^^^^^
-
-If a `Graph` requires custom serializers for IO to work properly, implement the `Graph.io` method. A typical example
-of where a `Graph` would require such a custom serializers is if their identifier system uses non-primitive values,
-such as OrientDB's `Rid` class. From basic serialization of a single `Vertex` all the way up the stack to Gremlin
-Server, the need to know how to handle these complex identifiers is an important requirement.
-
-The first step to implementing custom serializers is to first implement the `IoRegistry` interface and register the
-custom classes and serializers to it. Each `Io` implementation has different requirements for what it expects from the
-`IoRegistry`:
-
-* *GraphML* - No custom serializers expected/allowed.
-* *GraphSON* - Register a Jackson `SimpleModule`. The `SimpleModule` encapsulates specific classes to be serialized,
-so it does not need to be registered to a specific class in the `IoRegistry` (use `null`).
-* *Gryo* - Expects registration of one of three objects:
-** Register just the custom class with a `null` Kryo `Serializer` implementation - this class will use default "field-level" Kryo serialization.
-** Register the custom class with a specific Kryo `Serializer' implementation.
-** Register the custom class with a `Function<Kryo, Serializer>` for those cases where the Kryo `Serializer` requires the `Kryo` instance to get constructed.
-
-This implementation should provide a zero-arg constructor as the stack may require instantiation via reflection.
-Consider extending `AbstractIoRegistry` for convenience as follows:
-
-[source,java]
-----
-public class MyGraphIoRegistry extends AbstractIoRegistry {
- public MyGraphIoRegistry() {
- register(GraphSONIo.class, null, new MyGraphSimpleModule());
- register(GryoIo.class, MyGraphIdClass.class, new MyGraphIdSerializer());
- }
-}
-----
-
-In the `Graph.io` method, provide the `IoRegistry` object to the supplied `Builder` and call the `create` method to
-return that `Io` instance as follows:
-
-[source,java]
-----
-public <I extends Io> I io(final Io.Builder<I> builder) {
- return (I) builder.graph(this).registry(myGraphIoRegistry).create();
-}}
-----
-
-In this way, `Graph` implementations can pre-configure custom serializers for IO interactions and users will not need
-to know about those details. Following this pattern will ensure proper execution of the test suite as well as
-simplified usage for end-users.
-
-IMPORTANT: Proper implementation of IO is critical to successful `Graph` operations in Gremlin Server. The Test Suite
-does have "serialization" tests that provide some assurance that an implementation is working properly, but those
-tests cannot make assertions against any specifics of a custom serializer. It is the responsibility of the
-implementer to test the specifics of their custom serializers.
-
-TIP: Consider separating serializer code into its own module, if possible, so that clients that use the `Graph`
-implementation remotely don't need a full dependency on the entire `Graph` - just the IO components and related
-classes being serialized.
-
-[[validating-with-gremlin-test]]
-Validating with Gremlin-Test
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-image:gremlin-edumacated.png[width=225]
-
-[source,xml]
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-test</artifactId>
- <version>x.y.z</version>
-</dependency>
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-groovy-test</artifactId>
- <version>x.y.z</version>
-</dependency>
-
-The operational semantics of any OLTP or OLAP implementation are validated by `gremlin-test` and functional
-interoperability with the Groovy environment is ensured by `gremlin-groovy-test`. To implement these tests, provide
-test case implementations as shown below, where `XXX` below denotes the name of the graph implementation (e.g.
-TinkerGraph, Neo4jGraph, HadoopGraph, etc.).
-
-[source,java]
-----
-// Structure API tests
-@RunWith(StructureStandardSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
-public class XXXStructureStandardTest {}
-
-// Process API tests
-@RunWith(ProcessComputerSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
-public class XXXProcessComputerTest {}
-
-@RunWith(ProcessStandardSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = XXXGraph.class)
-public class XXXProcessStandardTest {}
-
-@RunWith(GroovyEnvironmentSuite.class)
-@GraphProviderClass(provider = XXXProvider.class, graph = TinkerGraph.class)
-public class XXXGroovyEnvironmentTest {}
-
-@RunWith(GroovyProcessStandardSuite.class)
-@GraphProviderClass(provider = XXXGraphProvider.class, graph = TinkerGraph.class)
-public class XXXGroovyProcessStandardTest {}
-
-@RunWith(GroovyProcessComputerSuite.class)
-@GraphProviderClass(provider = XXXGraphComputerProvider.class, graph = TinkerGraph.class)
-public class XXXGroovyProcessComputerTest {}
-----
-
-The above set of tests represent the minimum test suite set to implement. There are other "integration" and
-"performance" tests that should be considered optional. Implementing those tests requires the same pattern as shown above.
-
-IMPORTANT: It is as important to look at "ignored" tests as it is to look at ones that fail. The `gremlin-test`
-suite utilizes the `Feature` implementation exposed by the `Graph` to determine which tests to execute. If a test
-utilizes features that are not supported by the graph, it will ignore them. While that may be fine, implementers
-should validate that the ignored tests are appropriately bypassed and that there are no mistakes in their feature
-definitions. Moreover, implementers should consider filling gaps in their own test suites, especially when
-IO-related tests are being ignored.
-
-The only test-class that requires any code investment is the `GraphProvider` implementation class. This class is a
-used by the test suite to construct `Graph` configurations and instances and provides information about the
-implementation itself. In most cases, it is best to simply extend `AbstractGraphProvider` as it provides many
-default implementations of the `GraphProvider` interface.
-
-Finally, specify the test suites that will be supported by the `Graph` implementation using the `@Graph.OptIn`
-annotation. See the `TinkerGraph` implementation below as an example:
-
-[source,java]
-----
-@Graph.OptIn(Graph.OptIn.SUITE_STRUCTURE_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
-@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_PROCESS_COMPUTER)
-@Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT)
-public class TinkerGraph implements Graph {
-----
-
-Only include annotations for the suites the implementation will support. Note that implementing the suite, but
-not specifying the appropriate annotation will prevent the suite from running (an obvious error message will appear
-in this case when running the mis-configured suite).
-
-There are times when there may be a specific test in the suite that the implementation cannot support (despite the
-features it implements) or should not otherwise be executed. It is possible for implementers to "opt-out" of a test
-by using the `@Graph.OptOut` annotation. The following is an example of this annotation usage as taken from
-`HadoopGraph`:
-
-[source, java]
-----
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_STANDARD)
-@Graph.OptIn(Graph.OptIn.SUITE_PROCESS_COMPUTER)
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
- method = "g_V_matchXa_hasXname_GarciaX__a_inXwrittenByX_b__a_inXsungByX_bX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.graph.step.map.MatchTest$Traversals",
- method = "g_V_matchXa_inXsungByX_b__a_inXsungByX_c__b_outXwrittenByX_d__c_outXwrittenByX_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldNotAllowBadMemoryKeys",
- reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldRequireRegisteringMemoryKeys",
- reason = "Hadoop does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-public class HadoopGraph implements Graph {
-----
-
-The above examples show how to ignore individual tests. It is also possible to:
-
-* Ignore an entire test case (i.e. all the methods within the test) by setting the `method` to "*".
-* Ignore a "base" test class such that test that extend from those classes will all be ignored. This style of
-ignoring is useful for Gremlin "process" tests that have bases classes that are extended by various Gremlin flavors (e.g. groovy).
-* Ignore a `GraphComputer` test based on the type of `GraphComputer` being used. Specify the "computer" attribute on
-the `OptOut` (which is an array specification) which should have a value of the `GraphComputer` implementation class
-that should ignore that test. This attribute should be left empty for "standard" execution and by default all
-`GraphComputer` implementations will be included in the `OptOut` so if there are multiple implementations, explicitly
-specify the ones that should be excluded.
-
-Also note that some of the tests in the Gremlin Test Suite are parameterized tests and require an additional level of
-specificity to be properly ignored. To ignore these types of tests, examine the name template of the parameterized
-tests. It is defined by a Java annotation that looks like this:
-
-[source, java]
-@Parameterized.Parameters(name = "expect({0})")
-
-The annotation above shows that the name of each parameterized test will be prefixed with "expect" and have
-parentheses wrapped around the first parameter (at index 0) value supplied to each test. This information can
-only be garnered by studying the test set up itself. Once the pattern is determined and the specific unique name of
-the parameterized test is identified, add it to the `specific` property on the `OptOut` annotation in addition to
-the other arguments.
-
-These annotations help provide users a level of transparency into test suite compliance (via the
-xref:describe-graph[describeGraph()] utility function). It also allows implementers to have a lot of flexibility in
-terms of how they wish to support TinkerPop. For example, maybe there is a single test case that prevents an
-implementer from claiming support of a `Feature`. The implementer could choose to either not support the `Feature`
-or to support it but "opt-out" of the test with a "reason" as to why so that users understand the limitation.
-
-IMPORTANT: Before using `OptOut` be sure that the reason for using it is sound and it is more of a last resort.
-It is possible that a test from the suite doesn't properly represent the expectations of a feature, is too broad or
-narrow for the semantics it is trying to enforce or simply contains a bug. Please consider raising issues in the
-developer mailing list with such concerns before assuming `OptOut` is the only answer.
-
-IMPORTANT: There are no tests that specifically validate complete compliance with Gremlin Server. Generally speaking,
-a `Graph` that passes the full Test Suite, should be compliant with Gremlin Server. The one area where problems can
-occur is in serialization. Always ensure that IO is properly implemented, that custom serializers are tested fully
-and ultimately integration test the `Graph` with an actual Gremlin Server instance.
-
-CAUTION: Configuring tests to run in parallel might result in errors that are difficult to debug as there is some
-shared state in test execution around graph configuration. It is therefore recommended that parallelism be turned
-off for the test suite (the Maven SureFire Plugin is configured this way by default). It may also be important to
-include this setting, `<reuseForks>false</reuseForks>`, in the SureFire configuration if tests are failing in an
-unexplainable way.
-
-Accessibility via GremlinPlugin
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-image:gremlin-plugin.png[width=100,float=left] The applications distributed with TinkerPop3 do not distribute with
-any graph system implementations besides TinkerGraph. If your implementation is stored in a Maven repository (e.g.
-Maven Central Repository), then it is best to provide a `GremlinPlugin` implementation so the respective jars can be
-downloaded according and when required by the user. Neo4j's GremlinPlugin is provided below for reference.
-
-[source,java]
-----
-public class Neo4jGremlinPlugin implements GremlinPlugin {
-
- private static final String IMPORT = "import ";
- private static final String DOT_STAR = ".*";
-
- private static final Set<String> IMPORTS = new HashSet<String>() {{
- add(IMPORT + Neo4jGraph.class.getPackage().getName() + DOT_STAR);
- }};
-
- @Override
- public String getName() {
- return "neo4j";
- }
-
- @Override
- public void pluginTo(final PluginAcceptor pluginAcceptor) {
- pluginAcceptor.addImports(IMPORTS);
- }
-}
-----
-
-With the above plugin implementations, users can now download respective binaries for Gremlin Console, Gremlin Server, etc.
-
-[source,groovy]
-gremlin> g = Neo4jGraph.open('/tmp/neo4j')
-No such property: Neo4jGraph for class: groovysh_evaluate
-Display stack trace? [yN]
-gremlin> :install org.apache.tinkerpop neo4j-gremlin x.y.z
-==>loaded: [org.apache.tinkerpop, neo4j-gremlin, …]
-gremlin> :plugin use tinkerpop.neo4j
-==>tinkerpop.neo4j activated
-gremlin> g = Neo4jGraph.open('/tmp/neo4j')
-==>neo4jgraph[EmbeddedGraphDatabase [/tmp/neo4j]]
-
-In-Depth Implementations
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-image:gremlin-painting.png[width=200,float=right] The graph system implementation details presented thus far are
-minimum requirements necessary to yield a valid TinkerPop3 implementation. However, there are other areas that a
-graph system provider can tweak to provide an implementation more optimized for their underlying graph engine. Typical
-areas of focus include:
-
-* Traversal Strategies: A <<traversalstrategy,TraversalStrategy>> can be used to alter a traversal prior to its
-execution. A typical example is converting a pattern of `g.V().has('name','marko')` into a global index lookup for
-all vertices with name "marko". In this way, a `O(|V|)` lookup becomes an `O(log(|V|))`. Please review
-`TinkerGraphStepStrategy` for ideas.
-* Step Implementations: Every <<graph-traversal-steps,step>> is ultimately referenced by the `GraphTraversal`
-interface. It is possible to extend `GraphTraversal` to use a graph system specific step implementation.
-
-
-[[tinkergraph-gremlin]]
-TinkerGraph-Gremlin
--------------------
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>tinkergraph-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-----
-
-image:tinkerpop-character.png[width=100,float=left] TinkerGraph is a single machine, in-memory (with optional
-persistence), non-transactional graph engine that provides both OLTP and OLAP functionality. It is deployed with
-TinkerPop3 and serves as the reference implementation for other providers to study in order to understand the
-semantics of the various methods of the TinkerPop3 API. Constructing a simple graph in Java8 is presented below.
-
-[source,java]
-Graph g = TinkerGraph.open();
-Vertex marko = g.addVertex("name","marko","age",29);
-Vertex lop = g.addVertex("name","lop","lang","java");
-marko.addEdge("created",lop,"weight",0.6d);
-
-The above graph creates two vertices named "marko" and "lop" and connects them via a created-edge with a weight=0.6
-property. Next, the graph can be queried as such.
-
-[source,java]
-g.V().has("name","marko").out("created").values("name")
-
-The `g.V().has("name","marko")` part of the query can be executed in two ways.
-
- * A linear scan of all vertices filtering out those vertices that don't have the name "marko"
- * A `O(log(|V|))` index lookup for all vertices with the name "marko"
-
-Given the initial graph construction in the first code block, no index was defined and thus, a linear scan is executed.
-However, if the graph was constructed as such, then an index lookup would be used.
-
-[source,java]
-Graph g = TinkerGraph.open();
-g.createIndex("name",Vertex.class)
-
-The execution times for a vertex lookup by property is provided below for both no-index and indexed version of
-TinkerGraph over the Grateful Dead graph.
-
-[gremlin-groovy]
-----
-graph = TinkerGraph.open()
-g = graph.traversal()
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
-clock(1000) {g.V().has('name','Garcia').iterate()} <1>
-graph = TinkerGraph.open()
-g = graph.traversal()
-graph.createIndex('name',Vertex.class)
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
-clock(1000){g.V().has('name','Garcia').iterate()} <2>
-----
-
-<1> Determine the average runtime of 1000 vertex lookups when no `name`-index is defined.
-<2> Determine the average runtime of 1000 vertex lookups when a `name`-index is defined.
-
-IMPORTANT: Each graph system will have different mechanism by which indices and schemas are defined. TinkerPop3
-does not require any conformance in this area. In TinkerGraph, the only definitions are around indices. With other
-graph systems, property value types, indices, edge labels, etc. may be required to be defined _a priori_ to adding
-data to the graph.
-
-NOTE: TinkerGraph is distributed with Gremlin Server and is therefore automatically available to it for configuration.
-
-Configuration
-~~~~~~~~~~~~~
-
-TinkerGraph has several settings that can be provided on creation via `Configuration` object:
-
-[width="100%",cols="2,10",options="header"]
-|=========================================================
-|Property |Description
-|gremlin.graph |`org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph`
-|gremlin.tinkergraph.vertexIdManager |The `IdManager` implementation to use for vertices.
-|gremlin.tinkergraph.edgeIdManager |The `IdManager` implementation to use for edges.
-|gremlin.tinkergraph.vertexPropertyIdManager |The `IdManager` implementation to use for vertex properties.
-|gremlin.tinkergraph.defaultVertexPropertyCardinality |The default `VertexProperty.Cardinality` to use when `Vertex.property(k,v)` is called.
-|gremlin.tinkergraph.graphLocation |The path and file name for where TinkerGraph should persist the graph data. If a
-value is specified here, the the `gremlin.tinkergraph.graphFormat` should also be specified. If this value is not
-included (default), then the graph will stay in-memory and not be loaded/persisted to disk.
-|gremlin.tinkergraph.graphFormat |The format to use to serialize the graph which may be one of the following:
-`graphml`, `graphson`, `gryo`, or a fully qualified class name that implements Io.Builder interface (which allows for
-external third party graph reader/writer formats to be used for persistence).
-If a value is specified here, then the `gremlin.tinkergraph.graphLocation` should
-also be specified. If this value is not included (default), then the graph will stay in-memory and not be
-loaded/persisted to disk.
-|=========================================================
-
-The `IdManager` settings above refer to how TinkerGraph will control identifiers for vertices, edges and vertex
-properties. There are several options for each of these settings: `ANY`, `LONG`, `INTEGER`, `UUID`, or the fully
-qualified class name of an `IdManager` implementation on the classpath. When not specified, the default values
-for all settings is `ANY`, meaning that the graph will work with any object on the JVM as the identifier and will
-generate new identifiers from `Long` when the identifier is not user supplied. TinkerGraph will also expect the
-user to understand the types used for identifiers when querying, meaning that `g.V(1)` and `g.V(1L)` could return
-two different vertices. `LONG`, `INTEGER` and `UUID` settings will try to coerce identifier values to the expected
-type as well as generate new identifiers with that specified type.
-
-If the TinkerGraph is configured for persistence with `gremlin.tinkergraph.graphLocation` and
-`gremlin.tinkergraph.graphFormat`, then the graph will be written to the specified location with the specified
-format when `Graph.close()` is called. In addition, if these settings are present, TinkerGraph will attempt to
-load the graph from the specified location.
-
-IMPORTANT: If choosing `graphson` as the `gremlin.tinkergraph.graphFormat`, be sure to also establish the various
-`IdManager` settings as well to ensure that identifiers are properly coerced to the appropriate types as GraphSON
-can lose the identifier's type during serialization (i.e. it will assume `Integer` when the default for TinkerGraph
-is `Long`, which could lead to load errors that result in a message like, "Vertex with id already exists").
-
-It is important to consider the data being imported to TinkerGraph with respect to `defaultVertexPropertyCardinality`
-setting. For example, if a `.gryo` file is known to contain multi-property data, be sure to set the default
-cardinality to `list` or else the data will import as `single`. Consider the following:
-
-[gremlin-groovy]
-----
-graph = TinkerGraph.open()
-graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
-g = graph.traversal()
-g.V().properties()
-conf = new BaseConfiguration()
-conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality","list")
-graph = TinkerGraph.open(conf)
-graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
-g = graph.traversal()
-g.V().properties()
-----
-
-[[neo4j-gremlin]]
-Neo4j-Gremlin
--------------
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>neo4j-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-<!-- neo4j-tinkerpop-api-impl is NOT Apache 2 licensed - more information below -->
-<dependency>
- <groupId>org.neo4j</groupId>
- <artifactId>neo4j-tinkerpop-api-impl</artifactId>
- <version>0.1-2.2</version>
-</dependency>
-----
-
-link:http://neotechnology.com[Neo Technology] are the developers of the OLTP-based link:http://neo4j.org[Neo4j graph database].
-
-CAUTION: Unless under a commercial agreement with Neo Technology, Neo4j is licensed
-link:http://en.wikipedia.org/wiki/Affero_General_Public_License[AGPL]. The `neo4j-gremlin` module is licensed Apache2
-because it only references the Apache2-licensed Neo4j API (not its implementation). Note that neither the
-<<gremlin-console,Gremlin Console>> nor <<gremlin-server,Gremlin Server>> distribute with the Neo4j implementation
-binaries. To access the binaries, use the `:install` command to download binaries from
-link:http://search.maven.org/[Maven Central Repository].
-
-[source,groovy]
-----
-gremlin> :install org.apache.tinkerpop neo4j-gremlin x.y.z
-==>Loaded: [org.apache.tinkerpop, neo4j-gremlin, x.y.z] - restart the console to use [tinkerpop.neo4j]
-gremlin> :q
-...
-gremlin> :plugin use tinkerpop.neo4j
-==>tinkerpop.neo4j activated
-gremlin> graph = Neo4jGraph.open('/tmp/neo4j')
-==>neo4jgraph[EmbeddedGraphDatabase [/tmp/neo4j]]
-----
-
-NOTE: Neo4j link:http://docs.neo4j.org/chunked/stable/ha.html[High Availability] is currently not supported by
-Neo4j-Gremlin.
-
-TIP: To host Neo4j in <<gremlin-server,Gremlin Server>>, the dependencies must first be "installed" or otherwise
-copied to the Gremlin Server path. The automated method for doing this would be to execute
-`bin/gremlin-server.sh -i org.apache.tinkerpop neo4j-gremlin x.y.z`.
-
-Indices
-~~~~~~~
-
-Neo4j 2.x indices leverage vertex labels to partition the index space. TinkerPop3 does not provide method interfaces
-for defining schemas/indices for the underlying graph system. Thus, in order to create indices, it is important to
-call the Neo4j API directly.
-
-NOTE: `Neo4jGraphStep` will attempt to discern which indices to use when executing a traversal of the form `g.V().has()`.
-
-The Gremlin-Console session below demonstrates Neo4j indices. For more information, please refer to the Neo4j documentation:
-
-* Manipulating indices with link:http://docs.neo4j.org/chunked/stable/query-schema-index.html[Cypher].
-* Manipulating indices with the Neo4j link:http://docs.neo4j.org/chunked/stable/tutorials-java-embedded-new-index.html[Java API].
-
-[gremlin-groovy]
-----
-graph = Neo4jGraph.open('/tmp/neo4j')
-graph.cypher("CREATE INDEX ON :person(name)")
-graph.tx().commit() <1>
-graph.addVertex(label,'person','name','marko')
-graph.addVertex(label,'dog','name','puppy')
-g = graph.traversal()
-g.V().hasLabel('person').has('name','marko').values('name')
-graph.close()
-----
-
-<1> Schema mutations must happen in a different transaction than graph mutations
-
-Below demonstrates the runtime benefits of indices and demonstrates how if there is no defined index (only vertex
-labels), a linear scan of the vertex-label partition is still faster than a linear scan of all vertices.
-
-[gremlin-groovy]
-----
-graph = Neo4jGraph.open('/tmp/neo4j')
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
-g = graph.traversal()
-g.tx().commit()
-clock(1000) {g.V().hasLabel('artist').has('name','Garcia').iterate()} <1>
-graph.cypher("CREATE INDEX ON :artist(name)") <2>
-g.tx().commit()
-Thread.sleep(5000) <3>
-clock(1000) {g.V().hasLabel('artist').has('name','Garcia').iterate()} <4>
-clock(1000) {g.V().has('name','Garcia').iterate()} <5>
-graph.cypher("DROP INDEX ON :artist(name)") <6>
-g.tx().commit()
-graph.close()
-----
-
-<1> Find all artists whose name is Garcia which does a linear scan of the artist vertex-label partition.
-<2> Create an index for all artist vertices on their name property.
-<3> Neo4j indices are eventually consistent so this stalls to give the index time to populate itself.
-<4> Find all artists whose name is Garcia which uses the pre-defined schema index.
-<5> Find all vertices whose name is Garcia which requires a linear scan of all the data in the graph.
-<6> Drop the created index.
-
-Multi/Meta-Properties
-~~~~~~~~~~~~~~~~~~~~~
-
-`Neo4jGraph` supports both multi- and meta-properties (see <<_vertex_properties,vertex properties>>). These features
-are not native to Neo4j and are implemented using "hidden" Neo4j nodes. For example, when a vertex has multiple
-"name" properties, each property is a new node (multi-properties) which can have properties attached to it
-(meta-properties). As such, the native, underlying representation may become difficult to query directly using
-another graph language such as <<_cypher,Cypher>>. The default setting is to disable multi- and meta-properties.
-However, if this feature is desired, then it can be activated via `gremlin.neo4j.metaProperties` and
-`gremlin.neo4j.multiProperties` configurations being set to `true`. Once the configuration is set, it can not be
-changed for the lifetime of the graph.
-
-[gremlin-groovy]
-----
-conf = new BaseConfiguration()
-conf.setProperty('gremlin.neo4j.directory','/tmp/neo4j')
-conf.setProperty('gremlin.neo4j.multiProperties',true)
-conf.setProperty('gremlin.neo4j.metaProperties',true)
-graph = Neo4jGraph.open(conf)
-g = graph.traversal()
-g.addV('name','michael','name','michael hunger','name','mhunger')
-g.V().properties('name').property('acl', 'public')
-g.V(0).valueMap()
-g.V(0).properties()
-g.V(0).properties().valueMap()
-graph.close()
-----
-
-WARNING: `Neo4jGraph` without multi- and meta-properties is in 1-to-1 correspondence with the native, underlying Neo4j
-representation. It is recommended that if the user does not require multi/meta-properties, then they should not
-enable them. Without multi- and meta-properties enabled, Neo4j can be interacted with with other tools and technologies
-that do not leverage TinkerPop.
-
-IMPORTANT: When using a multi-property enabled `Neo4jGraph`, vertices may represent their properties on "hidden
-nodes" adjacent to the vertex. If a vertex property key/value is required for indexing, then two indices are
-required -- e.g. `CREATE INDEX ON :person(name)` and `CREATE INDEX ON :vertexProperty(name)`
-(see <<_indices,Neo4j indices>>).
-
-Cypher
-~~~~~~
-
-image::gremlin-loves-cypher.png[width=400]
-
-NeoTechnology are the creators of the graph pattern-match query language link:http://www.neo4j.org/learn/cypher[Cypher].
-It is possible to leverage Cypher from within Gremlin by using the `Neo4jGraph.cypher()` graph traversal method.
-
-[gremlin-groovy]
-----
-graph = Neo4jGraph.open('/tmp/neo4j')
-graph.io(gryo()).readGraph('data/tinkerpop-modern.kryo')
-graph.cypher('MATCH (a {name:"marko"}) RETURN a')
-graph.cypher('MATCH (a {name:"marko"}) RETURN a').select('a').out('knows').values('name')
-graph.close()
-----
-
-Thus, like <<match-step,`match()`>>-step in Gremlin, it is possible to do a declarative pattern match and then move
-back into imperative Gremlin.
-
-TIP: For those developers using <<gremlin-server,Gremlin Server>> against Neo4j, it is possible to do Cypher queries
-by simply placing the Cypher string in `graph.cypher(...)` before submission to the server.
-
-Multi-Label
-~~~~~~~~~~~
-
-TinkerPop3 requires every `Element` to have a single, immutable string label (i.e. a `Vertex`, `Edge`, and
-`VertexProperty`). In Neo4j, a `Node` (vertex) can have an
-link:http://neo4j.com/docs/stable/graphdb-neo4j-labels.html[arbitrary number of labels] while a `Relationship`
-(edge) can have one and only one. Furthermore, in Neo4j, `Node` labels are mutable while `Relationship` labels are
-not. In order to handle this mismatch, three `Neo4jVertex` specific methods exist in Neo4j-Gremlin.
-
-[source,java]
-public Set<String> labels() // get all the labels of the vertex
-public void addLabel(String label) // add a label to the vertex
-public void removeLabel(String label) // remove a label from the vertex
-
-An example use case is presented below.
-
-[gremlin-groovy]
-----
-graph = Neo4jGraph.open('/tmp/neo4j')
-vertex = (Neo4jVertex) graph.addVertex('human::animal') <1>
-vertex.label() <2>
-vertex.labels() <3>
-vertex.addLabel('organism') <4>
-vertex.label()
-vertex.removeLabel('human') <5>
-vertex.labels()
-vertex.addLabel('organism') <6>
-vertex.labels()
-vertex.removeLabel('human') <7>
-vertex.label()
-g = graph.traversal()
-g.V().has(label,'organism') <8>
-g.V().has(label,of('organism')) <9>
-g.V().has(label,of('organism')).has(label,of('animal'))
-g.V().has(label,of('organism').and(of('animal')))
-graph.close()
-----
-
-<1> Typecasting to a `Neo4jVertex` is only required in Java.
-<2> The standard `Vertex.label()` method returns all the labels in alphabetical order concatenated using `::`.
-<3> `Neo4jVertex.labels()` method returns the individual labels as a set.
-<4> `Neo4jVertex.addLabel()` method adds a single label.
-<5> `Neo4jVertex.removeLabel()` method removes a single label.
-<6> Labels are unique and thus duplicate labels don't exist.
-<7> If a label that does not exist is removed, nothing happens.
-<8> `P.eq()` does a full string match and should only be used if multi-labels are not leveraged.
-<9> `LabelP.of()` is specific to `Neo4jGraph` and used for multi-label matching.
-
-IMPORTANT: `LabelP.of()` is only required if multi-labels are leveraged. `LabelP.of()` is used when
-filtering/looking-up vertices by their label(s) as the standard `P.eq()` does a direct match on the `::`-representation
-of `vertex.label()`
-
-Loading with BulkLoaderVertexProgram
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load
-large amounts of data to and from Neo4j. The following code demonstrates how to load the modern graph from TinkerGraph
-into Neo4j:
-
-[gremlin-groovy]
-----
-wgConf = 'conf/neo4j-standalone.properties'
-modern = TinkerFactory.createModern()
-blvp = BulkLoaderVertexProgram.build().
- keepOriginalIds(false).
- writeGraph(wgConf).create(modern)
-modern.compute().workers(1).program(blvp).submit().get()
-graph = GraphFactory.open(wgConf)
-g = graph.traversal()
-g.V().valueMap()
-graph.close()
-----
-
-[source,properties]
-----
-# neo4j-standalone.properties
-
-gremlin.graph=org.apache.tinkerpop.gremlin.neo4j.structure.Neo4jGraph
-gremlin.neo4j.directory=/tmp/neo4j
-gremlin.neo4j.conf.node_auto_indexing=true
-gremlin.neo4j.conf.relationship_auto_indexing=true
-----
-
-[[hadoop-gremlin]]
-Hadoop-Gremlin
---------------
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>hadoop-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-----
-
-image:hadoop-logo-notext.png[width=100,float=left] link:http://hadoop.apache.org/[Hadoop] is a distributed
-computing framework that is used to process data represented across a multi-machine compute cluster. When the
-data in the Hadoop cluster represents a TinkerPop3 graph, then Hadoop-Gremlin can be used to process the graph
-using both TinkerPop3's OLTP and OLAP graph computing models.
-
-IMPORTANT: This section assumes that the user has a Hadoop 2.x cluster functioning. For more information on getting
-started with Hadoop, please see the
-link:http://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-common/SingleCluster.html[Single Node Setup]
-tutorial. Moreover, if using `GiraphGraphComputer` or `SparkGraphComputer` it is advisable that the reader also
-familiarize their self with Giraph (link:http://giraph.apache.org/quick_start.html[Getting Started]) and Spark
-(link:http://spark.apache.org/docs/latest/quick-start.html[Quick Start]).
-
-Installing Hadoop-Gremlin
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The `HADOOP_GREMLIN_LIBS` references locations that contains jars that should be uploaded to a respective
-distributed cache (link:http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html[YARN] or SparkServer).
-Note that the locations in `HADOOP_GREMLIN_LIBS` can be a colon-separated (`:`) and all jars from all locations will
-be loaded into the cluster. Typically, only the jars of the respective GraphComputer are required to be loaded (e.g.
-`GiraphGraphComputer` plugin lib directory).
-
-[source,shell]
-export HADOOP_GREMLIN_LIBS=/usr/local/gremlin-console/ext/giraph-gremlin/lib
-
-If using <<gremlin-console,Gremlin Console>>, it is important to install the Hadoop-Gremlin plugin. Note that
-Hadoop-Gremlin requires a Gremlin Console restart after installing.
-
-[source,text]
-----
-$ bin/gremlin.sh
-
- \,,,/
- (o o)
------oOOo-(3)-oOOo-----
-plugin activated: tinkerpop.server
-plugin activated: tinkerpop.utilities
-plugin activated: tinkerpop.tinkergraph
-gremlin> :install org.apache.tinkerpop hadoop-gremlin x.y.z
-==>loaded: [org.apache.tinkerpop, hadoop-gremlin, x.y.z] - restart the console to use [tinkerpop.hadoop]
-gremlin> :q
-$ bin/gremlin.sh
-
- \,,,/
- (o o)
------oOOo-(3)-oOOo-----
-plugin activated: tinkerpop.server
-plugin activated: tinkerpop.utilities
-plugin activated: tinkerpop.tinkergraph
-gremlin> :plugin use tinkerpop.hadoop
-==>tinkerpop.hadoop activated
-gremlin>
-----
-
-Properties Files
-~~~~~~~~~~~~~~~~
-
-`HadoopGraph` makes use of properties files which ultimately get turned into Apache configurations and/or
-Hadoop configurations. The example properties file presented below is located at `conf/hadoop/hadoop-gryo.properties`.
-
-[source,text]
-gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
-gremlin.hadoop.inputLocation=tinkerpop-modern.kryo
-gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
-gremlin.hadoop.outputLocation=output
-gremlin.hadoop.graphOutputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
-gremlin.hadoop.jarsInDistributedCache=true
-####################################
-# Spark Configuration #
-####################################
-spark.master=local[4]
-spark.executor.memory=1g
-spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
-####################################
-# SparkGraphComputer Configuration #
-####################################
-gremlin.spark.graphInputRDD=org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat
-gremlin.spark.graphOutputRDD=org.apache.tinkerpop.gremlin.spark.structure.io.OutputRDDFormat
-gremlin.spark.persistContext=true
-#####################################
-# GiraphGraphComputer Configuration #
-#####################################
-giraph.minWorkers=2
-giraph.maxWorkers=2
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapreduce.map.java.opts=-Xmx1024m
-mapreduce.reduce.java.opts=-Xmx1024m
-giraph.numInputThreads=2
-giraph.numComputeThreads=2
-
-A review of the Hadoop-Gremlin specific properties are provided in the table below. For the respective OLAP
-engines (<<sparkgraphcomputer,`SparkGraphComputer`>> or <<giraphgraphcomputer,`GiraphGraphComputer`>>) refer
-to their respective documentation for configuration options.
-
-[width="100%",cols="2,10",options="header"]
-|=========================================================
-|Property |Description
-|gremlin.graph |The class of the graph to construct using GraphFactory.
-|gremlin.hadoop.inputLocation |The location of the input file(s) for Hadoop-Gremlin to read the graph from.
-|gremlin.hadoop.graphInputFormat |The format that the graph input file(s) are represented in.
-|gremlin.hadoop.outputLocation |The location to write the computed HadoopGraph to.
-|gremlin.hadoop.graphOutputFormat |The format that the output file(s) should be represented in.
-|gremlin.hadoop.jarsInDistributedCache |Whether to upload the Hadoop-Gremlin jars to a distributed cache (necessary if jars are not on the machines' classpaths).
-|=========================================================
-
-
-
-Along with the properties above, the numerous link:http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/core-default.xml[Hadoop specific properties]
-can be added as needed to tune and parameterize the executed Hadoop-Gremlin job on the respective Hadoop cluster.
-
-IMPORTANT: As the size of the graphs being processed becomes large, it is important to fully understand how the
-underlying OLAP engine (e.g. Spark, Giraph, etc.) works and understand the numerous parameterizations offered by
-these systems. Such knowledge can help alleviate out of memory exceptions, slow load times, slow processing times,
-garbage collection issues, etc.
-
-OLTP Hadoop-Gremlin
-~~~~~~~~~~~~~~~~~~~
-
-image:hadoop-pipes.png[width=180,float=left] It is possible to execute OLTP operations over a `HadoopGraph`.
-However, realize that the underlying HDFS files are not random access and thus, to retrieve a vertex, a linear scan
-is required. OLTP operations are useful for peeking into the graph prior to executing a long running OLAP job -- e.g.
-`g.V().valueMap().limit(10)`.
-
-CAUTION: OLTP operations on `HadoopGraph` are not efficient. They require linear scans to execute and are unreasonable
-for large graphs. In such large graph situations, make use of <<traversalvertexprogram,TraversalVertexProgram>>
-which is the OLAP Gremlin machine.
-
-[gremlin-groovy]
-----
-hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo')
-hdfs.ls()
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal()
-g.V().count()
-g.V().out().out().values('name')
-g.V().group().by{it.value('name')[1]}.by('name').next()
-----
-
-OLAP Hadoop-Gremlin
-~~~~~~~~~~~~~~~~~~~
-
-image:hadoop-furnace.png[width=180,float=left] Hadoop-Gremlin was designed to execute OLAP operations via
-`GraphComputer`. The OLTP examples presented previously are reproduced below, but using `TraversalVertexProgram`
-for the execution of the Gremlin traversal.
-
-A `Graph` in TinkerPop3 can support any number of `GraphComputer` implementations. Out of the box, Hadoop-Gremlin
-supports the following three implementations.
-
-* <<mapreducegraphcomputer,`MapReduceGraphComputer`>>: Leverages Hadoop's MapReduce engine to execute TinkerPop3 OLAP
-computations. (*coming soon*)
-** The graph must fit within the total disk space of the Hadoop cluster (supports massive graphs). Message passing is
-coordinated via MapReduce jobs over the on-disk graph (slow traversals).
-* <<sparkgraphcomputer,`SparkGraphComputer`>>: Leverages Apache Spark to execute TinkerPop3 OLAP computations.
-** The graph may fit within the total RAM of the cluster (supports larger graphs). Message passing is coordinated via
-Spark map/reduce/join operations on in-memory and disk-cached data (average speed traversals).
-* <<giraphgraphcomputer,`GiraphGraphComputer`>>: Leverages Apache Giraph to execute TinkerPop3 OLAP computations.
-** The graph should fit within the total RAM of the Hadoop cluster (graph size restriction), though "out-of-core"
-processing is possible. Message passing is coordinated via ZooKeeper for the in-memory graph (speedy traversals).
-
-TIP: image:gremlin-sugar.png[width=50,float=left] For those wanting to use the <<sugar-plugin,SugarPlugin>> with
-their submitted traversal, do `:remote config useSugar true` as well as `:plugin use tinkerpop.sugar` at the start of
-the Gremlin Console session if it is not already activated.
-
-Note that `SparkGraphComputer` and `GiraphGraphComputer` are loaded via their respective plugins. Typically only
-one plugin or the other is loaded depending on the desired `GraphComputer` to use.
-
-[source,text]
-----
-$ bin/gremlin.sh
-
- \,,,/
- (o o)
------oOOo-(3)-oOOo-----
-plugin activated: tinkerpop.server
-plugin activated: tinkerpop.utilities
-plugin activated: tinkerpop.tinkergraph
-plugin activated: tinkerpop.hadoop
-gremlin> :install org.apache.tinkerpop giraph-gremlin x.y.z
-==>loaded: [org.apache.tinkerpop, giraph-gremlin, x.y.z] - restart the console to use [tinkerpop.giraph]
-gremlin> :install org.apache.tinkerpop spark-gremlin x.y.z
-==>loaded: [org.apache.tinkerpop, spark-gremlin, x.y.z] - restart the console to use [tinkerpop.spark]
-gremlin> :q
-$ bin/gremlin.sh
-
- \,,,/
- (o o)
------oOOo-(3)-oOOo-----
-plugin activated: tinkerpop.server
-plugin activated: tinkerpop.utilities
-plugin activated: tinkerpop.tinkergraph
-plugin activated: tinkerpop.hadoop
-gremlin> :plugin use tinkerpop.giraph
-==>tinkerpop.giraph activated
-gremlin> :plugin use tinkerpop.spark
-==>tinkerpop.spark activated
-----
-
-WARNING: Hadoop, Spark, and Giraph all depend on many of the same libraries (e.g. ZooKeeper, Snappy, Netty, Guava,
-etc.). Unfortunately, typically these dependencies are not to the same versions of the respective libraries. As such,
-it is best to *not* have both Spark and Giraph plugins loaded in the same console session nor in the same Java
-project (though intelligent `<exclusion>`-usage can help alleviate conflicts in a Java project).
-
-CAUTION: It is important to note that when doing an OLAP traversal, any resulting vertices, edges, or properties will be
-attached to the source graph. For Hadoop-based graphs, this may lead to linear search times on massive graphs. Thus,
-if vertex, edge, or property objects are to be returns (as a final result), it is best to `.id()` to get the id
-of the object and not the actual attached object.
-
-[[mapreducegraphcomputer]]
-MapReduceGraphComputer
-^^^^^^^^^^^^^^^^^^^^^^
-
-*COMING SOON*
-
-[[sparkgraphcomputer]]
-SparkGraphComputer
-^^^^^^^^^^^^^^^^^^
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>spark-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-----
-
-image:spark-logo.png[width=175,float=left] link:http://spark.apache.org[Spark] is an Apache Software Foundation
-project focused on general-purpose OLAP data processing. Spark provides a hybrid in-memory/disk-based distributed
-computing model that is similar to Hadoop's MapReduce model. Spark maintains a fluent function chaining DSL that is
-arguably easier for developers to work with than native Hadoop MapReduce. Spark-Gremlin provides an implementation of
-the bulk-synchronous parallel, distributed message passing algorithm within Spark and thus, any `VertexProgram` can be
-executed over `SparkGraphComputer`.
-
-If `SparkGraphComputer` will be used as the `GraphComputer` for `HadoopGraph` then its `lib` directory should be
-specified in `HADOOP_GREMLIN_LIBS`.
-
-[source,shell]
-export HADOOP_GREMLIN_LIBS=$HADOOP_GREMLIN_LIBS:/usr/local/gremlin-console/ext/spark-gremlin/lib
-
-Furthermore the `lib/` directory should be distributed across all machines in the SparkServer cluster. For this purpose TinkerPop
-provides a helper script, which takes the Spark installation directory and the the Spark machines as input:
-
-[source,shell]
-bin/init-tp-spark.sh /usr/local/spark spark@10.0.0.1 spark@10.0.0.2 spark@10.0.0.3
-
-Once the `lib/` directory is distributed, `SparkGraphComputer` can be used as follows.
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal(computer(SparkGraphComputer))
-g.V().count()
-g.V().out().out().values('name')
-----
-
-For using lambdas in Gremlin-Groovy, simply provide `:remote connect` a `TraversalSource` which leverages SparkGraphComputer.
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal(computer(SparkGraphComputer))
-:remote connect tinkerpop.hadoop graph g
-:> g.V().group().by{it.value('name')[1]}.by('name')
-----
-
-The `SparkGraphComputer` algorithm leverages Spark's caching abilities to reduce the amount of data shuffled across
-the wire on each iteration of the <<vertexprogram,`VertexProgram`>>. When the graph is loaded as a Spark RDD
-(Resilient Distributed Dataset) it is immediately cached as `graphRDD`. The `graphRDD` is a distributed adjacency
-list which encodes the vertex, its properties, and all its incident edges. On the first iteration, each vertex
-(in parallel) is passed through `VertexProgram.execute()`. This yields an output of the vertex's mutated state
-(i.e. updated compute keys -- `propertyX`) and its outgoing messages. This `viewOutgoingRDD` is then reduced to
-`viewIncomingRDD` where the outgoing messages are sent to their respective vertices. If a `MessageCombiner` exists
-for the vertex program, then messages are aggregated locally and globally to ultimately yield one incoming message
-for the vertex. This reduce sequence is the "message pass." If the vertex program does not terminate on this
-iteration, then the `viewIncomingRDD` is joined with the cached `graphRDD` and the process continues. When there
-are no more iterations, there is a final join and the resultant RDD is stripped of its edges and messages. This
-`mapReduceRDD` is cached and is processed by each <<mapreduce,`MapReduce`>> job in the
-<<graphcomputer,`GraphComputer`>> computation.
-
-image::spark-algorithm.png[width=775]
-
-[width="100%",cols="2,10",options="header"]
-|========================================================
-|Property |Description
-|gremlin.spark.graphInputRDD |A class for creating RDD's from underlying graph data, defaults to Hadoop `InputFormat`.
-|gremlin.spark.graphOutputRDD |A class for output RDD's, defaults to Hadoop `OutputFormat`.
-|gremlin.spark.graphStorageLevel |What `StorageLevel` to use for the cached graph during job execution (default `MEMORY_ONLY`).
-|gremlin.spark.persistContext |Whether to create a new `SparkContext` for every `SparkGraphComputer` or to reuse an existing one.
-|gremlin.spark.persistStorageLevel |What `StorageLevel` to use when persisted RDDs via `PersistedOutputRDD` (default `MEMORY_ONLY`).
-|========================================================
-
-InputRDD and OutputRDD
-++++++++++++++++++++++
-
-If the provider/user does not want to use Hadoop `InputFormats`, it is possible to leverage Spark's RDD
-constructs directly. There is a `gremlin.spark.graphInputRDD` configuration that references a `Class<? extends
-InputRDD>`. An `InputRDD` provides a read method that takes a `SparkContext` and returns a graphRDD. Likewise, use
-`gremlin.spark.graphOutputRDD` and the respective `OutputRDD`.
-
-If the graph system provider uses an `InputRDD`, the RDD should maintain an associated `org.apache.spark.Partitioner`. By doing so,
-`SparkGraphComputer` will not partition the loaded graph across the cluster as it has already been partitioned by the graph system provider.
-This can save a significant amount of time and space resources.
-If the `InputRDD` does not have a registered partitioner, `SparkGraphComputer` will partition the graph using
-a `org.apache.spark.HashPartitioner` with the number of partitions being either the number of existing partitions in the input (e.g. input splits)
-or the user specified number of `GraphComputer.workers()`.
-
-Using a Persisted Context
-+++++++++++++++++++++++++
-
-It is possible to persist the graph RDD between jobs within the `SparkContext` (e.g. SparkServer) by leveraging `PersistedOutputRDD`.
-Note that `gremlin.spark.persistContext` should be set to `true` or else the persisted RDD will be destroyed when the `SparkContext` closes.
-The persisted RDD is named by the `gremlin.hadoop.outputLocation` configuration. Similarly, `PersistedInputRDD` is used with respective
-`gremlin.hadoop.inputLocation` to retrieve the persisted RDD from the `SparkContext`.
-
-When using a persistent `SparkContext` the configuration used by the original Spark Configuration will be inherited by all threaded
-references to that Spark Context. The exception to this rule are those properties which have a specific thread local effect.
-
-.Thread Local Properties
-. spark.jobGroup.id
-. spark.job.description
-. spark.job.interruptOnCancel
-. spark.scheduler.pool
-
-Finally, there is a `spark` object that can be used to manage persisted RDDs (see <<interacting-with-spark, Interacting with Spark>>).
-
-[[bulkdumpervertexprogramusingspark]]
-Exporting with BulkDumperVertexProgram
-++++++++++++++++++++++++++++++++++++++
-
-The <<bulkdumpervertexprogram, BulkDumperVertexProgram>> exports a whole graph in any of the supported Hadoop GraphOutputFormats (`GraphSONOutputFormat`,
-`GryoOutputFormat` or `ScriptOutputFormat`). The example below takes a Hadoop graph as the input (in `GryoInputFormat`) and exports it as a GraphSON file
-(`GraphSONOutputFormat`).
-
-[gremlin-groovy]
-----
-hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo')
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-graph.configuration().setProperty('gremlin.hadoop.graphOutputFormat', 'org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat')
-graph.compute(SparkGraphComputer).program(BulkDumperVertexProgram.build().create()).submit().get()
-hdfs.ls('output')
-hdfs.head('output/~g')
-----
-
-Loading with BulkLoaderVertexProgram
-++++++++++++++++++++++++++++++++++++
-
-The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load large
-amounts of data to and from different `Graph` implementations. The following code demonstrates how to load the
-Grateful Dead graph from HadoopGraph into TinkerGraph over Spark:
-
-[gremlin-groovy]
-----
-hdfs.copyFromLocal('data/grateful-dead.kryo', 'grateful-dead.kryo')
-readGraph = GraphFactory.open('conf/hadoop/hadoop-grateful-gryo.properties')
-writeGraph = 'conf/tinkergraph-gryo.properties'
-blvp = BulkLoaderVertexProgram.build().
- keepOriginalIds(false).
- writeGraph(writeGraph).create(readGraph)
-readGraph.compute(SparkGraphComputer).workers(1).program(blvp).submit().get()
-:set max-iteration 10
-graph = GraphFactory.open(writeGraph)
-g = graph.traversal()
-g.V().valueMap()
-graph.close()
-----
-
-[source,properties]
-----
-# hadoop-grateful-gryo.properties
-
-#
-# Hadoop Graph Configuration
-#
-gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
-gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
-gremlin.hadoop.inputLocation=grateful-dead.kryo
-gremlin.hadoop.outputLocation=output
-gremlin.hadoop.jarsInDistributedCache=true
-
-#
-# SparkGraphComputer Configuration
-#
-spark.master=local[1]
-spark.executor.memory=1g
-spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
-----
-
-[source,properties]
-----
-# tinkergraph-gryo.properties
-
-gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
-gremlin.tinkergraph.graphFormat=gryo
-gremlin.tinkergraph.graphLocation=/tmp/tinkergraph.kryo
-----
-
-IMPORTANT: The path to TinkerGraph jars needs to be included in the `HADOOP_GREMLIN_LIBS` for the above example to work.
-
-[[giraphgraphcomputer]]
-GiraphGraphComputer
-^^^^^^^^^^^^^^^^^^^
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>giraph-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-----
-
-image:giraph-logo.png[width=100,float=left] link:http://giraph.apache.org[Giraph] is an Apache Software Foundation
-project focused on OLAP-based graph processing. Giraph makes use of the distributed graph computing paradigm made
-popular by Google's Pregel. In Giraph, developers write "vertex programs" that get executed at each vertex in
-parallel. These programs communicate with one another in a bulk synchronous parallel (BSP) manner. This model aligns
-with TinkerPop3's `GraphComputer` API. TinkerPop3 provides an implementation of `GraphComputer` that works for Giraph
-called `GiraphGraphComputer`. Moreover, with TinkerPop3's <<mapreduce,MapReduce>>-framework, the standard
-Giraph/Pregel model is extended to support an arbitrary number of MapReduce phases to aggregate and yield results
-from the graph. Below are examples using `GiraphGraphComputer` from the <<gremlin-console,Gremlin-Console>>.
-
-WARNING: Giraph uses a large number of Hadoop counters. The default for Hadoop is 120. In `mapred-site.xml` it is
-possible to increase the limit it via the `mapreduce.job.counters.max` property. A good value to use is 1000. This
-is a cluster-wide property so be sure to restart the cluster after updating.
-
-WARNING: The maximum number of workers can be no larger than the number of map-slots in the Hadoop cluster minus 1.
-For example, if the Hadoop cluster has 4 map slots, then `giraph.maxWorkers` can not be larger than 3. One map-slot
-is reserved for the master compute node and all other slots can be allocated as workers to execute the VertexPrograms
-on the vertices of the graph.
-
-If `GiraphGraphComputer` will be used as the `GraphComputer` for `HadoopGraph` then its `lib` directory should be
-specified in `HADOOP_GREMLIN_LIBS`.
-
-[source,shell]
-export HADOOP_GREMLIN_LIBS=$HADOOP_GREMLIN_LIBS:/usr/local/gremlin-console/ext/giraph-gremlin/lib
-
-Or, the user can specify the directory in the Gremlin Console.
-
-[source,groovy]
-System.setProperty('HADOOP_GREMLIN_LIBS',System.getProperty('HADOOP_GREMLIN_LIBS') + ':' + '/usr/local/gremlin-console/ext/giraph-gremlin/lib')
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal(computer(GiraphGraphComputer))
-g.V().count()
-g.V().out().out().values('name')
-----
-
-IMPORTANT: The examples above do not use lambdas (i.e. closures in Gremlin-Groovy). This makes the traversal
-serializable and thus, able to be distributed to all machines in the Hadoop cluster. If a lambda is required in a
-traversal, then the traversal must be sent as a `String` and compiled locally at each machine in the cluster. The
-following example demonstrates the `:remote` command which allows for submitting Gremlin traversals as a `String`.
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal(computer(GiraphGraphComputer))
-:remote connect tinkerpop.hadoop graph g
-:> g.V().group().by{it.value('name')[1]}.by('name')
-result
-result.memory.runtime
-result.memory.keys()
-result.memory.get('~reducing')
-----
-
-NOTE: If the user explicitly specifies `giraph.maxWorkers` and/or `giraph.numComputeThreads` in the configuration,
-then these values will be used by Giraph. However, if these are not specified and the user never calls
-`GraphComputer.workers()` then `GiraphGraphComputer` will try to compute the number of workers/threads to use based
-on the cluster's profile.
-
-Loading with BulkLoaderVertexProgram
-++++++++++++++++++++++++++++++++++++
-
-The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load
-large amounts of data to and from different `Graph` implementations. The following code demonstrates how to load
-the Grateful Dead graph from HadoopGraph into TinkerGraph over Giraph:
-
-[gremlin-groovy]
-----
-hdfs.copyFromLocal('data/grateful-dead.kryo', 'grateful-dead.kryo')
-readGraph = GraphFactory.open('conf/hadoop/hadoop-grateful-gryo.properties')
-writeGraph = 'conf/tinkergraph-gryo.properties'
-blvp = BulkLoaderVertexProgram.build().
- keepOriginalIds(false).
- writeGraph(writeGraph).create(readGraph)
-readGraph.compute(GiraphGraphComputer).workers(1).program(blvp).submit().get()
-:set max-iteration 10
-graph = GraphFactory.open(writeGraph)
-g = graph.traversal()
-g.V().valueMap()
-graph.close()
-----
-
-[source,properties]
-----
-# hadoop-grateful-gryo.properties
-
-#
-# Hadoop Graph Configuration
-#
-gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
-gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
-gremlin.hadoop.graphOutputFormat=org.apache.hadoop.mapreduce.lib.output.NullOutputFormat
-gremlin.hadoop.inputLocation=grateful-dead.kryo
-gremlin.hadoop.outputLocation=output
-gremlin.hadoop.jarsInDistributedCache=true
-
-#
-# GiraphGraphComputer Configuration
-#
-giraph.minWorkers=1
-giraph.maxWorkers=1
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapred.map.child.java.opts=-Xmx1024m
-mapred.reduce.child.java.opts=-Xmx1024m
-giraph.numInputThreads=4
-giraph.numComputeThreads=4
-giraph.maxMessagesInMemory=100000
-----
-
-[source,properties]
-----
-# tinkergraph-gryo.properties
-
-gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
-gremlin.tinkergraph.graphFormat=gryo
-gremlin.tinkergraph.graphLocation=/tmp/tinkergraph.kryo
-----
-
-NOTE: The path to TinkerGraph needs to be included in the `HADOOP_GREMLIN_LIBS` for the above example to work.
-
-Input/Output Formats
-~~~~~~~~~~~~~~~~~~~~
-
-image:adjacency-list.png[width=300,float=right] Hadoop-Gremlin provides various I/O formats -- i.e. Hadoop
-`InputFormat` and `OutputFormat`. All of the formats make use of an link:http://en.wikipedia.org/wiki/Adjacency_list[adjacency list]
-representation of the graph where each "row" represents a single vertex, its properties, and its incoming and
-outgoing edges.
-
-{empty} +
-
-[[gryo-io-format]]
-Gryo I/O Format
-^^^^^^^^^^^^^^^
-
-* **InputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat`
-* **OutputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat`
-
-<<gryo-reader-writer,Gryo>> is a binary graph format that leverages link:https://github.com/EsotericSoftware/kryo[Kryo]
-to make a compact, binary representation of a vertex. It is recommended that users leverage Gryo given its space/time
-savings over text-based representations.
-
-NOTE: The `GryoInputFormat` is splittable.
-
-[[graphson-io-format]]
-GraphSON I/O Format
-^^^^^^^^^^^^^^^^^^^
-
-* **InputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat`
-* **OutputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat`
-
-<<graphson-reader-writer,GraphSON>> is a JSON based graph format. GraphSON is a space-expensive graph format in that
-it is a text-based markup language. However, it is convenient for many developers to work with as its structure is
-simple (easy to create and parse).
-
-The data below represents an adjacency list representation of the classic TinkerGraph toy graph in GraphSON format.
-
-[source,json]
-----
-{"id":1,"label":"person","outE":{"created":[{"id":9,"inV":3,"properties":{"weight":0.4}}],"knows":[{"id":7,"inV":2,"properties":{"weight":0.5}},{"id":8,"inV":4,"properties":{"weight":1.0}}]},"properties":{"name":[{"id":0,"value":"marko"}],"age":[{"id":1,"value":29}]}}
-{"id":2,"label":"person","inE":{"knows":[{"id":7,"outV":1,"properties":{"weight":0.5}}]},"properties":{"name":[{"id":2,"value":"vadas"}],"age":[{"id":3,"value":27}]}}
-{"id":3,"label":"software","inE":{"created":[{"id":9,"outV":1,"properties":{"weight":0.4}},{"id":11,"outV":4,"properties":{"weight":0.4}},{"id":12,"outV":6,"properties":{"weight":0.2}}]},"properties":{"name":[{"id":4,"value":"lop"}],"lang":[{"id":5,"value":"java"}]}}
-{"id":4,"label":"person","inE":{"knows":[{"id":8,"outV":1,"properties":{"weight":1.0}}]},"outE":{"created":[{"id":10,"inV":5,"properties":{"weight":1.0}},{"id":11,"inV":3,"properties":{"weight":0.4}}]},"properties":{"name":[{"id":6,"value":"josh"}],"age":[{"id":7,"value":32}]}}
-{"id":5,"label":"software","inE":{"created":[{"id":10,"outV":4,"properties":{"weight":1.0}}]},"properties":{"name":[{"id":8,"value":"ripple"}],"lang":[{"id":9,"value":"java"}]}}
-{"id":6,"label":"person","outE":{"created":[{"id":12,"inV":3,"properties":{"weight":0.2}}]},"properties":{"name":[{"id":10,"value":"peter"}],"age":[{"id":11,"value":35}]}}
-----
-
-[[script-io-format]]
-Script I/O Format
-^^^^^^^^^^^^^^^^^
-
-* **InputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptInputFormat`
-* **OutputFormat**: `org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptOutputFormat`
-
-`ScriptInputFormat` and `ScriptOutputFormat` take an arbitrary script and use that script to either read or write
-`Vertex` objects, respectively. This can be considered the most general `InputFormat`/`OutputFormat` possible in that
-Hadoop-Gremlin uses the user provided script for all reading/writing.
-
-ScriptInputFormat
-+++++++++++++++++
-
-The data below represents an adjacency list representation of the classic TinkerGraph toy graph. First line reads,
-"vertex `1`, labeled `person` having 2 property values (`marko` and `29`) has 3 outgoing edges; the first edge is
-labeled `knows`, connects the current vertex `1` with vertex `2` and has a property value `0.4`, and so on."
-
-[source]
-1:person:marko:29 knows:2:0.5,knows:4:1.0,created:3:0.4
-2:person:vadas:27
-3:project:lop:java
-4:person:josh:32 created:3:0.4,created:5:1.0
-5:project:ripple:java
-6:person:peter:35 created:3:0.2
-
-There is no corresponding `InputFormat` that can parse this particular file (or some adjacency list variant of it).
-As such, `ScriptInputFormat` can be used. With `ScriptInputFormat` a script is stored in HDFS and leveraged by each
-mapper in the Hadoop job. The script must have the following method defined:
-
-[source,groovy]
-def parse(String line, ScriptElementFactory factory) { ... }
-
-`ScriptElementFactory` is a legacy from previous versions and, although it's still functional, it should no longer be used.
-In order to create vertices and edges, the `parse()` method gets access to a global variable named `graph`, which holds
-the local `StarGraph` for the current line/vertex.
-
-An appropriate `parse()` for the above adjacency list file is:
-
-[source,groovy]
-def parse(line, factory) {
- def parts = line.split(/ /)
- def (id, label, name, x) = parts[0].split(/:/).toList()
- def v1 = graph.addVertex(T.id, id, T.label, label)
- if (name != null) v1.property('name', name) // first value is always the name
- if (x != null) {
- // second value depends on the vertex label; it's either
- // the age of a person or the language of a project
- if (label.equals('project')) v1.property('lang', x)
- else v1.property('age', Integer.valueOf(x))
- }
- if (parts.length == 2) {
- parts[1].split(/,/).grep { !it.isEmpty() }.each {
- def (eLabel, refId, weight) = it.split(/:/).toList()
- def v2 = graph.addVertex(T.id, refId)
- v1.addOutEdge(eLabel, v2, 'weight', Double.valueOf(weight))
- }
- }
- return v1
-}
-
-The resultant `Vertex` denotes whether the line parsed yielded a valid Vertex. As such, if the line is not valid
-(e.g. a comment line, a skip line, etc.), then simply return `null`.
-
-ScriptOutputFormat Support
-++++++++++++++++++++++++++
-
-The principle above can also be used to convert a vertex to an arbitrary `String` representation that is ultimately
-streamed back to a file in HDFS. This is the role of `ScriptOutputFormat`. `ScriptOutputFormat` requires that the
-provided script maintains a method with the following signature:
-
-[source,groovy]
-def stringify(Vertex vertex) { ... }
-
-An appropriate `stringify()` to produce output in the same format that was shown in the `ScriptInputFormat` sample is:
-
-[source,groovy]
-def stringify(vertex) {
- def v = vertex.values('name', 'age', 'lang').inject(vertex.id(), vertex.label()).join(':')
- def outE = vertex.outE().map {
- def e = it.get()
- e.values('weight').inject(e.label(), e.inV().next().id()).join(':')
- }.join(',')
- return [v, outE].join('\t')
-}
-
-
-
-Storage Systems
-~~~~~~~~~~~~~~~
-
-Hadoop-Gremlin provides two implementations of the `Storage` API:
-
-* `FileSystemStorage`: Access HDFS and local file system data.
-* `SparkContextStorage`: Access Spark persisted RDD data.
-
-[[interacting-with-hdfs]]
-Interacting with HDFS
-^^^^^^^^^^^^^^^^^^^^^
-
-The distributed file system of Hadoop is called link:http://en.wikipedia.org/wiki/Apache_Hadoop#Hadoop_distributed_file_system[HDFS].
-The results of any OLAP operation are stored in HDFS accessible via `hdfs`. For local file system access, there is `local`.
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-graph.compute(SparkGraphComputer).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey('clusterCount').create()).submit().get();
-hdfs.ls()
-hdfs.ls('output')
-hdfs.head('output', GryoInputFormat)
-hdfs.head('output', 'clusterCount', SequenceFileInputFormat)
-hdfs.rm('output')
-hdfs.ls()
-----
-
-[[interacting-with-spark]]
-Interacting with Spark
-^^^^^^^^^^^^^^^^^^^^^^
-
-If a Spark context is persisted, then Spark RDDs will remain the Spark cache and accessible over subsequent jobs.
-RDDs are retrieved and saved to the `SparkContext` via `PersistedInputRDD` and `PersistedOutputRDD` respectivly.
-Persisted RDDs can be accessed using `spark`.
-
-[gremlin-groovy]
-----
-Spark.create('local[4]')
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-graph.configuration().setProperty('gremlin.spark.graphOutputRDD', PersistedOutputRDD.class.getCanonicalName())
-graph.configuration().clearProperty('gremlin.hadoop.graphOutputFormat')
-graph.configuration().setProperty('gremlin.spark.persistContext',true)
-graph.compute(SparkGraphComputer).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey('clusterCount').create()).submit().get();
-spark.ls()
-spark.ls('output')
-spark.head('output', PersistedInputRDD)
-spark.head('output', 'clusterCount', PersistedInputRDD)
-spark.rm('output')
-spark.ls()
-----
-
-A Command Line Example
-~~~~~~~~~~~~~~~~~~~~~~
-
-image::pagerank-logo.png[width=300]
-
-The classic link:http://en.wikipedia.org/wiki/PageRank[PageRank] centrality algorithm can be executed over the
-TinkerPop graph from the command line using `GiraphGraphComputer`.
-
-WARNING: Be sure that the `HADOOP_GREMLIN_LIBS` references the location `lib` directory of the respective
-`GraphComputer` engine being used or else the requisite dependencies will not be uploaded to the Hadoop cluster.
-
-[source,text]
-----
-$ hdfs dfs -copyFromLocal data/tinkerpop-modern.json tinkerpop-modern.json
-$ hdfs dfs -ls
-Found 2 items
--rw-r--r-- 1 marko supergroup 2356 2014-07-28 13:00 /user/marko/tinkerpop-modern.json
-$ hadoop jar target/giraph-gremlin-x.y.z-job.jar org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer ../hadoop-gremlin/conf/hadoop-graphson.properties
-15/09/11 08:02:08 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
-15/09/11 08:02:11 INFO computer.GiraphGraphComputer: HadoopGremlin(Giraph): PageRankVertexProgram[alpha=0.85,iterations=30]
-15/09/11 08:02:12 INFO mapreduce.JobSubmitter: number of splits:3
-15/09/11 08:02:12 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1441915907347_0028
-15/09/11 08:02:12 INFO impl.YarnClientImpl: Submitted application application_1441915907347_0028
-15/09/11 08:02:12 INFO job.GiraphJob: Tracking URL: http://markos-macbook:8088/proxy/application_1441915907347_0028/
-15/09/11 08:02:12 INFO job.GiraphJob: Waiting for resources... Job will start only when it gets all 3 mappers
-15/09/11 08:03:54 INFO mapreduce.Job: Running job: job_1441915907347_0028
-15/09/11 08:03:55 INFO mapreduce.Job: Job job_1441915907347_0028 running in uber mode : false
-15/09/11 08:03:55 INFO mapreduce.Job: map 33% reduce 0%
-15/09/11 08:03:57 INFO mapreduce.Job: map 67% reduce 0%
-15/09/11 08:04:01 INFO mapreduce.Job: map 100% reduce 0%
-15/09/11 08:06:17 INFO mapreduce.Job: Job job_1441915907347_0028 completed successfully
-15/09/11 08:06:17 INFO mapreduce.Job: Counters: 80
- File System Counters
- FILE: Number of bytes read=0
- FILE: Number of bytes written=483918
- FILE: Number of read operations=0
- FILE: Number of large read operations=0
- FILE: Number of write operations=0
- HDFS: Number of bytes read=1465
- HDFS: Number of bytes written=1760
- HDFS: Number of read operations=39
- HDFS: Number of large read operations=0
- HDFS: Number of write operations=20
- Job Counters
- Launched map tasks=3
- Other local map tasks=3
- Total time spent by all maps in occupied slots (ms)=458105
- Total time spent by all reduces in occupied slots (ms)=0
- Total time spent by all map tasks (ms)=458105
- Total vcore-seconds taken by all map tasks=458105
- Total megabyte-seconds taken by all map tasks=469099520
- Map-Reduce Framework
- Map input records=3
- Map output records=0
- Input split bytes=132
- Spilled Records=0
- Failed Shuffles=0
- Merged Map outputs=0
- GC time elapsed (ms)=1594
- CPU time spent (ms)=0
- Physical memory (bytes) snapshot=0
- Virtual memory (bytes) snapshot=0
- Total committed heap usage (bytes)=527958016
- Giraph Stats
- Aggregate edges=0
- Aggregate finished vertices=0
- Aggregate sent message message bytes=13535
- Aggregate sent messages=186
- Aggregate vertices=6
- Current master task partition=0
- Current workers=2
- Last checkpointed superstep=0
- Sent message bytes=438
- Sent messages=6
- Superstep=31
- Giraph Timers
- Initialize (ms)=2996
- Input superstep (ms)=5209
- Setup (ms)=59
- Shutdown (ms)=9324
- Superstep 0 GiraphComputation (ms)=3861
- Superstep 1 GiraphComputation (ms)=4027
- Superstep 10 GiraphComputation (ms)=4000
- Superstep 11 GiraphComputation (ms)=4004
- Superstep 12 GiraphComputation (ms)=3999
- Superstep 13 GiraphComputation (ms)=4000
- Superstep 14 GiraphComputation (ms)=4005
- Superstep 15 GiraphComputation (ms)=4003
- Superstep 16 GiraphComputation (ms)=4001
- Superstep 17 GiraphComputation (ms)=4007
- Superstep 18 GiraphComputation (ms)=3998
- Superstep 19 GiraphComputation (ms)=4006
- Superstep 2 GiraphComputation (ms)=4007
- Superstep 20 GiraphComputation (ms)=3996
- Superstep 21 GiraphComputation (ms)=4006
- Superstep 22 GiraphComputation (ms)=4002
- Superstep 23 GiraphComputation (ms)=3998
- Superstep 24 GiraphComputation (ms)=4003
- Superstep 25 GiraphComputation (ms)=4001
- Superstep 26 GiraphComputation (ms)=4003
- Superstep 27 GiraphComputation (ms)=4005
- Superstep 28 GiraphComputation (ms)=4002
- Superstep 29 GiraphComputation (ms)=4001
- Superstep 3 GiraphComputation (ms)=3988
- Superstep 30 GiraphComputation (ms)=4248
- Superstep 4 GiraphComputation (ms)=4010
- Superstep 5 GiraphComputation (ms)=3998
- Superstep 6 GiraphComputation (ms)=3996
- Superstep 7 GiraphComputation
<TRUNCATED>
[15/43] incubator-tinkerpop git commit: Downloading hadoop
winutils.exe when using windows
Posted by sp...@apache.org.
Downloading hadoop winutils.exe when using windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a6cfc3b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a6cfc3b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a6cfc3b8
Branch: refs/heads/TINKERPOP-1107
Commit: a6cfc3b822dc00107375f175003fb9379201b21d
Parents: daa6379
Author: Marvin Froeder <ma...@vizexplorer.com>
Authored: Mon Feb 1 17:19:55 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 09:30:54 2016 +1300
----------------------------------------------------------------------
pom.xml | 40 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a6cfc3b8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 62eb81c..311ddad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1001,5 +1001,45 @@ limitations under the License.
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>windows</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <os>
+ <family>windows</family>
+ </os>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>wagon-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>download-hadoop-winutils</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>download-single</goal>
+ </goals>
+ <configuration>
+ <url>http://public-repo-1.hortonworks.com/</url>
+ <fromFile>hdp-win-alpha/winutils.exe</fromFile>
+ <toDir>${project.build.directory}/hadoop_home/bin</toDir>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <argLine>-Dlog4j.configuration=${log4j-test.properties} -Dbuild.dir=${project.build.directory} -Dhadoop.home.dir=${project.build.directory}/hadoop_home</argLine>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
[23/43] incubator-tinkerpop git commit: Installing maven using choco
Posted by sp...@apache.org.
Installing maven using choco
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bc58b8ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bc58b8ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bc58b8ba
Branch: refs/heads/TINKERPOP-1107
Commit: bc58b8baa3debec17a076469894c0074afbaad1b
Parents: 884e45d
Author: Marvin Froeder <ve...@gmail.com>
Authored: Mon Feb 15 16:45:40 2016 +1300
Committer: Marvin Froeder <ma...@vizexplorer.com>
Committed: Mon Feb 22 11:29:00 2016 +1300
----------------------------------------------------------------------
appveyor.yml | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bc58b8ba/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index e31d2ea..a99752a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -7,17 +7,8 @@ environment:
os: Windows Server 2012
install:
- - ps: |
- Add-Type -AssemblyName System.IO.Compression.FileSystem
- if (!(Test-Path -Path "C:\maven" )) {
- (new-object System.Net.WebClient).DownloadFile('https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip', 'C:\maven-bin.zip')
- [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\maven-bin.zip", "C:\maven")
- }
- - cmd: SET PATH=C:\maven\apache-maven-3.3.9\bin;%JAVA_HOME%\bin;%PATH%
- - cmd: SET MAVEN_OPTS=-XX:MaxPermSize=2g -Xmx4g
- - cmd: SET JAVA_OPTS=-XX:MaxPermSize=2g -Xmx4g
- - cmd: mvn --version
- - cmd: java -version
+ - cmd: choco install maven -y -f
+ - cmd: refreshenv
build_script:
- mvn clean install -B -Dci
[12/43] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1168'
into tp31
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-1168' into tp31
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/26f81b99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/26f81b99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/26f81b99
Branch: refs/heads/TINKERPOP-1107
Commit: 26f81b991b7fed66180949fabcff57f5c93ab6d1
Parents: 23e90f8 606c68b
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Fri Feb 19 19:38:12 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Fri Feb 19 19:38:12 2016 +0100
----------------------------------------------------------------------
docs/preprocessor/preprocess-file.sh | 29 +-
docs/preprocessor/preprocess.sh | 2 +-
.../reference/implementations-hadoop.asciidoc | 929 +++++++++
.../reference/implementations-intro.asciidoc | 545 ++++++
.../reference/implementations-neo4j.asciidoc | 261 +++
.../implementations-tinkergraph.asciidoc | 144 ++
docs/src/reference/implementations.asciidoc | 1835 ------------------
docs/src/reference/index.asciidoc | 5 +-
8 files changed, 1912 insertions(+), 1838 deletions(-)
----------------------------------------------------------------------