You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/11/07 15:33:33 UTC
[09/15] incubator-tinkerpop git commit: Merge remote-tracking branch
'tinkerpop/master' into TP-885
Merge remote-tracking branch 'tinkerpop/master' into TP-885
Conflicts:
docs/src/upgrade-release-3.1.x-incubating.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/7ce67844
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/7ce67844
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/7ce67844
Branch: refs/heads/TINKERPOP3-885
Commit: 7ce6784466e8714edac465f2bbfee6dda5c019dd
Parents: 956d34d 93573ed
Author: Dylan Millikin <dy...@brightzone.fr>
Authored: Fri Nov 6 22:40:57 2015 +0100
Committer: Dylan Millikin <dy...@brightzone.fr>
Committed: Fri Nov 6 22:44:37 2015 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 44 +-
CONTRIBUTING.asciidoc | 236 +-----
RELEASE.asciidoc | 172 -----
docs/src/acknowledgements.asciidoc | 19 +-
docs/src/conclusion.asciidoc | 6 +-
docs/src/developer-administration.asciidoc | 91 +++
docs/src/developer-contributing.asciidoc | 308 ++++++++
docs/src/developer-meetings.asciidoc | 64 ++
docs/src/developer-release.asciidoc | 171 +++++
docs/src/developer.asciidoc | 33 +
docs/src/gremlin-applications.asciidoc | 721 +++++++++++++++----
docs/src/implementations.asciidoc | 562 +++++++++++----
docs/src/intro.asciidoc | 173 ++++-
docs/src/preface.asciidoc | 42 +-
docs/src/the-graph.asciidoc | 245 +++++--
docs/src/the-graphcomputer.asciidoc | 201 +++++-
docs/src/the-traversal.asciidoc | 666 ++++++++++++-----
.../upgrade-release-3.0.x-incubating.asciidoc | 4 +-
.../upgrade-release-3.1.x-incubating.asciidoc | 113 ++-
docs/static/images/gremlin-gym.pdf | Bin 0 -> 963978 bytes
docs/static/images/gremlin-gym.png | Bin 0 -> 273111 bytes
docs/static/images/gremlin-standing-strong.png | Bin 0 -> 30985 bytes
.../images/keep-on-traversin-tiedye-title.png | Bin 0 -> 943206 bytes
docs/static/images/keep-on-traversin-tiedye.png | Bin 0 -> 581914 bytes
docs/static/images/keep-on-traversin-title.png | Bin 0 -> 669529 bytes
docs/static/images/keep-on-traversin.png | Bin 0 -> 400941 bytes
docs/static/images/not-tinkerpop3-gremlin.png | Bin 0 -> 186676 bytes
docs/static/images/provider-integration.png | Bin 372429 -> 226464 bytes
docs/static/images/quantum-gremlin.png | Bin 0 -> 69142 bytes
docs/static/images/tinkerpop3.graffle | 236 +++---
.../process/computer/GiraphGraphComputer.java | 16 +-
.../computer/io/GiraphVertexInputFormat.java | 70 --
.../computer/io/GiraphVertexOutputFormat.java | 65 --
.../process/computer/io/GiraphVertexReader.java | 67 --
.../process/computer/io/GiraphVertexWriter.java | 57 --
.../structure/io/GiraphVertexInputFormat.java | 70 ++
.../structure/io/GiraphVertexOutputFormat.java | 65 ++
.../giraph/structure/io/GiraphVertexReader.java | 67 ++
.../giraph/structure/io/GiraphVertexWriter.java | 57 ++
.../GephiTraversalVisualizationStrategy.groovy | 2 +-
.../gremlin/process/computer/GraphComputer.java | 19 +-
.../computer/bulkloading/BulkLoader.java | 31 +
.../bulkloading/BulkLoaderVertexProgram.java | 115 ++-
.../bulkloading/IncrementalBulkLoader.java | 14 +-
.../traversal/TraversalVertexProgram.java | 4 +-
.../gremlin/process/traversal/NumberHelper.java | 288 ++++++++
.../gremlin/process/traversal/Operator.java | 78 +-
.../traversal/dsl/graph/GraphTraversal.java | 23 +-
.../dsl/graph/GraphTraversalSource.java | 10 +-
.../gremlin/process/traversal/dsl/graph/__.java | 9 +-
.../process/traversal/lambda/LoopTraversal.java | 4 +
.../traversal/step/branch/RepeatStep.java | 8 +
.../process/traversal/step/map/GraphStep.java | 149 ++++
.../process/traversal/step/map/MatchStep.java | 46 +-
.../traversal/step/map/MaxGlobalStep.java | 14 +-
.../traversal/step/map/MaxLocalStep.java | 5 +-
.../traversal/step/map/MeanGlobalStep.java | 61 +-
.../traversal/step/map/MeanLocalStep.java | 11 +-
.../traversal/step/map/MinGlobalStep.java | 14 +-
.../traversal/step/map/MinLocalStep.java | 5 +-
.../traversal/step/map/SumGlobalStep.java | 25 +-
.../traversal/step/map/SumLocalStep.java | 5 +-
.../traversal/step/sideEffect/GraphStep.java | 101 ---
.../step/util/TraversalComparator.java | 5 +-
.../strategy/decoration/ConnectiveStrategy.java | 3 +-
.../strategy/decoration/ElementIdStrategy.java | 2 +-
.../strategy/decoration/PartitionStrategy.java | 2 +-
.../strategy/decoration/SubgraphStrategy.java | 2 +-
.../finalization/LazyBarrierStrategy.java | 2 +-
.../ComputerVerificationStrategy.java | 39 +-
.../process/traversal/util/TraversalHelper.java | 3 +
.../structure/util/AbstractTransaction.java | 2 -
.../traversal/OperatorExceptionTest.java | 39 +-
.../gremlin/process/traversal/OperatorTest.java | 100 ++-
.../traversal/step/map/GraphStepTest.java | 41 ++
.../traversal/step/map/MatchStepTest.java | 82 ++-
.../traversal/step/map/MeanGlobalStepTest.java | 14 +
.../traversal/step/map/MeanLocalStepTest.java | 14 +
.../ElementIdStrategyTraverseTest.java | 4 +-
.../PartitionStrategyTraverseTest.java | 4 +-
.../SubgraphStrategyTraverseTest.java | 5 +-
.../apache/tinkerpop/gremlin/driver/Client.java | 71 +-
.../apache/tinkerpop/gremlin/driver/Tokens.java | 6 +
.../driver/ser/GryoMessageSerializerV1d0.java | 13 +-
.../gremlin/driver/AbstractResultQueueTest.java | 23 +
.../gremlin/driver/ResultQueueTest.java | 15 +-
.../ser/GryoMessageSerializerV1d0Test.java | 55 ++
.../traversal/step/map/GroovyGraphTest.groovy | 50 ++
.../traversal/step/map/GroovyOrderTest.groovy | 10 +
.../step/sideEffect/GroovyGroupTest.groovy | 2 +-
.../step/sideEffect/GroovySackTest.groovy | 4 +-
.../process/GroovyProcessComputerSuite.java | 2 +
.../process/GroovyProcessStandardSuite.java | 2 +
...linGroovyScriptEngineTimedInterruptTest.java | 12 +-
.../groovy/util/DependencyGrabberTest.java | 8 +-
gremlin-server/pom.xml | 25 +
.../tinkerpop/gremlin/server/GremlinServer.java | 53 +-
.../tinkerpop/gremlin/server/Settings.java | 5 +
.../handler/HttpGremlinEndpointHandler.java | 47 +-
.../handler/NioGremlinBinaryRequestDecoder.java | 5 +
.../handler/WsGremlinBinaryRequestDecoder.java | 5 +
.../handler/WsGremlinCloseRequestDecoder.java | 5 +
.../handler/WsGremlinTextRequestDecoder.java | 5 +
.../server/op/AbstractEvalOpProcessor.java | 2 +-
.../server/op/session/SessionOpProcessor.java | 9 +-
.../server/op/standard/StandardOpProcessor.java | 20 +-
.../AbstractGremlinServerIntegrationTest.java | 7 +-
.../server/GremlinDriverIntegrateTest.java | 101 ++-
.../server/GremlinServerAuthIntegrateTest.java | 35 +
.../server/GremlinServerHttpIntegrateTest.java | 38 +-
.../server/GremlinServerIntegrateTest.java | 125 ++++
.../gremlin/process/ProcessComputerSuite.java | 2 +
.../gremlin/process/ProcessStandardSuite.java | 2 +
.../process/computer/GraphComputerTest.java | 34 +-
.../traversal/step/map/AddVertexTest.java | 2 +-
.../process/traversal/step/map/GraphTest.java | 109 +++
.../process/traversal/step/map/OrderTest.java | 58 +-
.../process/traversal/step/map/SumTest.java | 2 +-
.../traversal/step/sideEffect/GroupTest.java | 22 +-
.../traversal/step/sideEffect/SackTest.java | 20 +-
.../PartitionStrategyProcessTest.java | 13 +-
...ComputerVerificationStrategyProcessTest.java | 3 +-
.../conf/hadoop-grateful-gryo.properties | 2 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 1 -
.../structure/hdfs/HadoopElementIterator.java | 4 +-
.../hadoop/structure/io/HadoopPools.java | 2 +-
.../hadoop/structure/io/InputOutputHelper.java | 22 +
.../hadoop/structure/util/HadoopHelper.java | 50 --
.../groovy/plugin/HadoopGremlinPluginTest.java | 2 +-
neo4j-gremlin/pom.xml | 28 +-
.../step/sideEffect/Neo4jGraphStep.java | 10 +-
.../optimization/Neo4jGraphStepStrategy.java | 13 +-
.../gremlin/neo4j/structure/Neo4jGraph.java | 1 +
.../gremlin/neo4j/structure/Neo4jHelper.java | 16 +-
.../traversal/strategy/Neo4jStrategySuite.java | 44 ++
.../traversal/strategy/Neo4jStrategyTest.java | 32 +
.../Neo4jGraphStepStrategyTest.java | 76 ++
pom.xml | 34 +-
.../process/computer/SparkGraphComputer.java | 102 ++-
.../process/computer/io/InputFormatRDD.java | 47 --
.../spark/process/computer/io/InputRDD.java | 41 --
.../process/computer/io/OutputFormatRDD.java | 49 --
.../spark/process/computer/io/OutputRDD.java | 31 -
.../spark/structure/io/InputFormatRDD.java | 47 ++
.../spark/structure/io/InputOutputHelper.java | 81 +++
.../gremlin/spark/structure/io/InputRDD.java | 41 ++
.../spark/structure/io/OutputFormatRDD.java | 49 ++
.../gremlin/spark/structure/io/OutputRDD.java | 31 +
.../spark/structure/io/PersistedInputRDD.java | 60 ++
.../spark/structure/io/PersistedOutputRDD.java | 41 ++
.../process/computer/LocalPropertyTest.java | 100 +++
.../process/computer/io/ExampleInputRDD.java | 47 --
.../process/computer/io/ExampleOutputRDD.java | 45 --
.../process/computer/io/InputOutputRDDTest.java | 59 --
.../spark/process/computer/io/InputRDDTest.java | 54 --
.../process/computer/io/OutputRDDTest.java | 62 --
.../spark/structure/io/ExampleInputRDD.java | 51 ++
.../spark/structure/io/ExampleOutputRDD.java | 50 ++
.../spark/structure/io/InputOutputRDDTest.java | 60 ++
.../spark/structure/io/InputRDDTest.java | 55 ++
.../spark/structure/io/OutputRDDTest.java | 63 ++
.../io/PersistedInputOutputRDDTest.java | 216 ++++++
.../step/sideEffect/TinkerGraphStep.java | 10 +-
.../optimization/TinkerGraphStepStrategy.java | 13 +-
.../tinkergraph/structure/TinkerFactory.java | 8 +-
.../tinkergraph/structure/TinkerGraph.java | 59 +-
.../tinkergraph/TinkerGraphProvider.java | 14 +-
.../strategy/TinkerGraphStrategySuite.java | 44 ++
.../strategy/TinkerGraphStrategyTest.java | 32 +
.../TinkerGraphStepStrategyTest.java | 33 +-
.../structure/TinkerGraphIdManagerTest.java | 18 +-
.../structure/TinkerGraphPlayTest.java | 41 +-
.../tinkergraph/structure/TinkerGraphTest.java | 16 +-
173 files changed, 6706 insertions(+), 2473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7ce67844/docs/src/upgrade-release-3.1.x-incubating.asciidoc
----------------------------------------------------------------------
diff --cc docs/src/upgrade-release-3.1.x-incubating.asciidoc
index e42be2b,5632468..ead97ae
--- a/docs/src/upgrade-release-3.1.x-incubating.asciidoc
+++ b/docs/src/upgrade-release-3.1.x-incubating.asciidoc
@@@ -35,11 -35,19 +35,20 @@@ Additional upgrade information can be f
Important Changes
~~~~~~~~~~~~~~~~~
- * The Gephi Plugin has improved integration with Gephi, where manually inserting {{store}} steps to visualize a running traversal is no longer required.
+ * `sum()`, `min()`, `max()` and `mean()` make now use of `NumberHelper`, leading to a more appropriate result type, instead of just `Double`.
+ * `GraphStep` is no longer in `sideEffect`-package, but now in `map`-package as traversals support mid-traversal `V()`.
+ * Traversals now support mid-traversal `V()`-steps. Graph system providers should ensure that a mid-traversal `V()` can leverage any suitable index.
+ * The data type of `Operator` enums will now always be the highest common data type of the two given numbers, rather than the data type of the first number, as it's been before.
+ * The Gephi Plugin has improved integration with Gephi, where manually inserting `store` steps to visualize a running traversal is no longer required.
* Entire TinkerGraph instances can be serialized over Gryo.
- * Hadoop1 support has been dropped. Hadoop2 is now supported. Giraph 1.1.0 is now supported and Spark of Hadoop2 YARN.
+ * Hadoop1 support has been dropped. Hadoop2 is now supported. Giraph and Spark can work over Hadoop2 YARN.
* The implementation and semantics of `GraphTraversal.group()` has changed. The previous model is deprecated and renamed to `groupV3d0()`.
+* `Transaction.onReadWrite()` and `Transaction.onClose()` are now `ThreadLocal` settings
+ * `PartitionStrategy` now supports `VertexProperty` such those properties can be added to partitions.
+ * The Jackson library is now "shaded" and included in the `gremlin-shaded` module.
+ * Configuration for Netty link:http://netty.io/wiki/native-transports.html[native transport] support in Gremlin Server.
+ * The `Channelizer` implementation for Gremlin Driver can now be changed via configuration.
+ * TinkerGraph is now serializable over Gryo so it is possible to request subgraphs via Gremlin Server (and `:remote`).
Please see the link:https://github.com/apache/incubator-tinkerpop/blob/3.1.0-incubating/CHANGELOG.asciidoc#XXXXXXXXXXXXXXXXXXXXXXXXXXXX[changelog] for a complete list of all the modifications that are part of this release.
@@@ -58,16 -66,25 +67,35 @@@ to closing where a user must now explic
See link:https://issues.apache.org/jira/browse/TINKERPOP3-805[TINKERPOP3-805] for more information.
+Transaction.onReadWrite() and Transaction.onClose() are now `ThreadLocal` settings
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+The `Transaction.onReadWrite()` and `Transaction.onClose()` settings now need to be set for each thread (if another behavior than the default is desired).
+For gremlin-server users that may be changing these settings via scripts :
+If the settings are changed from a sessionless request they will now only apply to that one request.
+If the settings are changed from an in-session request they will now only apply to all future requests made in the scope of that session.
+
+See link:https://issues.apache.org/jira/browse/TINKERPOP3-885[TINKERPOP3-885]
+
+ Hadoop-Gremlin Updates
+ ++++++++++++++++++++++
+
+ * Hadoop1 is no longer supported. Hadoop2 is now the only supported Hadoop version in TinkerPop.
+ * Spark and Giraph have been split out of Hadoop-Gremlin into their own respective packages (Spark-Gremlin and Giraph-Gremlin).
+ * The directory where application jars are stored in HDFS is now `hadoop-gremlin-x.y.z-libs`.
+ ** This versioning is important so that cross-version TinkerPop use does not cause jar conflicts.
+
+ Spark-Gremlin Updates
+ +++++++++++++++++++++
+
+ * Providers that wish to reuse a graphRDD can leverage the new `PersistedInputRDD` and `PersistedOutputRDD`.
+ ** This allows the graphRDD to avoid serialization into HDFS for reuse. Be sure to enabled persisted `SparkContext` (see documentation).
+
+ TinkerGraph-Gremlin Updates
+ +++++++++++++++++++++++++++
+
+ * The `public static String` configurations have been renamed. The old `public static` variables have been deprecated.
+
Gremlin Process
^^^^^^^^^^^^^^^
@@@ -121,17 -147,49 +158,67 @@@ If these tests were referenced in an `O
See link:https://issues.apache.org/jira/browse/TINKERPOP3-805[TINKERPOP3-805] for more information.
++AbstractTransaction implementation changes
++++++++++++++++++++++++++++++++++++++++++++
++The `AbstractTransaction` class now requires implementing four new methods:
++
++* `doReadWrite` that should execute the read-write consumer.
++* `doClose` that should execute the close consumer.
++* `setReadWrite` that should set the read-write consumer for the transaction.
++* `setClose` that should set the close consumer for the transaction.
++
++You can refer to `AbstractThreadLocalTransaction` and `AbstractThreadedTransaction` for examples.
++
Graph Traversal Updates
+++++++++++++++++++++++
- There were numerous changes to the `GraphTraversal` API. Nearly all changes are backwards compatible with respective "deprecated" annotations. Please review the respective updates specified in the "Graph System Users" section.
+ There were numerous changes to the `GraphTraversal` API. Nearly all changes are backwards compatible with respective
+ "deprecated" annotations. Please review the respective updates specified in the "Graph System Users" section.
+
+ * `GraphStep` is no longer in `sideEffect` package. Now in `map` package.
+ * Make sure mid-traversal `GraphStep` calls are folding `HasContainers` in for index-lookups.
+ * Think about copying `TinkerGraphStepStrategyTest` for your implementation so you know folding is happening correctly.
+
+ Element Removal
+ +++++++++++++++
+
+ `Element.Exceptions.elementAlreadyRemoved` has been deprecated and test enforcement for consistency have been removed.
+ Providers are free to deal with deleted elements as they see fit.
+
+ See link:https://issues.apache.org/jira/browse/TINKERPOP3-297[TINKERPOP3-297] for more information.
+
+ VendorOptimizationStrategy Rename
+ +++++++++++++++++++++++++++++++++
+
+ The `VendorOptimizationStrategy` has been renamed to `ProviderOptimizationStrategy`. This renaming is consistent
+ with revised terminology for what were formerly referred to as "vendors".
+
+ See link:https://issues.apache.org/jira/browse/TINKERPOP3-876[TINKERPOP3-876] for more information.
+
+ GraphComputer Updates
+ +++++++++++++++++++++
+
+ `GraphComputer.configure(String key, Object value)` is now a method (with default implementation).
+ This allows the user to specify engine-specific parameters to the underlying OLAP system. These parameters are not intended
+ to be cross engine supported. Moreover, if there are not parameters that can be altered (beyond the standard `GraphComputer`
+ methods), then the provider's `GraphComputer` implementation should simply return and do nothing.
+
+ Driver Providers
+ ^^^^^^^^^^^^^^^^
+
+ Aliases Parameter
+ +++++++++++++++++
+
+ The "rebindings" argument to the "standard" `OpProcessor` has been renamed to "aliases". While "rebindings" is still
+ supported it is recommended that the upgrade to "aliases" be made as soon as possible as support will be removed in
+ the future. Gremlin Server will not accept both parameters at the same time - a request must contain either one
+ parameter or the other if either is supplied.
- Driver Implementers
- ^^^^^^^^^^^^^^^^^^^
-See link:https://issues.apache.org/jira/browse/TINKERPOP3-913[TINKERPOP3-913] for more information.
++See link:https://issues.apache.org/jira/browse/TINKERPOP3-913[TINKERPOP3-913] for more information.
+
+Transaction.onReadWrite() and Transaction.onClose() are now `ThreadLocal` settings
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+If your driver configures the `Transaction.onReadWrite()` or `Transaction.onClose()` settings, note that these no longer apply to all future requests.
+If the settings are changed from a sessionless request they will only apply to that one request.
- If the settings are changed from an in-session request they will only apply to all future requests made in the scope of that session.
++If the settings are changed from an in-session request they will only apply to all future requests made in the scope of that session.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7ce67844/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/AbstractTransaction.java
----------------------------------------------------------------------
diff --cc gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/AbstractTransaction.java
index de7b6ee,56c6f36..54e7d9c
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/AbstractTransaction.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/AbstractTransaction.java
@@@ -35,7 -37,9 +35,6 @@@ import java.util.function.Function
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
public abstract class AbstractTransaction implements Transaction {
-
- protected Consumer<Transaction> readWriteConsumer;
- protected Consumer<Transaction> closeConsumer;
-
private Graph g;
public AbstractTransaction(final Graph g) {
@@@ -73,31 -83,6 +72,30 @@@
* {@link #addTransactionListener(Consumer)}.
*/
protected abstract void fireOnRollback();
+
-
+ /**
+ * Called {@link #readWrite}.
+ * Implementers should run their readWrite consumer here.
+ */
+ protected abstract void doReadWrite();
+
+ /**
+ * Called {@link #close}.
+ * Implementers should run their readWrite consumer here.
+ */
+ protected abstract void doClose();
+
+ /**
+ * Called {@link #onReadWrite}.
+ * Implementers should set their readWrite consumer here.
+ */
+ protected abstract void setReadWrite(final Consumer<Transaction> consumer);
+
+ /**
+ * Called {@link #onClose}.
+ * Implementers should set their close consumer here.
+ */
+ protected abstract void setClose(final Consumer<Transaction> consumer);
/**
* {@inheritDoc}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7ce67844/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinDriverIntegrateTest.java
----------------------------------------------------------------------