You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2018/07/31 17:38:07 UTC
[01/50] tinkerpop git commit: TINKERPOP-1996 Added IO to imports and
javadoc fixes [Forced Update!]
Repository: tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1967 b9f112bb7 -> 47ab1c2b3 (forced update)
TINKERPOP-1996 Added IO to imports and javadoc fixes
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/328737a3
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/328737a3
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/328737a3
Branch: refs/heads/TINKERPOP-1967
Commit: 328737a371f8a2040d02f9c2dbb06d049ce3c881
Parents: be9db8d
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 12 11:19:10 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:39:30 2018 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/jsr223/CoreImports.java | 2 ++
.../tinkerpop/gremlin/process/traversal/IO.java | 34 ++++++++++++++++----
2 files changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/328737a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
index 9d7eb52..72ad47a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
@@ -56,6 +56,7 @@ import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.optimiza
import org.apache.tinkerpop.gremlin.process.remote.RemoteConnection;
import org.apache.tinkerpop.gremlin.process.remote.RemoteGraph;
import org.apache.tinkerpop.gremlin.process.traversal.Bindings;
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.apache.tinkerpop.gremlin.process.traversal.P;
@@ -184,6 +185,7 @@ public final class CoreImports {
CLASS_IMPORTS.add(GraphReader.class);
CLASS_IMPORTS.add(GraphWriter.class);
CLASS_IMPORTS.add(Io.class);
+ CLASS_IMPORTS.add(IO.class);
CLASS_IMPORTS.add(IoCore.class);
CLASS_IMPORTS.add(Storage.class);
CLASS_IMPORTS.add(GraphMLIo.class);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/328737a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
index f76c2bc..6668cf1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
@@ -23,6 +23,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
/**
* Fields that can be provided to the {@link GraphTraversalSource#io(String)} using the
@@ -34,21 +40,37 @@ public class IO {
private IO() {}
+ /**
+ * A value to supply to {@link IO#reader} or {@link IO#writer} to indicate the format to use. Using this shorthand
+ * will configure a default {@link GraphSONReader} or {@link GraphSONWriter} respectively,
+ */
public static final String graphson = "graphson";
+
+ /**
+ * A value to supply to {@link IO#reader} or {@link IO#writer} to indicate the format to use. Using this shorthand
+ * will configure a default {@link GryoReader} or {@link GryoWriter} respectively,
+ */
public static final String gryo = "gryo";
+
+ /**
+ * A value to supply to {@link IO#reader} or {@link IO#writer} to indicate the format to use. Using this shorthand
+ * will configure a default {@link GraphMLReader} or {@link GraphMLWriter} respectively,
+ */
public static final String graphml = "graphml";
/**
- * The specific {@link GraphReader} instance to use or the name of the fully qualified classname of such an
- * instance. If this value is not specified then {@link GraphTraversalSource#io(String)} will attempt to construct
- * a default {@link GraphReader} based on the file extension provided to it.
+ * The specific {@link GraphReader} instance to use, the name of the fully qualified classname of such an
+ * instance or one of {@link IO#graphson}, {@link IO#gryo} or {@link IO#graphml}. If this value is not specified
+ * then {@link GraphTraversalSource#io(String)} will attempt to construct a default {@link GraphReader} based on
+ * the file extension provided to it.
*/
public static final String reader = Graph.Hidden.hide("tinkerpop.io.reader");
/**
- * The specific {@link GraphWriter} instance to use or the name of the fully qualified classname of such an
- * instance. If this value is not specified then {@link GraphTraversalSource#io(String)} will attempt to construct
- * a default {@link GraphWriter} based on the file extension provided to it.
+ * The specific {@link GraphWriter} instance to use, the name of the fully qualified classname of such an
+ * instance or one of {@link IO#graphson}, {@link IO#gryo} or {@link IO#graphml}. If this value is not specified
+ * then {@link GraphTraversalSource#io(String)} will attempt to construct a default {@link GraphWriter} based on
+ * the file extension provided to it.
*/
public static final String writer = Graph.Hidden.hide("tinkerpop.io.writer");
}
[27/50] tinkerpop git commit: TINKERPOP-1996 Prevent OLTP style
execution in Hadoop of io()
Posted by sp...@apache.org.
TINKERPOP-1996 Prevent OLTP style execution in Hadoop of io()
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/fdb35c69
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/fdb35c69
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/fdb35c69
Branch: refs/heads/TINKERPOP-1967
Commit: fdb35c69422f95199563e938ec30a2408d419a58
Parents: e9ebacf
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 16:13:47 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 16:13:47 2018 -0400
----------------------------------------------------------------------
.../process/computer/traversal/strategy/HadoopIoStrategy.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/fdb35c69/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
index af3db6e..3614745 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.VerificationException;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import java.util.Arrays;
@@ -54,6 +55,11 @@ public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalS
@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
+ // since hadoopgraph can't be modified we can't try to use the existing IoStep for standard processing
+ // without graphcomputer
+ if (traversal.getStartStep() instanceof IoStep)
+ throw new VerificationException("HadoopGraph requires a GraphComputer for io() step", traversal);
+
// VertexProgramStrategy should wrap up the IoStep in a TraversalVertexProgramStep. use that to grab the
// GraphComputer that was injected in there and push that in to the HadoopIoStep. this step pattern match
// is fairly specific and since you really can't chain together steps after io() this approach should work
[21/50] tinkerpop git commit: TINKERPOP-1996 Added upgrade docs
Posted by sp...@apache.org.
TINKERPOP-1996 Added upgrade docs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ded7c187
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ded7c187
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ded7c187
Branch: refs/heads/TINKERPOP-1967
Commit: ded7c187480a68b6f7be2d0cb777461f57a2871c
Parents: 8fd3bf2
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 11:20:03 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 11:20:03 2018 -0400
----------------------------------------------------------------------
docs/src/upgrade/release-3.4.x.asciidoc | 49 ++++++++++++++++++++++++++--
1 file changed, 47 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ded7c187/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.4.x.asciidoc b/docs/src/upgrade/release-3.4.x.asciidoc
index 97e14e3..0bb3903 100644
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@ -36,7 +36,7 @@ modulator. This modulator allows the step it is modifying to accept configuratio
behavior of the step itself. A good example of its usage is shown with the revised syntax of the `pageRank()` step
which now uses `with()` to replace the old `by()` options:
-[groovy]
+[source,groovy]
----
g.V().hasLabel('person').
pageRank().
@@ -49,7 +49,7 @@ g.V().hasLabel('person').
A similar change was made for `peerPressure()` step:
-[groovy]
+[source,groovy]
----
g.V().hasLabel('person').
peerPressure().
@@ -65,6 +65,33 @@ release where breaking changes are allowed.
See: link:https://issues.apache.org/jira/browse/TINKERPOP-1975[TINKERPOP-1975],
link:http://tinkerpop.apache.org/docs/3.4.0/reference/#with-step[Reference Documentation]
+==== io() Step
+
+There have been some important changes to IO operations for reading and writing graph data. The use of `Graph.io()`
+has been deprecated to further remove dependence on the Graph (Structure) API for users and to extend these basic
+operations to GLV users by making these features available as part of the Gremlin language.
+
+It is now possible to simply use Gremlin:
+
+[source,groovy]
+----
+graph = ...
+g = graph.traversal()
+g.io(someInputFile).read().iterate()
+g.io(someOutputFile).write().iterate()
+----
+
+While `io()` step is still single-threaded for OLTP style loading, it can be utilized in conjunction with OLAP which
+internally uses `CloneVertexProgram` and therefore any graph `InputFormat` or `OutputFormat` can be configured in
+conjunction with this step for parallel loads of large datasets.
+
+It is also worth noting that the `io()`-step may be overriden by graph providers to utilize their native bulk-loading
+features, so consult the documentation of the implementation being used to determine if there are any improved
+efficiencies there.
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-1996[TINKERPOP-1996],
+link:http://tinkerpop.apache.org/docs/3.4.0/reference/#io-step[Reference Documentation]
+
==== Removal of Giraph Support
Support for Giraph has been removed as of this version. There were a number of reasons for this decision which were
@@ -284,6 +311,24 @@ See: link:https://issues.apache.org/jira/browse/TINKERPOP-1522[TINKERPOP-1522]
==== Graph Database Providers
+===== io() Step
+
+The new `io()`-step that was introduced provides some new changes to consider. Note that `Graph.io()` has been
+deprecated and users are no longer instructed to utilize that method. It is not yet decided when that method will be
+removed completely, but given the public nature of it and the high chance of common usage, it should be hanging around
+for some time.
+
+As with any step in Gremlin, it is possible to replace it with a more provider specific implementation that could be
+more efficient. Developing a `TraversalStrategy` to do this is encouraged, especially for those graph providers who
+might have special bulk loaders that could be abstracted by this step. Examples of this are already shown with
+`HadoopGraph` which replaces the simple single-threaded loader with `CloneVertexProgram`. Graph providers are
+encouraged to use the `with()` step to capture any necessary configurations required for their underlying loader to
+work. Graph providers should not feel restricted to `graphson`, `gryo` and `graphml` formats either. If a graph
+supports CSV or some custom graph specific format, it shouldn't be difficult to gather the configurations necessary to
+make that available to users.
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-1996[TINKERPOP-1996]
+
===== Caching Graph Features
For graph implementations that have expensive creation times, it can be time consuming to run the TinkerPop test suite
[11/50] tinkerpop git commit: TINKERPOP-1996 Removed unecessary enum
Posted by sp...@apache.org.
TINKERPOP-1996 Removed unecessary enum
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/f8e3b8a1
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/f8e3b8a1
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/f8e3b8a1
Branch: refs/heads/TINKERPOP-1967
Commit: f8e3b8a1df1639405a29f261a9d7da147e88b356
Parents: 8187016
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 16 14:47:47 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:41:01 2018 -0400
----------------------------------------------------------------------
.../process/traversal/step/sideEffect/IoStep.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f8e3b8a1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
index 74b295b..9804333 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
@@ -18,6 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect;
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
@@ -52,12 +54,6 @@ import java.lang.reflect.Method;
*/
public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
- private enum Format {
- GRYO,
- GRAPHSON,
- GRAPHML
- }
-
private Parameters parameters = new Parameters();
private boolean first = true;
private String file;
@@ -219,6 +215,12 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension");
}
+ private Configuration getConfFromParameters() {
+ final Configuration conf = new BaseConfiguration();
+ parameters.getRaw().forEach((key, value) -> conf.setProperty(key.toString(), value.get(0)));
+ return conf;
+ }
+
@Override
public int hashCode() {
final int hash = super.hashCode() ^ this.parameters.hashCode();
[24/50] tinkerpop git commit: TINKERPOP-1996 Testing for GraphSON and
IoRegistry configuration
Posted by sp...@apache.org.
TINKERPOP-1996 Testing for GraphSON and IoRegistry configuration
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ae3f685a
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ae3f685a
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ae3f685a
Branch: refs/heads/TINKERPOP-1967
Commit: ae3f685ad7af326c9f1282da296e9db49f94da03
Parents: 7f1bf17
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 11:55:09 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 11:55:09 2018 -0400
----------------------------------------------------------------------
.../step/sideEffect/TinkerGraphIoStepTest.java | 23 ++++++++++++++++++++
1 file changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae3f685a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
index 06c4db8..8bab7da 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
@@ -72,4 +72,27 @@ public class TinkerGraphIoStepTest {
assertEquals(1, emptyG.V().has("custom", new CustomId("a", uuid)).count().next().intValue());
}
+
+ @Test
+ public void shouldWriteReadWithCustomIoRegistryGraphSON() throws Exception {
+ final UUID uuid = UUID.randomUUID();
+ g.addV("person").property("name","stephen").property("custom", new CustomId("a", uuid)).iterate();
+
+ final File file = TestHelper.generateTempFile(TinkerGraphIoStepTest.class, "shouldWriteReadWithCustomIoRegistryGraphSON", ".json");
+ g.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.class.getName()).write().iterate();
+
+ final Graph emptyGraph = TinkerGraph.open();
+ final GraphTraversalSource emptyG = emptyGraph.traversal();
+
+ try {
+ emptyG.io(file.getAbsolutePath()).read().iterate();
+ fail("Can't read without a registry");
+ } catch (Exception ignored) {
+ // do nothing
+ }
+
+ emptyG.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.instance()).read().iterate();
+
+ assertEquals(1, emptyG.V().has("custom", new CustomId("a", uuid)).count().next().intValue());
+ }
}
[32/50] tinkerpop git commit: Merge branch 'tp33'
Posted by sp...@apache.org.
Merge branch 'tp33'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/6358715e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/6358715e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/6358715e
Branch: refs/heads/TINKERPOP-1967
Commit: 6358715e9dfa69627a425e4b930636f271141022
Parents: 9ab5efa 7985106
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 27 16:29:50 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 27 16:29:50 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/element-existence.asciidoc | 91 ++++++++++++++++++++++++
docs/src/recipes/index.asciidoc | 2 +
2 files changed, 93 insertions(+)
----------------------------------------------------------------------
[22/50] tinkerpop git commit: TINKERPOP-1996 Verification strategy to
prevent io() from misuse
Posted by sp...@apache.org.
TINKERPOP-1996 Verification strategy to prevent io() from misuse
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/e6e4413e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/e6e4413e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/e6e4413e
Branch: refs/heads/TINKERPOP-1967
Commit: e6e4413e46594ae8f6d408de4e3d3bd85c228f53
Parents: ded7c18
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 11:47:24 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 11:47:24 2018 -0400
----------------------------------------------------------------------
.../strategy/verification/StandardVerificationStrategy.java | 7 +++++++
.../verification/StandardVerificationStrategyTest.java | 5 +++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/e6e4413e/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
index 50faa25..258d345 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.finaliza
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
import org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.NoneStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileSideEffectStep;
@@ -39,6 +40,7 @@ import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Stephen Mallette (http://stephen.genoprime.com)
*/
public final class StandardVerificationStrategy extends AbstractTraversalStrategy<TraversalStrategy.VerificationStrategy> implements TraversalStrategy.VerificationStrategy {
@@ -81,6 +83,11 @@ public final class StandardVerificationStrategy extends AbstractTraversalStrateg
if (TraversalHelper.getStepsOfClass(ProfileSideEffectStep.class, traversal).size() > 1) {
throw new VerificationException("The profile()-Step cannot be specified multiple times.", traversal);
}
+
+ if (traversal.getStartStep() instanceof ReadWriting && traversal.getSteps().size() != 1) {
+ if (TraversalHelper.getStepsOfClass(NoneStep.class, traversal).size() != traversal.getSteps().size() - 1)
+ throw new VerificationException("The io() step must be the first and only step in the traversal - it cannot be used with other steps", traversal);
+ }
}
public static StandardVerificationStrategy instance() {
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/e6e4413e/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
index ee96266..deec002 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
@@ -24,6 +24,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.RequirementsStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies;
+import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -50,7 +51,11 @@ public class StandardVerificationStrategyTest {
{"__.repeat(out().fold().unfold()).times(2)", repeat(out().fold().unfold()).times(2), false},
{"__.repeat(sum()).times(2)", repeat(sum()).times(2), false},
{"__.repeat(out().count())", repeat(out().count()), false},
+ {"__.io().read().V()", EmptyGraph.instance().traversal().io("junk.kryo").read().V(), false},
+ {"__.io().write().V()", EmptyGraph.instance().traversal().io("junk.kryo").write().V(), false},
// traversals that should pass verification
+ {"__.io().read().V().none()", EmptyGraph.instance().traversal().io("junk.kryo").read().none(), true},
+ {"__.io().write().V().none()", EmptyGraph.instance().traversal().io("junk.kryo").write().none(), true},
{"__.V().profile()",
__.V().profile(), true},
{"__.V().profile('metrics').cap('metrics')",
[17/50] tinkerpop git commit: TINKERPOP-1996 Added some docs around
IO.registry
Posted by sp...@apache.org.
TINKERPOP-1996 Added some docs around IO.registry
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/51dc8212
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/51dc8212
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/51dc8212
Branch: refs/heads/TINKERPOP-1967
Commit: 51dc82122af6b8d8c783de58a0451b8b2071c051
Parents: 9423397
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 07:15:07 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 07:15:07 2018 -0400
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 30 ++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/51dc8212/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index cd2f5f2..c3b9300 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1114,6 +1114,36 @@ IMPORTANT: Remote Gremlin Console users or Gremlin Language Variant (GLV) users
the `io()` step should recall that their `read()` or `write()` operation will occur on the server and not locally
and therefore the file specified for import/export must be something accessible by the server.
+GraphSON and Gryo formats are extensible allowing users and graph providers to extend supported serialization options.
+These extensions are exposed through `IoRegistry` implementations. To apply an `IoRegistry` use the `with()` option
+and the `IO.registry` key, where the value is either an actual `IoRegistry` instance or the fully qualified class
+name of one.
+
+[source,java]
+----
+g.io(someInputFile).
+ with(IO.reader, IO.gryo).
+ with(IO.registry, TinkerIoRegistryV3d0.instance())
+ read().iterate()
+g.io(someOutputFile).
+ with(IO.writer,IO.graphson).
+ with(IO.registry, "org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerIoRegistryV3d0")
+ write().iterate()
+----
+
+GLVs will obviously always be forced to use the latter form as they can't explicitly create an instance of an
+`IoRegistry` to pass to the server (nor are `IoRegistry` instances necessarily serializable).
+
+The version of the formats (e.g. GraphSON 2.0 or 3.0) utilized by `io()` is determined entirely by the `IO.reader` and
+`IO.writer` configurations or their defaults. The defaults will always be the latest version for the current release
+of TinkerPop. It is also possible for graph providers to override these defaults, so consult the documentation of the
+underlying graph database in use for any details on that.
+
+For more advanced configuration of `GraphReader` and `GraphWriter` operations (e.g. normalized output for GraphSON,
+disabling class registrations for Gryo, etc.) then construct the appropriate `GraphReader` and `GraphWriter` using
+the `build()` method on their implementations and use it directly. It can be passed directly to the `IO.reader` or
+`IO.writer` options. Obviously, these are JVM based operations and thus not available to GLVs as portable features.
+
[[_graphml_reader_writer]]
[[graphml]]
==== GraphML
[08/50] tinkerpop git commit: TINKERPOP-1996 Updated changelog
Posted by sp...@apache.org.
TINKERPOP-1996 Updated changelog
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/576649fd
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/576649fd
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/576649fd
Branch: refs/heads/TINKERPOP-1967
Commit: 576649fd5456f6390bf9481d01438a7e78db083e
Parents: f148e93
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 13 15:22:43 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:10 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/576649fd/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index bc78bbe..1add03c 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,6 +28,7 @@ This release also includes changes from <<release-3-3-3, 3.3.3>>.
* Bumped to Netty 4.1.25.
* Bumped to Spark 2.3.1.
* Added the `io()` start step and `read()` and `write()` termination steps to the Gremlin language.
+* Added `GraphFeatures.supportsIoRead()` and `GraphFeatures.supportsIoWrite()`.
* Deprecated `Graph.io()` and related infrastructure.
* Moved `Parameterizing` interface to the `org.apache.tinkerpop.gremlin.process.traversal.step` package with other marker interfaces of its type.
* Replaced `Parameterizing.addPropertyMutations()` with `Configuring.configure()`.
[50/50] tinkerpop git commit: TINKERPOP-1967 Added a component field
to the ConnectedComponent class
Posted by sp...@apache.org.
TINKERPOP-1967 Added a component field to the ConnectedComponent class
In this way the user can access the default more readily and is available in GLVs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/669d9fc7
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/669d9fc7
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/669d9fc7
Branch: refs/heads/TINKERPOP-1967
Commit: 669d9fc7ca2cc804288cd825b564ab35cf6bd8f7
Parents: 7fd7afd
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 08:29:13 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/connected-components.asciidoc | 5 ++++-
.../computer/traversal/step/map/ConnectedComponent.java | 7 +++++++
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/669d9fc7/docs/src/recipes/connected-components.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/connected-components.asciidoc b/docs/src/recipes/connected-components.asciidoc
index e6d0f7a..c46180f 100644
--- a/docs/src/recipes/connected-components.asciidoc
+++ b/docs/src/recipes/connected-components.asciidoc
@@ -64,10 +64,13 @@ The traversal looks like:
[gremlin-groovy,existing]
----
g.withComputer().V().connectedComponent().
- group().by('gremlin.connectedComponentVertexProgram.component').
+ group().by(component).
select(values).unfold()
----
+NOTE: The `component` option passed to `by()` is statically imported from `ConnectedComponent` and refers to the
+default property key within which the result of the algorithm is stored.
+
A straightforward way to detect the various subgraphs with an OLTP traversal is to do this:
[gremlin-groovy,existing]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/669d9fc7/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
index 85558bc..a2223d8 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.connected.ConnectedComponentVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -26,6 +27,12 @@ import org.apache.tinkerpop.gremlin.structure.Graph;
* {@link GraphTraversal#connectedComponent()} ()}.
*/
public class ConnectedComponent {
+
+ /**
+ * The default property key name that will hold the result of the algorithm.
+ */
+ public static final String component = ConnectedComponentVertexProgram.COMPONENT;
+
/**
* Configures the edge to traverse when calculating the pagerank.
*/
[48/50] tinkerpop git commit: TINKERPOP-1967 Added a test for
regressions on halted traversers
Posted by sp...@apache.org.
TINKERPOP-1967 Added a test for regressions on halted traversers
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/47ab1c2b
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/47ab1c2b
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/47ab1c2b
Branch: refs/heads/TINKERPOP-1967
Commit: 47ab1c2b30fc4aa28f0cb7f58c57e68e5989263c
Parents: f54d836
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 14:13:33 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
.../step/map/ConnectedComponentTest.java | 21 ++++++++++++++++++++
1 file changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47ab1c2b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
index 8b1904f..3133382 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
@@ -39,6 +39,8 @@ public abstract class ConnectedComponentTest extends AbstractGremlinProcessTest
public abstract Traversal<Vertex, Vertex> get_g_V_hasLabelXsoftwareX_connectedComponent();
+ public abstract Traversal<Vertex, Vertex> get_g_V_dedup_connectedComponent();
+
public abstract Traversal<Vertex, Vertex> get_g_V_connectedComponent_withXedges_bothEXknowsXX_withXpropertyName_clusterX();
@Test
@@ -57,6 +59,20 @@ public abstract class ConnectedComponentTest extends AbstractGremlinProcessTest
@Test
@LoadGraphWith(MODERN)
+ public void g_V_dedup_connectedComponent() {
+ final Traversal<Vertex, Vertex> traversal = get_g_V_dedup_connectedComponent();
+ printTraversalForm(traversal);
+ int counter = 0;
+ while (traversal.hasNext()) {
+ final Vertex vertex = traversal.next();
+ counter++;
+ assertEquals("1", vertex.value(ConnectedComponentVertexProgram.COMPONENT));
+ }
+ assertEquals(6, counter);
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
public void g_V_hasLabelXsoftwareX_connectedComponent() {
final Traversal<Vertex, Vertex> traversal = get_g_V_hasLabelXsoftwareX_connectedComponent();
printTraversalForm(traversal);
@@ -106,6 +122,11 @@ public abstract class ConnectedComponentTest extends AbstractGremlinProcessTest
}
@Override
+ public Traversal<Vertex, Vertex> get_g_V_dedup_connectedComponent() {
+ return g.V().dedup().connectedComponent();
+ }
+
+ @Override
public Traversal<Vertex, Vertex> get_g_V_hasLabelXsoftwareX_connectedComponent() {
return g.V().hasLabel("software").connectedComponent();
}
[14/50] tinkerpop git commit: TINKERPOP-1996 Added iterate() to
read()/write() steps in docs
Posted by sp...@apache.org.
TINKERPOP-1996 Added iterate() to read()/write() steps in docs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ae3b1497
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ae3b1497
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ae3b1497
Branch: refs/heads/TINKERPOP-1967
Commit: ae3b149789b8a9779d0549cee8a6ca2e2febbd2b
Parents: a580b6f
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 19 14:24:15 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 14:24:15 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/centrality.asciidoc | 2 +-
.../reference/implementations-neo4j.asciidoc | 4 +--
.../implementations-tinkergraph.asciidoc | 8 ++---
docs/src/reference/the-traversal.asciidoc | 35 +++++++++++---------
4 files changed, 26 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae3b1497/docs/src/recipes/centrality.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/centrality.asciidoc b/docs/src/recipes/centrality.asciidoc
index 0c1d4bc..59e6a4d 100644
--- a/docs/src/recipes/centrality.asciidoc
+++ b/docs/src/recipes/centrality.asciidoc
@@ -156,7 +156,7 @@ give it the highest rank. Consider the following example using the Grateful Dead
[gremlin-groovy]
----
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
g.V().repeat(groupCount('m').by('name').out()).times(5).cap('m'). <1>
order(local).by(values, desc).limit(local, 10).next() <2>
g.V().repeat(groupCount('m').by('name').out().timeLimit(100)).times(5).cap('m'). <3>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae3b1497/docs/src/reference/implementations-neo4j.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-neo4j.asciidoc b/docs/src/reference/implementations-neo4j.asciidoc
index 1760bd6..cfbf612 100644
--- a/docs/src/reference/implementations-neo4j.asciidoc
+++ b/docs/src/reference/implementations-neo4j.asciidoc
@@ -94,7 +94,7 @@ labels), a linear scan of the vertex-label partition is still faster than a line
----
graph = Neo4jGraph.open('/tmp/neo4j')
g = graph.traversal()
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
g.tx().commit()
clock(1000) {g.V().hasLabel('artist').has('name','Garcia').iterate()} <1>
graph.cypher("CREATE INDEX ON :artist(name)") <2>
@@ -162,7 +162,7 @@ It is possible to leverage Cypher from within Gremlin by using the `Neo4jGraph.c
----
graph = Neo4jGraph.open('/tmp/neo4j')
g = graph.traversal()
-g.io('data/tinkerpop-modern.kryo').read()
+g.io('data/tinkerpop-modern.kryo').read().iterate()
graph.cypher('MATCH (a {name:"marko"}) RETURN a')
graph.cypher('MATCH (a {name:"marko"}) RETURN a').select('a').out('knows').values('name')
graph.close()
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae3b1497/docs/src/reference/implementations-tinkergraph.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
index acd37dd..fe14d0f 100644
--- a/docs/src/reference/implementations-tinkergraph.asciidoc
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -90,12 +90,12 @@ TinkerGraph over the Grateful Dead graph.
----
graph = TinkerGraph.open()
g = graph.traversal()
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
clock(1000) {g.V().has('name','Garcia').iterate()} <1>
graph = TinkerGraph.open()
g = graph.traversal()
graph.createIndex('name',Vertex.class)
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
clock(1000){g.V().has('name','Garcia').iterate()} <2>
----
@@ -159,12 +159,12 @@ cardinality to `list` or else the data will import as `single`. Consider the fo
----
graph = TinkerGraph.open()
g = graph.traversal()
-g.io("data/tinkerpop-crew.kryo").read()
+g.io("data/tinkerpop-crew.kryo").read().iterate()
g.V().properties()
conf = new BaseConfiguration()
conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality","list")
graph = TinkerGraph.open(conf)
g = graph.traversal()
-g.io("data/tinkerpop-crew.kryo").read()
+g.io("data/tinkerpop-crew.kryo").read().iterate()
g.V().properties()
----
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae3b1497/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index cca71f6..cd2f5f2 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -401,7 +401,7 @@ made more salient on a larger graph. Therefore, the example below leverages the
----
graph = TinkerGraph.open()
g = graph.traversal()
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
g = graph.traversal().withoutStrategies(LazyBarrierStrategy) <1>
clockWithResult(1){g.V().both().both().both().count().next()} <2>
clockWithResult(1){g.V().repeat(both()).times(3).count().next()} <3>
@@ -424,7 +424,7 @@ optimization scenario with the added benefit of reducing the risk of an out-of-m
----
graph = TinkerGraph.open()
g = graph.traversal() <1>
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
clockWithResult(1){g.V().both().both().both().count().next()}
g.V().both().both().both().count().iterate().toString() <2>
----
@@ -1066,10 +1066,13 @@ execute. Therefore, a typical usage of the `io()` step would look like this:
[source,java]
----
-g.io(someInputFile).read()
-g.io(someOutputFile).write()
+g.io(someInputFile).read().iterate()
+g.io(someOutputFile).write().iterate()
----
+IMPORTANT: The commands above are still traversals and therefore require iteration to be executed, hence the use of
+`iterate()` as a termination step.
+
By default, the `io()` step will try to detect the right file format using the file name extension. To gain greater
control of the format use the `with()` step modulator to provide further information to `io()`. For example:
@@ -1077,13 +1080,13 @@ control of the format use the `with()` step modulator to provide further informa
----
g.io(someInputFile).
with(IO.reader, IO.graphson).
- read()
+ read().iterate()
g.io(someOutputFile).
with(IO.writer,IO.graphml).
- write()
+ write().iterate()
----
-The `IO` class is a helper that for the `io()` step that provides expressions that can be used to help configure it
+The `IO` class is a helper for the `io()` step that provides expressions that can be used to help configure it
and in this case it allows direct specification of the "reader" or "writer" to use. The "reader" actually refers to
a `GraphReader` implementation and the `writer" refers to a `GraphWriter` implementation. The implementations of
those interfaces provided by default are the standard TinkerPop implementations.
@@ -1136,8 +1139,8 @@ GraphML to appear before all `<edge>` elements if vertex labels are important to
[source,java]
----
-g.io("graph.xml").read()
-g.io("graph.xml").write()
+g.io("graph.xml").read().iterate()
+g.io("graph.xml").write().iterate()
----
NOTE: If using GraphML generated from TinkerPop 2.x, read more about its incompatibilities in the
@@ -1157,8 +1160,8 @@ but it is generally best used in two cases:
[source,java]
----
-g.io("graph.json").read()
-g.io("graph.json").write()
+g.io("graph.json").read().iterate()
+g.io("graph.json").write().iterate()
----
NOTE: Additional documentation for GraphSON can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/#graphson[IO Reference].
@@ -1182,8 +1185,8 @@ other. Failure to do so, may result in errors.
[source,java]
----
-g.io("graph.kryo").read()
-g.io("graph.kryo").write()
+g.io("graph.kryo").read().iterate()
+g.io("graph.kryo").write().iterate()
----
*Additional References*
@@ -1403,7 +1406,7 @@ songs which Jerry Garcia has both sung and written (using the Grateful Dead grap
[gremlin-groovy]
----
g = graph.traversal()
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
g.V().match(
__.as('a').has('name', 'Garcia'),
__.as('a').in('writtenBy').as('b'),
@@ -2495,7 +2498,7 @@ ranking.
[gremlin-groovy]
----
g = graph.traversal()
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
order(local).by(values,desc).limit(local, 5)
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
@@ -2509,7 +2512,7 @@ Similarly, for extracting the values from a path or map.
[gremlin-groovy]
----
g = graph.traversal()
-g.io('data/grateful-dead.xml').read()
+g.io('data/grateful-dead.xml').read().iterate()
g.V().hasLabel('song').out('sungBy').groupCount().by('name') <1>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values) <2>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values).unfold().
[38/50] tinkerpop git commit: Merge branch 'tp33'
Posted by sp...@apache.org.
Merge branch 'tp33'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7d21ee0b
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7d21ee0b
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7d21ee0b
Branch: refs/heads/TINKERPOP-1967
Commit: 7d21ee0b8b8770a68e6b5fb6f84500170f6c2a82
Parents: 4dccf14 ddc6694
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 11:41:28 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Jul 30 11:41:28 2018 -0400
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[37/50] tinkerpop git commit: Merge branch 'tp32' into tp33
Posted by sp...@apache.org.
Merge branch 'tp32' into tp33
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ddc66941
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ddc66941
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ddc66941
Branch: refs/heads/TINKERPOP-1967
Commit: ddc66941ed6340edd9009f0aa277aa8326255c68
Parents: 88b6e14 00a085a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 11:41:21 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Jul 30 11:41:21 2018 -0400
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ddc66941/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
----------------------------------------------------------------------
[35/50] tinkerpop git commit: Merge branch 'tp33'
Posted by sp...@apache.org.
Merge branch 'tp33'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4dccf144
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4dccf144
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4dccf144
Branch: refs/heads/TINKERPOP-1967
Commit: 4dccf144c71603612ce976827fd24c3eead4fb24
Parents: 6358715 88b6e14
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 07:16:33 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Jul 30 07:16:33 2018 -0400
----------------------------------------------------------------------
gremlin-server/src/main/bin/gremlin-server.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[07/50] tinkerpop git commit: TINKERPOP-1996 Added docs for io()
Posted by sp...@apache.org.
TINKERPOP-1996 Added docs for io()
Killed all the old IO documentation that utilized the GraphReader/Writer classes directly as well as the Graph.io() method that is now deprecated.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/62175c22
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/62175c22
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/62175c22
Branch: refs/heads/TINKERPOP-1967
Commit: 62175c228b77bdbda96c11015f2974828df8f3aa
Parents: 576649f
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 13 17:31:46 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:10 2018 -0400
----------------------------------------------------------------------
docs/src/reference/the-graph.asciidoc | 370 -------------------------
docs/src/reference/the-traversal.asciidoc | 140 ++++++++++
2 files changed, 140 insertions(+), 370 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/62175c22/docs/src/reference/the-graph.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-graph.asciidoc b/docs/src/reference/the-graph.asciidoc
index 1bcc96f..e9305b2 100644
--- a/docs/src/reference/the-graph.asciidoc
+++ b/docs/src/reference/the-graph.asciidoc
@@ -345,376 +345,6 @@ In the above case, the call to `graph.tx().createThreadedTx()` creates a new `Gr
`ThreadLocal` transaction, thus allowing each thread to operate on it in the same context. In this case, there would
be three separate vertices persisted to the `Graph`.
-== Gremlin I/O
-
-image:gremlin-io.png[width=250,float=right] The task of getting data in and out of `Graph` instances is the job of
-the Gremlin I/O packages. Gremlin I/O provides two interfaces for reading and writing `Graph` instances: `GraphReader`
-and `GraphWriter`. These interfaces expose methods that support:
-
-* Reading and writing an entire `Graph`
-* Reading and writing a `Traversal<Vertex>` as adjacency list format
-* Reading and writing a single `Vertex` (with and without associated `Edge` objects)
-* Reading and writing a single `Edge`
-* Reading and writing a single `VertexProperty`
-* Reading and writing a single `Property`
-* Reading and writing an arbitrary `Object`
-
-In all cases, these methods operate in the currency of `InputStream` and `OutputStream` objects, allowing graphs and
-their related elements to be written to and read from files, byte arrays, etc. The `Graph` interface offers the `io`
-method, which provides access to "reader/writer builder" objects that are pre-configured with serializers provided by
-the `Graph`, as well as helper methods for the various I/O capabilities. Unless there are very advanced requirements
-for the serialization process, it is always best to utilize the methods on the `Io` interface to construct
-`GraphReader` and `GraphWriter` instances, as the implementation may provide some custom settings that would otherwise
-have to be configured manually by the user to do the serialization.
-
-It is up to the implementations of the `GraphReader` and `GraphWriter` interfaces to choose the methods they
-implement and the manner in which they work together. The only characteristic enforced and expected is that the write
-methods should produce output that is compatible with the corresponding read method. For example, the output of
-`writeVertices` should be readable as input to `readVertices` and the output of `writeProperty` should be readable as
-input to `readProperty`.
-
-NOTE: Additional documentation for TinkerPop IO formats can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/[IO Reference].
-
-=== GraphML Reader/Writer
-
-image:gremlin-graphml.png[width=350,float=left] The link:http://graphml.graphdrawing.org/[GraphML] file format is a
-common XML-based representation of a graph. It is widely supported by graph-related tools and libraries making it a
-solid interchange format for TinkerPop. In other words, if the intent is to work with graph data in conjunction with
-applications outside of TinkerPop, GraphML may be the best choice to do that. Common use cases might be:
-
-* Generate a graph using link:https://networkx.github.io/[NetworkX], export it with GraphML and import it to TinkerPop.
-* Produce a subgraph and export it to GraphML to be consumed by and visualized in link:https://gephi.org/[Gephi].
-* Migrate the data of an entire graph to a different graph database not supported by TinkerPop.
-
-As GraphML is a specification for the serialization of an entire graph and not the individual elements of a graph,
-methods that support input and output of single vertices, edges, etc. are not supported.
-
-WARNING: GraphML is a "lossy" format in that it only supports primitive values for properties and does not have
-support for `Graph` variables. It will use `toString` to serialize property values outside of those primitives.
-
-WARNING: GraphML as a specification allows for `<edge>` and `<node>` elements to appear in any order. Most software
-that writes GraphML (including as TinkerPop's `GraphMLWriter`) write `<node>` elements before `<edge>` elements. However it
-is important to note that `GraphMLReader` will read this data in order and order can matter. This is because TinkerPop
-does not allow the vertex label to be changed after the vertex has been created. Therefore, if an `<edge>` element
-comes before the `<node>`, the label on the vertex will be ignored. It is thus better to order `<node>` elements in the
-GraphML to appear before all `<edge>` elements if vertex labels are important to the graph.
-
-The following code shows how to write a `Graph` instance to file called `tinkerpop-modern.xml` and then how to read
-that file back into a different instance:
-
-[source,java]
-----
-Graph graph = TinkerFactory.createModern();
-graph.io(IoCore.graphml()).writeGraph("tinkerpop-modern.xml");
-Graph newGraph = TinkerGraph.open();
-newGraph.io(IoCore.graphml()).readGraph("tinkerpop-modern.xml");
-----
-
-If a custom configuration is required, then have the `Graph` generate a `GraphReader` or `GraphWriter` "builder" instance:
-
-[source,java]
-----
-Graph graph = TinkerFactory.createModern();
-try (OutputStream os = new FileOutputStream("tinkerpop-modern.xml")) {
- graph.io(IoCore.graphml()).writer().normalize(true).create().writeGraph(os, graph);
-}
-
-Graph newGraph = TinkerGraph.open();
-try (InputStream stream = new FileInputStream("tinkerpop-modern.xml")) {
- newGraph.io(IoCore.graphml()).reader().create().readGraph(stream, newGraph);
-}
-----
-
-NOTE: If using GraphML generated from TinkerPop 2.x, you can read more about its incompatibilities in the
-link:http://tinkerpop.apache.org/docs/x.y.z/upgrade/#graphml-format[Upgrade Documentation].
-
-[[graphson-reader-writer]]
-=== GraphSON Reader/Writer
-
-image:gremlin-graphson.png[width=350,float=left] GraphSON is a link:http://json.org/[JSON]-based format extended
-from earlier versions of TinkerPop. It is important to note that TinkerPop's GraphSON is not backwards compatible
-with prior TinkerPop GraphSON versions. GraphSON has some support from graph-related application outside of TinkerPop,
-but it is generally best used in two cases:
-
-* A text format of the graph or its elements is desired (e.g. debugging, usage in source control, etc.)
-* The graph or its elements need to be consumed by code that is not JVM-based (e.g. JavaScript, Python, .NET, etc.)
-
-GraphSON supports all of the `GraphReader` and `GraphWriter` interface methods and can therefore read or write an
-entire `Graph`, vertices, arbitrary objects, etc. The following code shows how to write a `Graph` instance to file
-called `tinkerpop-modern.json` and then how to read that file back into a different instance:
-
-[source,java]
-----
-Graph graph = TinkerFactory.createModern();
-graph.io(graphson()).writeGraph("tinkerpop-modern.json");
-
-Graph newGraph = TinkerGraph.open();
-newGraph.io(graphson()).readGraph("tinkerpop-modern.json");
-----
-
-NOTE: Using `graphson()`, which is a static helper method of `IoCore`, will default to the most current version of GraphSON which is 3.0.
-
-If a custom configuration is required, then have the `Graph` generate a `GraphReader` or `GraphWriter` "builder" instance:
-
-[source,java]
-----
-Graph graph = TinkerFactory.createModern();
-try (OutputStream os = new FileOutputStream("tinkerpop-modern.json")) {
- GraphSONMapper mapper = graph.io(IoCore.graphson()).mapper().normalize(true).create()
- graph.io(graphson()).writer().mapper(mapper).create().writeGraph(os, graph)
-}
-
-Graph newGraph = TinkerGraph.open();
-try (InputStream stream = new FileInputStream("tinkerpop-modern.json")) {
- newGraph.io(graphson()).reader().create().readGraph(stream, newGraph);
-}
-----
-
-The following example shows how a single `Vertex` is written to GraphSON using the Gremlin Console:
-
-[gremlin-groovy]
-----
-graph = TinkerFactory.createModern()
-g = graph.traversal()
-f = new ByteArrayOutputStream()
-graph.io(graphson()).writer().create().writeVertex(f, g.V(1).next(), BOTH)
-f.close()
-----
-
-The following GraphSON example shows the output of `GraphSONWriter.writeVertex()` with associated edges:
-
-[source,json]
-----
-{
- "id": {
- "@type": "g:Int32",
- "@value": 1
- },
- "label": "person",
- "outE": {
- "created": [{
- "id": {
- "@type": "g:Int32",
- "@value": 9
- },
- "inV": {
- "@type": "g:Int32",
- "@value": 3
- },
- "properties": {
- "weight": {
- "@type": "g:Double",
- "@value": 0.4
- }
- }
- }],
- "knows": [{
- "id": {
- "@type": "g:Int32",
- "@value": 7
- },
- "inV": {
- "@type": "g:Int32",
- "@value": 2
- },
- "properties": {
- "weight": {
- "@type": "g:Double",
- "@value": 0.5
- }
- }
- }, {
- "id": {
- "@type": "g:Int32",
- "@value": 8
- },
- "inV": {
- "@type": "g:Int32",
- "@value": 4
- },
- "properties": {
- "weight": {
- "@type": "g:Double",
- "@value": 1.0
- }
- }
- }]
- },
- "properties": {
- "name": [{
- "id": {
- "@type": "g:Int64",
- "@value": 0
- },
- "value": "marko"
- }],
- "age": [{
- "id": {
- "@type": "g:Int64",
- "@value": 1
- },
- "value": {
- "@type": "g:Int32",
- "@value": 29
- }
- }]
- }
-}
-----
-
-GraphSON has several versions and each has differences that prevent complete compatibility with one another. While the
-default version provided by `IoCore.graphson()` is recommended, it is possible to make changes to revert to an earlier
-version. The following shows an example of how to use 1.0 (with type embedding):
-
-[gremlin-groovy]
-----
-graph = TinkerFactory.createModern()
-g = graph.traversal()
-f = new ByteArrayOutputStream()
-mapper = graph.io(GraphSONIo.build(GraphSONVersion.V1_0)).mapper().typeInfo(TypeInfo.PARTIAL_TYPES).create()
-graph.io(GraphSONIo.build(GraphSONVersion.V1_0)).writer().mapper(mapper).create().writeVertex(f, g.V(1).next(), BOTH)
-f.close()
-----
-
-NOTE: Additional documentation for GraphSON can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/#graphson[IO Reference].
-
-IMPORTANT: When using the extended type system in Gremlin Server, support for these types when used in the context of
-Gremlin Language Variants is dependent on the programming language, the driver and its serializers. These
-implementations are only required to support the core types and not the extended ones.
-
-Here's the same previous example of GraphSON 1.0, but with GraphSON 2.0:
-
-[gremlin-groovy]
-----
-graph = TinkerFactory.createModern()
-g = graph.traversal()
-f = new ByteArrayOutputStream()
-mapper = graph.io(graphson()).mapper().version(GraphSONVersion.V2_0).create()
-graph.io(graphson()).writer().mapper(mapper).create().writeVertex(f, g.V(1).next(), BOTH)
-f.close()
-----
-
-Creating a GraphSON 2.0 mapper is done by calling `.version(GraphSONVersion.V2_0)` on the mapper builder. Here's is the
-example output from the code above:
-
-[source,json]
-----
-{
- "@type": "g:Vertex",
- "@value": {
- "id": {
- "@type": "g:Int32",
- "@value": 1
- },
- "label": "person",
- "properties": {
- "name": [{
- "@type": "g:VertexProperty",
- "@value": {
- "id": {
- "@type": "g:Int64",
- "@value": 0
- },
- "value": "marko",
- "label": "name"
- }
- }],
- "uuid": [{
- "@type": "g:VertexProperty",
- "@value": {
- "id": {
- "@type": "g:Int64",
- "@value": 12
- },
- "value": {
- "@type": "g:UUID",
- "@value": "829c7ddb-3831-4687-a872-e25201230cd3"
- },
- "label": "uuid"
- }
- }],
- "age": [{
- "@type": "g:VertexProperty",
- "@value": {
- "id": {
- "@type": "g:Int64",
- "@value": 1
- },
- "value": {
- "@type": "g:Int32",
- "@value": 29
- },
- "label": "age"
- }
- }]
- }
- }
-}
-----
-
-Types can be disabled when creating a GraphSON 2.0 `Mapper` with:
-
-[source,groovy]
-----
-graph.io(graphson()).mapper().
- version(GraphSONVersion.V2_0).
- typeInfo(GraphSONMapper.TypeInfo.NO_TYPES).create()
-----
-
-By disabling types, the JSON payload produced will lack the extra information that is written for types. Please note,
-disabling types can be unsafe with regards to the written data in that types can be lost.
-
-[[gryo-reader-writer]]
-=== Gryo Reader/Writer
-
-image:gremlin-kryo.png[width=400,float=left] link:https://github.com/EsotericSoftware/kryo[Kryo] is a popular
-serialization package for the JVM. Gremlin-Kryo is a binary `Graph` serialization format for use on the JVM by JVM
-languages. It is designed to be space efficient, non-lossy and is promoted as the standard format to use when working
-with graph data inside of the TinkerPop stack. A list of common use cases is presented below:
-
-* Migration from one Gremlin Structure implementation to another (e.g. `TinkerGraph` to `Neo4jGraph`)
-* Serialization of individual graph elements to be sent over the network to another JVM.
-* Backups of in-memory graphs or subgraphs.
-
-WARNING: When migrating between Gremlin Structure implementations, Kryo may not lose data, but it is important to
-consider the features of each `Graph` and whether or not the data types supported in one will be supported in the
-other. Failure to do so, may result in errors.
-
-Kryo supports all of the `GraphReader` and `GraphWriter` interface methods and can therefore read or write an entire
-`Graph`, vertices, edges, etc. The following code shows how to write a `Graph` instance to file called
-`tinkerpop-modern.kryo` and then how to read that file back into a different instance:
-
-[source,java]
-----
-Graph graph = TinkerFactory.createModern();
-graph.io(gryo()).writeGraph("tinkerpop-modern.kryo");
-
-Graph newGraph = TinkerGraph.open();
-newGraph.io(gryo()).readGraph("tinkerpop-modern.kryo");
-----
-
-NOTE: Using `gryo()`, which is a static helper method of `IoCore`, will default to the most current version of Gryo which is 3.0.
-
-If a custom configuration is required, then have the `Graph` generate a `GraphReader` or `GraphWriter` "builder" instance:
-
-[source,java]
-----
-Graph graph = TinkerFactory.createModern();
-try (OutputStream os = new FileOutputStream("tinkerpop-modern.kryo")) {
- graph.io(GryoIo.build(GryoVersion.V1_0)).writer().create().writeGraph(os, graph);
-}
-
-Graph newGraph = TinkerGraph.open();
-try (InputStream stream = new FileInputStream("tinkerpop-modern.kryo")) {
- newGraph.io(GryoIo.build(GryoVersion.V1_0)).reader().create().readGraph(stream, newGraph);
-}
-----
-
-NOTE: The preferred extension for files names produced by Gryo is `.kryo`.
-
-NOTE: Data migrations from TinkerPop 2.x are discussed in the Appendix of the
-link:http://tinkerpop.apache.org/docs/x.y.z/upgrade/#appendix[Upgrade Documentation].
-
== Namespace Conventions
End users, <<implementations,graph system providers>>, <<graphcomputer,`GraphComputer`>> algorithm designers,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/62175c22/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 1fb8abd..34f6b27 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1049,6 +1049,146 @@ inject(1,2).map {it.get() + 1}.map {g.V(it.get()).next()}.values('name')
link:++http://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#inject-E...-++[`inject(Object)`]
+[[_gremlin_i_o]]
+[[io-step]]
+=== IO Step
+
+image:gremlin-io.png[width=250,float=left] The task of importing and exporting the data of `Graph` instances is the
+job of the `io()`-step. By default, TinkerPop supports three formats for importing and exporting graph data in
+<<graphml,GraphML>>, <<graphson,GraphSON>>, and <<gryo,Gryo>>.
+
+NOTE: Additional documentation for TinkerPop IO formats can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/[IO Reference].
+
+By itself the `io()` step merely configures the kind of importing and exporting that is going
+to occur and it is the follow-on call to the `read()` or `write()` step that determines which of those actions will
+execute. Therefore, a typical usage of the `io()` step would look like this:
+
+[source,java]
+----
+g.io(someInputFile).read()
+g.io(someOutputFile).write()
+----
+
+By default, the `io()` step will try to detect the right file format using the file name extension. To gain greater
+control of the format use the `with()` step modulator to provide further information to `io()`. For example:
+
+[source,java]
+----
+g.io(someInputFile).
+ with(IO.reader, IO.graphson).
+ read()
+g.io(someOutputFile).
+ with(IO.writer,IO.graphml).
+ write()
+----
+
+The `IO` class is a helper that for the `io()` step that provides expressions that can be used to help configure it
+and in this case it allows direct specification of the "reader" or "writer" to use. The "reader" actually refers to
+a `GraphReader` implementation and the `writer" refers to a `GraphWriter` implementation. The implementations of
+those interfaces provided by default are the standard TinkerPop implementations.
+
+That default is an important point to consider for users. The default TinkerPop implementations are not designed with
+massive, complex, parallel bulk loading in mind. They are designed to do single-threaded, OLTP-style loading of data
+in the most generic way possible so as to accommodate the greatest number of graph databases out there. As such, from
+a reading perspective, they work best for small datasets (or perhaps medium datasets where memory is plentiful and
+time is not critical) that are loading to an empty graph - incremental loading is not supported. The story from the
+writing perspective is not that different in there are no parallel operations in play, however streaming the output
+to disk requires a single pass of the data without high memory requirements for larger datasets.
+
+In general, TinkerPop recommends that users examine the native bulk import/export tools of the graph implementation
+that they choose. Those tools will often outperform the `io()` step and perhaps be easier to use with a greater
+feature set. That said, graph providers do have the option to optimize `io()` to back it with their own
+import/export utilities and therefore the default behavior provided by TinkerPop described above might be overridden
+by the graph.
+
+An excellent example of this lies in <<hadoop-gremlin,HadoopGraph>> with <<sparkgraphcomputer,SparkGraphComputer>>
+which replaces the default single-threaded implementation with a more advanced OLAP style bulk import/export
+functionality internally using <<clonevertexprogram,CloneVertexProgram>>. With this model, graphs of arbitrary size
+can be imported/exported assuming that there is a Hadoop `InputFormat` or `OutputFormat` to support it.
+
+IMPORTANT: Remote Gremlin Console users or Gremlin Language Variant (GLV) users (e.g. gremlin-python) who utilize
+the `io()` step should recall that their `read()` or `write()` operation will occur on the server and not locally
+and therefore the file specified for import/export must be something accessible by the server.
+
+[[_graphml_reader_writer]]
+[[graphml]]
+==== GraphML
+
+image:gremlin-graphml.png[width=350,float=left] The link:http://graphml.graphdrawing.org/[GraphML] file format is a
+common XML-based representation of a graph. It is widely supported by graph-related tools and libraries making it a
+solid interchange format for TinkerPop. In other words, if the intent is to work with graph data in conjunction with
+applications outside of TinkerPop, GraphML may be the best choice to do that. Common use cases might be:
+
+* Generate a graph using link:https://networkx.github.io/[NetworkX], export it with GraphML and import it to TinkerPop.
+* Produce a subgraph and export it to GraphML to be consumed by and visualized in link:https://gephi.org/[Gephi].
+* Migrate the data of an entire graph to a different graph database not supported by TinkerPop.
+
+WARNING: GraphML is a "lossy" format in that it only supports primitive values for properties and does not have
+support for `Graph` variables. It will use `toString` to serialize property values outside of those primitives.
+
+WARNING: GraphML as a specification allows for `<edge>` and `<node>` elements to appear in any order. Most software
+that writes GraphML (including as TinkerPop's `GraphMLWriter`) write `<node>` elements before `<edge>` elements. However it
+is important to note that `GraphMLReader` will read this data in order and order can matter. This is because TinkerPop
+does not allow the vertex label to be changed after the vertex has been created. Therefore, if an `<edge>` element
+comes before the `<node>`, the label on the vertex will be ignored. It is thus better to order `<node>` elements in the
+GraphML to appear before all `<edge>` elements if vertex labels are important to the graph.
+
+[source,java]
+----
+g.io("graph.xml").read()
+g.io("graph.xml").write()
+----
+
+NOTE: If using GraphML generated from TinkerPop 2.x, read more about its incompatibilities in the
+link:http://tinkerpop.apache.org/docs/x.y.z/upgrade/#graphml-format[Upgrade Documentation].
+
+[[graphson-reader-writer]]
+[[graphson]]
+==== GraphSON
+
+image:gremlin-graphson.png[width=350,float=left] GraphSON is a link:http://json.org/[JSON]-based format extended
+from earlier versions of TinkerPop. It is important to note that TinkerPop's GraphSON is not backwards compatible
+with prior TinkerPop GraphSON versions. GraphSON has some support from graph-related application outside of TinkerPop,
+but it is generally best used in two cases:
+
+* A text format of the graph or its elements is desired (e.g. debugging, usage in source control, etc.)
+* The graph or its elements need to be consumed by code that is not JVM-based (e.g. JavaScript, Python, .NET, etc.)
+
+[source,java]
+----
+g.io("graph.json").read()
+g.io("graph.json").write()
+----
+
+NOTE: Additional documentation for GraphSON can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/#graphson[IO Reference].
+
+[[gryo-reader-writer]]
+[[gryo]]
+==== Gryo
+
+image:gremlin-kryo.png[width=400,float=left] link:https://github.com/EsotericSoftware/kryo[Kryo] is a popular
+serialization package for the JVM. Gremlin-Kryo is a binary `Graph` serialization format for use on the JVM by JVM
+languages. It is designed to be space efficient, non-lossy and is promoted as the standard format to use when working
+with graph data inside of the TinkerPop stack. A list of common use cases is presented below:
+
+* Migration from one Gremlin Structure implementation to another (e.g. `TinkerGraph` to `Neo4jGraph`)
+* Serialization of individual graph elements to be sent over the network to another JVM.
+* Backups of in-memory graphs or subgraphs.
+
+WARNING: When migrating between Gremlin Structure implementations, Kryo may not lose data, but it is important to
+consider the features of each `Graph` and whether or not the data types supported in one will be supported in the
+other. Failure to do so, may result in errors.
+
+[source,java]
+----
+g.io("graph.kryo").read()
+g.io("graph.kryo").write()
+----
+
+*Additional References*
+
+link:++http://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.html#io-String...-++[`io(String)`]
+
[[is-step]]
=== Is Step
[09/50] tinkerpop git commit: TINKERPOP-1996 Got read/write() tests
running for OLAP
Posted by sp...@apache.org.
TINKERPOP-1996 Got read/write() tests running for OLAP
Introduced new Graph.Features to provider better separation between graph mutation features and graph loading features - they are two different things as demonstrated by io(). Fixed HadoopIoStep/Strategy so that they properly handle the different input/output format types expected.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/f148e933
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/f148e933
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/f148e933
Branch: refs/heads/TINKERPOP-1967
Commit: f148e9331a945e0f4f707ea937b610e5902701c7
Parents: bd275a7
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 13 15:17:17 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:10 2018 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/structure/Graph.java | 27 +++++++++
.../gremlin/AbstractGraphProvider.java | 18 ++++--
.../gremlin/process/ProcessComputerSuite.java | 4 ++
.../process/traversal/step/map/ReadTest.java | 21 ++++---
.../process/traversal/step/map/WriteTest.java | 14 ++++-
.../traversal/step/map/HadoopIoStep.java | 60 +++++++++++++++++++-
.../traversal/strategy/HadoopIoStrategy.java | 2 +-
.../gremlin/hadoop/structure/HadoopGraph.java | 16 ++++++
8 files changed, 142 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
index f1fc54a..f62b897 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
@@ -23,6 +23,8 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.util.FeatureDescriptor;
@@ -439,6 +441,8 @@ public interface Graph extends AutoCloseable, Host {
public static final String FEATURE_PERSISTENCE = "Persistence";
public static final String FEATURE_THREADED_TRANSACTIONS = "ThreadedTransactions";
public static final String FEATURE_CONCURRENT_ACCESS = "ConcurrentAccess";
+ public static final String FEATURE_IO_READ = "IoRead";
+ public static final String FEATURE_IO_WRITE = "IoWrite";
/**
* Determines if the {@code Graph} implementation supports {@link GraphComputer} based processing.
@@ -489,6 +493,29 @@ public interface Graph extends AutoCloseable, Host {
}
/**
+ * Determines if the {@code Graph} implementations supports read operations as executed with the
+ * {@link GraphTraversalSource#io(String)} step. Graph implementations will generally support this by
+ * default as any graph that can support direct mutation through the Structure API will by default
+ * accept data from the standard TinkerPop {@link GraphReader} implementations. However, some graphs like
+ * {@code HadoopGraph} don't accept direct mutations but can still do reads from that {@code io()} step.
+ */
+ @FeatureDescriptor(name = FEATURE_IO_READ)
+ public default boolean supportsIoRead() {
+ return true;
+ }
+
+ /**
+ * Determines if the {@code Graph} implementations supports write operations as executed with the
+ * {@link GraphTraversalSource#io(String)} step. Graph implementations will generally support this by
+ * default given the standard TinkerPop {@link GraphWriter} implementations. However, some graphs like
+ * {@code HadoopGraph} will use a different approach to handle writes.
+ */
+ @FeatureDescriptor(name = FEATURE_IO_WRITE)
+ public default boolean supportsIoWrite() {
+ return true;
+ }
+
+ /**
* Gets the features related to "graph sideEffects" operation.
*/
public default VariableFeatures variables() {
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
index 95c6b57..75d033b 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
@@ -20,6 +20,7 @@ package org.apache.tinkerpop.gremlin;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.commons.configuration.BaseConfiguration;
@@ -131,12 +132,19 @@ public abstract class AbstractGraphProvider implements GraphProvider {
return methodName.replaceAll("[0-9, -]+$", String.valueOf(random));
}
- protected void readIntoGraph(final Graph g, final String path) throws IOException {
- final GraphReader reader = GryoReader.build()
- .mapper(g.io(GryoIo.build()).mapper().create())
- .create();
+ /**
+ * Used by the default implementation of {@link AbstractGraphProvider#loadGraphData(Graph, LoadGraphWith, Class, String)}
+ * to read the graph from a Kryo file using the default {@link GryoReader} implementation. If the default
+ * implementation does not work (perhaps a graph implementation needs to register some special {@link IoRegistry}
+ * then this method or its caller should be overridden to suit the implementation.
+ *
+ * @param graph the graph to load to
+ * @param path the path to the file to load into the graph
+ */
+ protected void readIntoGraph(final Graph graph, final String path) throws IOException {
+ final GraphReader reader = GryoReader.build().create();
try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream(path)) {
- reader.readGraph(stream, g);
+ reader.readGraph(stream, graph);
}
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
index b224c8b..eab562d 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
@@ -70,11 +70,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ExplainTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest;
@@ -163,10 +165,12 @@ public class ProcessComputerSuite extends AbstractGremlinSuite {
ProjectTest.Traversals.class,
ProgramTest.Traversals.class,
PropertiesTest.Traversals.class,
+ ReadTest.Traversals.class,
SelectTest.Traversals.class,
UnfoldTest.Traversals.class,
ValueMapTest.Traversals.class,
VertexTest.Traversals.class,
+ WriteTest.Traversals.class,
// sideEffect
AddEdgeTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
index 9e53169..ab59194 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
@@ -18,12 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.IoTest;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLResourceAccess;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
@@ -33,6 +35,7 @@ import org.junit.runner.RunWith;
import java.io.IOException;
+import static org.apache.tinkerpop.gremlin.structure.Graph.Features.GraphFeatures.FEATURE_IO_READ;
import static org.junit.Assert.assertFalse;
/**
@@ -54,8 +57,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
public abstract Traversal<Object,Object> get_g_io_read_withXreader_graphmlX(final String fileToRead) throws IOException;
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
- public void g_readXkryoX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
+ public void g_io_readXkryoX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXkryoX(fileToRead);
printTraversalForm(traversal);
@@ -65,7 +68,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
public void g_io_read_withXreader_gryoX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_gryoX(fileToRead);
@@ -76,8 +79,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
- public void g_readXjsonX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
+ public void g_io_readXjsonX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXjsonX(fileToRead);
printTraversalForm(traversal);
@@ -87,7 +90,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
public void g_io_read_withXreader_graphsonX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphsonX(fileToRead);
@@ -98,8 +101,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
- public void g_readXxmlX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
+ public void g_io_readXxmlX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXxmlX(fileToRead);
printTraversalForm(traversal);
@@ -109,7 +112,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
public void g_io_read_withXreader_graphmlX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphmlX(fileToRead);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
index e739c0a..55e6f9c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
@@ -18,12 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.structure.Graph;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -54,7 +56,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void g_writeXkryoX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
+ public void g_io_writeXkryoX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class, "tinkerpop-modern-v3d0", ".kryo").getAbsolutePath().replace('\\', '/');
final File f = new File(fileToWrite);
@@ -69,6 +72,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
public void g_io_write_withXwrite_gryoX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class, "tinkerpop-modern-v3d0", ".kryo").getAbsolutePath().replace('\\', '/');
@@ -84,7 +88,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void g_writeXjsonX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
+ public void g_io_writeXjsonX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern-v3d0", ".json").getAbsolutePath().replace('\\', '/');
final File f = new File(fileToWrite);
@@ -99,6 +104,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
public void g_io_write_withXwriter_graphsonX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern-v3d0", ".json").getAbsolutePath().replace('\\', '/');
@@ -114,7 +120,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void g_writeXxmlX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
+ public void g_io_writeXxmlX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern", ".xml").getAbsolutePath().replace('\\', '/');
final File f = new File(fileToWrite);
@@ -129,6 +136,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
public void g_io_write_withXwriter_graphmlX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern", ".xml").getAbsolutePath().replace('\\', '/');
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
index 62937da..97fdea4 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
@@ -18,18 +18,30 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map;
+import org.apache.hadoop.mapred.InputFormat;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import java.lang.reflect.Method;
+
/**
* An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the
* {@link CloneVertexProgram} for its implementation.
@@ -103,12 +115,56 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
}
private void configureForRead(final Graph graph) {
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat");
+ final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0);
+ String inputFormatClassName;
+ if (inputFormatClassNameOrKeyword.equals(IO.graphson))
+ inputFormatClassName = GraphSONInputFormat.class.getName();
+ else if (inputFormatClassNameOrKeyword.equals(IO.gryo))
+ inputFormatClassName = GryoInputFormat.class.getName();
+ else if (inputFormatClassNameOrKeyword.equals(IO.graphml))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ inputFormatClassName = inputFormatClassNameOrKeyword;
+
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
}
private void configureForWrite(final Graph graph) {
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat");
+ final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0);
+ String outputFormatClassName;
+ if (outputFormatClassNameOrKeyword.equals(IO.graphson))
+ outputFormatClassName = GraphSONOutputFormat.class.getName();
+ else if (outputFormatClassNameOrKeyword.equals(IO.gryo))
+ outputFormatClassName = GryoOutputFormat.class.getName();
+ else if (outputFormatClassNameOrKeyword.equals(IO.graphml))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ outputFormatClassName = outputFormatClassNameOrKeyword;
+
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
}
+
+ private String detectReader() {
+ if (file.endsWith(".kryo"))
+ return GryoInputFormat.class.getName();
+ else if (file.endsWith(".json"))
+ return GraphSONInputFormat.class.getName();
+ else if (file.endsWith(".xml"))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
+
+ private String detectWriter() {
+ if (file.endsWith(".kryo"))
+ return GryoOutputFormat.class.getName();
+ else if (file.endsWith(".json"))
+ return GraphSONOutputFormat.class.getName();
+ else if (file.endsWith(".xml"))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
index 6d3899e..cbc9b07 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -55,7 +55,7 @@ public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalS
final HadoopIoStep hadoopIoStep = new HadoopIoStep(traversal, readWriting.getFile());
hadoopIoStep.setMode(readWriting.getMode());
readWriting.getParameters().getRaw().entrySet().forEach(kv ->
- hadoopIoStep.configure(kv.getKey(), kv.getValue())
+ hadoopIoStep.configure(kv.getKey(), kv.getValue().get(0))
);
TraversalHelper.replaceStep((Step) readWriting, hadoopIoStep, traversal);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 14c5360..9ec0cfd 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -142,6 +142,22 @@ import java.util.stream.Stream;
method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals",
+ method = "g_io_readXxmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals",
+ method = "g_io_read_withXreader_graphmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals",
+ method = "g_io_writeXxmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals",
+ method = "g_io_write_withXwriter_graphmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
public final class HadoopGraph implements Graph {
public static final Logger LOGGER = LoggerFactory.getLogger(HadoopGraph.class);
[05/50] tinkerpop git commit: TINKERPOP-1996 Removed use of
graph.io() in docs
Posted by sp...@apache.org.
TINKERPOP-1996 Removed use of graph.io() in docs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/5bf19e2d
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/5bf19e2d
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/5bf19e2d
Branch: refs/heads/TINKERPOP-1967
Commit: 5bf19e2d35aa2f050ecca0a7b2190553b0422720
Parents: 6d05805
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 16 09:29:01 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:10 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/centrality.asciidoc | 2 +-
docs/src/reference/implementations-neo4j.asciidoc | 5 +++--
.../src/reference/implementations-tinkergraph.asciidoc | 8 ++++----
docs/src/reference/the-traversal.asciidoc | 13 +++++++------
4 files changed, 15 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5bf19e2d/docs/src/recipes/centrality.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/centrality.asciidoc b/docs/src/recipes/centrality.asciidoc
index 8504589..0c1d4bc 100644
--- a/docs/src/recipes/centrality.asciidoc
+++ b/docs/src/recipes/centrality.asciidoc
@@ -156,7 +156,7 @@ give it the highest rank. Consider the following example using the Grateful Dead
[gremlin-groovy]
----
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
+g.io('data/grateful-dead.xml').read()
g.V().repeat(groupCount('m').by('name').out()).times(5).cap('m'). <1>
order(local).by(values, desc).limit(local, 10).next() <2>
g.V().repeat(groupCount('m').by('name').out().timeLimit(100)).times(5).cap('m'). <3>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5bf19e2d/docs/src/reference/implementations-neo4j.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-neo4j.asciidoc b/docs/src/reference/implementations-neo4j.asciidoc
index 7e05221..1760bd6 100644
--- a/docs/src/reference/implementations-neo4j.asciidoc
+++ b/docs/src/reference/implementations-neo4j.asciidoc
@@ -93,8 +93,8 @@ labels), a linear scan of the vertex-label partition is still faster than a line
[gremlin-groovy]
----
graph = Neo4jGraph.open('/tmp/neo4j')
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
g = graph.traversal()
+g.io('data/grateful-dead.xml').read()
g.tx().commit()
clock(1000) {g.V().hasLabel('artist').has('name','Garcia').iterate()} <1>
graph.cypher("CREATE INDEX ON :artist(name)") <2>
@@ -161,7 +161,8 @@ It is possible to leverage Cypher from within Gremlin by using the `Neo4jGraph.c
[gremlin-groovy]
----
graph = Neo4jGraph.open('/tmp/neo4j')
-graph.io(gryo()).readGraph('data/tinkerpop-modern.kryo')
+g = graph.traversal()
+g.io('data/tinkerpop-modern.kryo').read()
graph.cypher('MATCH (a {name:"marko"}) RETURN a')
graph.cypher('MATCH (a {name:"marko"}) RETURN a').select('a').out('knows').values('name')
graph.close()
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5bf19e2d/docs/src/reference/implementations-tinkergraph.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
index 5b04126..acd37dd 100644
--- a/docs/src/reference/implementations-tinkergraph.asciidoc
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -90,12 +90,12 @@ TinkerGraph over the Grateful Dead graph.
----
graph = TinkerGraph.open()
g = graph.traversal()
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
+g.io('data/grateful-dead.xml').read()
clock(1000) {g.V().has('name','Garcia').iterate()} <1>
graph = TinkerGraph.open()
g = graph.traversal()
graph.createIndex('name',Vertex.class)
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
+g.io('data/grateful-dead.xml').read()
clock(1000){g.V().has('name','Garcia').iterate()} <2>
----
@@ -158,13 +158,13 @@ cardinality to `list` or else the data will import as `single`. Consider the fo
[gremlin-groovy]
----
graph = TinkerGraph.open()
-graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
g = graph.traversal()
+g.io("data/tinkerpop-crew.kryo").read()
g.V().properties()
conf = new BaseConfiguration()
conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality","list")
graph = TinkerGraph.open(conf)
-graph.io(gryo()).readGraph("data/tinkerpop-crew.kryo")
g = graph.traversal()
+g.io("data/tinkerpop-crew.kryo").read()
g.V().properties()
----
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5bf19e2d/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 34f6b27..cca71f6 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -400,7 +400,8 @@ made more salient on a larger graph. Therefore, the example below leverages the
[gremlin-groovy]
----
graph = TinkerGraph.open()
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
+g = graph.traversal()
+g.io('data/grateful-dead.xml').read()
g = graph.traversal().withoutStrategies(LazyBarrierStrategy) <1>
clockWithResult(1){g.V().both().both().both().count().next()} <2>
clockWithResult(1){g.V().repeat(both()).times(3).count().next()} <3>
@@ -422,8 +423,8 @@ optimization scenario with the added benefit of reducing the risk of an out-of-m
[gremlin-groovy]
----
graph = TinkerGraph.open()
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
-g = graph.traversal() <1>
+g = graph.traversal() <1>
+g.io('data/grateful-dead.xml').read()
clockWithResult(1){g.V().both().both().both().count().next()}
g.V().both().both().both().count().iterate().toString() <2>
----
@@ -1401,8 +1402,8 @@ songs which Jerry Garcia has both sung and written (using the Grateful Dead grap
[gremlin-groovy]
----
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
g = graph.traversal()
+g.io('data/grateful-dead.xml').read()
g.V().match(
__.as('a').has('name', 'Garcia'),
__.as('a').in('writtenBy').as('b'),
@@ -2493,8 +2494,8 @@ ranking.
[gremlin-groovy]
----
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
g = graph.traversal()
+g.io('data/grateful-dead.xml').read()
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
order(local).by(values,desc).limit(local, 5)
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
@@ -2507,8 +2508,8 @@ Similarly, for extracting the values from a path or map.
[gremlin-groovy]
----
-graph.io(graphml()).readGraph('data/grateful-dead.xml')
g = graph.traversal()
+g.io('data/grateful-dead.xml').read()
g.V().hasLabel('song').out('sungBy').groupCount().by('name') <1>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values) <2>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values).unfold().
[39/50] tinkerpop git commit: Merge branch 'TINKERPOP-1996'
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-1996'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/edd82346
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/edd82346
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/edd82346
Branch: refs/heads/TINKERPOP-1967
Commit: edd8234684bedb7c073479724b2ee645a801cbc5
Parents: 7d21ee0 10478be
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Jul 31 07:35:52 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 07:35:52 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 4 +
docs/src/recipes/centrality.asciidoc | 2 +-
.../reference/implementations-neo4j.asciidoc | 5 +-
.../implementations-tinkergraph.asciidoc | 8 +-
docs/src/reference/the-graph.asciidoc | 370 -------------------
docs/src/reference/the-traversal.asciidoc | 186 +++++++++-
docs/src/upgrade/release-3.4.x.asciidoc | 49 ++-
.../tinkerpop/gremlin/jsr223/CoreImports.java | 2 +
.../step/map/TraversalVertexProgramStep.java | 4 -
.../decoration/VertexProgramStrategy.java | 11 +-
.../tinkerpop/gremlin/process/traversal/IO.java | 83 +++++
.../traversal/dsl/graph/GraphTraversal.java | 37 ++
.../dsl/graph/GraphTraversalSource.java | 22 ++
.../process/traversal/step/ReadWriting.java | 48 +++
.../traversal/step/sideEffect/IoStep.java | 254 +++++++++++++
.../process/traversal/step/util/Parameters.java | 4 +-
.../ComputerVerificationStrategy.java | 3 +-
.../StandardVerificationStrategy.java | 9 +
.../tinkerpop/gremlin/structure/Graph.java | 30 ++
.../tinkerpop/gremlin/structure/io/IoCore.java | 3 +
.../traversal/dsl/graph/GraphTraversalTest.java | 5 +-
.../decoration/VertexProgramStrategyTest.java | 2 +-
.../StandardVerificationStrategyTest.java | 13 +-
.../glv/GraphTraversalSource.template | 4 +-
gremlin-dotnet/glv/IO.template | 46 +++
gremlin-dotnet/glv/generate.groovy | 13 +-
.../Process/Traversal/GraphTraversal.cs | 15 +-
.../Process/Traversal/GraphTraversalSource.cs | 11 +
.../src/Gremlin.Net/Process/Traversal/IO.cs | 56 +++
.../Gherkin/TraversalEvaluation/IOParameter.cs | 82 ++++
.../TraversalEvaluation/TraversalParser.cs | 7 +
.../groovy/jsr223/GroovyTranslatorProvider.java | 6 +
gremlin-javascript/glv/TraversalSource.template | 9 +
gremlin-javascript/glv/generate.groovy | 4 +
.../lib/process/graph-traversal.js | 30 ++
.../gremlin-javascript/lib/process/traversal.js | 29 ++
.../test/cucumber/feature-steps.js | 2 +
gremlin-python/glv/TraversalSource.template | 11 +
gremlin-python/glv/generate.groovy | 4 +
.../gremlin_python/process/graph_traversal.py | 13 +
.../jython/gremlin_python/process/traversal.py | 21 ++
.../src/main/jython/radish/feature_steps.py | 5 +-
gremlin-test/features/sideEffect/Read.feature | 84 +++++
gremlin-test/features/sideEffect/Write.feature | 60 +++
.../gremlin/AbstractGraphProvider.java | 22 +-
.../apache/tinkerpop/gremlin/TestHelper.java | 24 +-
.../gremlin/process/ProcessComputerSuite.java | 4 +
.../gremlin/process/ProcessStandardSuite.java | 4 +
.../process/traversal/step/map/ReadTest.java | 188 ++++++++++
.../process/traversal/step/map/WriteTest.java | 183 +++++++++
.../ElementIdStrategyProcessTest.java | 1 -
.../gremlin/structure/io/util/CustomId.java | 39 ++
.../gremlin/process/FeatureCoverageTest.java | 6 +-
.../traversal/step/sideEffect/HadoopIoStep.java | 188 ++++++++++
.../traversal/strategy/HadoopIoStrategy.java | 83 +++++
.../gremlin/hadoop/structure/HadoopGraph.java | 24 ++
.../step/sideEffect/TinkerGraphIoStepTest.java | 98 +++++
.../structure/TinkerGraphPlayTest.java | 6 +-
...ctTinkerGraphGraphSONTranslatorProvider.java | 5 +
.../gryo/TinkerGraphGryoTranslatorProvider.java | 7 +-
60 files changed, 2121 insertions(+), 427 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/edd82346/CHANGELOG.asciidoc
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/edd82346/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/edd82346/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --cc docs/src/upgrade/release-3.4.x.asciidoc
index 2d699dc,8d87397..9951024
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@@ -31,12 -31,12 +31,12 @@@ Please see the link:https://github.com/
==== with() Step
-This version of TinkerPop introduces the `with()` step to Gremlin. It isn't really a step but is instead a step
+This version of TinkerPop introduces the `with()`-step to Gremlin. It isn't really a step but is instead a step
modulator. This modulator allows the step it is modifying to accept configurations that can be used to alter the
-behavior of the step itself. A good example of its usage is shown with the revised syntax of the `pageRank()` step
+behavior of the step itself. A good example of its usage is shown with the revised syntax of the `pageRank()`-step
which now uses `with()` to replace the old `by()` options:
- [groovy]
+ [source,groovy]
----
g.V().hasLabel('person').
pageRank().
@@@ -47,9 -47,9 +47,9 @@@
valueMap('name','friendRank')
----
-A similar change was made for `peerPressure()` step:
+A similar change was made for `peerPressure()`-step:
- [groovy]
+ [source,groovy]
----
g.V().hasLabel('person').
peerPressure().
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/edd82346/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/edd82346/gremlin-javascript/src/main/javascript/gremlin-javascript/test/cucumber/feature-steps.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/edd82346/gremlin-python/src/main/jython/radish/feature_steps.py
----------------------------------------------------------------------
[16/50] tinkerpop git commit: TINKERPOP-1996 Fixed bad test
assertions after last body of changes.
Posted by sp...@apache.org.
TINKERPOP-1996 Fixed bad test assertions after last body of changes.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/94233970
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/94233970
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/94233970
Branch: refs/heads/TINKERPOP-1967
Commit: 94233970f8dc7c5d7b8fdc4b9e7bdbf0ff905c25
Parents: ff71c6a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 06:31:42 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 06:31:42 2018 -0400
----------------------------------------------------------------------
.../gremlin/process/traversal/step/map/WriteTest.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/94233970/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
index 55e6f9c..9798095 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
@@ -65,7 +65,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
final Traversal<Object,Object> traversal = get_g_io_writeXkryoX(fileToWrite);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
assertThat(f.length() > 0, is(true));
}
@@ -81,7 +81,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
final Traversal<Object,Object> traversal = get_g_io_write_withXwriter_gryoX(fileToWrite);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
assertThat(f.length() > 0, is(true));
}
@@ -97,7 +97,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
final Traversal<Object,Object> traversal = get_g_io_writeXjsonX(fileToWrite);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
assertThat(f.length() > 0, is(true));
}
@@ -113,7 +113,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
final Traversal<Object,Object> traversal = get_g_io_write_withXwriter_graphsonX(fileToWrite);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
assertThat(f.length() > 0, is(true));
}
@@ -129,7 +129,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
final Traversal<Object,Object> traversal = get_g_io_writeXxmlX(fileToWrite);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
assertThat(f.length() > 0, is(true));
}
[19/50] tinkerpop git commit: TINKERPOP-1996 Fixed up general
strategy application around io()
Posted by sp...@apache.org.
TINKERPOP-1996 Fixed up general strategy application around io()
The GraphComputer was not being set properly in the HadoopIoStep and therefore executions of OLAP runs would not work even if withComputer(SparkGraphComputer) was set. It only worked if the gremlin.hadoop.defaultGraphComputer property was set which was weird.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/c97d747f
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/c97d747f
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/c97d747f
Branch: refs/heads/TINKERPOP-1967
Commit: c97d747f4b05cef42eb0e53adae50fcdee083a2e
Parents: 4d979cf
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 10:30:00 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 10:30:00 2018 -0400
----------------------------------------------------------------------
.../step/map/TraversalVertexProgramStep.java | 4 ---
.../decoration/VertexProgramStrategy.java | 30 +++++++++-----------
.../traversal/step/sideEffect/IoStep.java | 2 --
.../traversal/strategy/HadoopIoStrategy.java | 28 ++++++++++++------
4 files changed, 33 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c97d747f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
index 4eb950f..30cfee5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
@@ -19,19 +19,15 @@
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.finalization.ComputerFinalizationStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
-import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c97d747f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
index cb99652..fa6e23f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java
@@ -101,23 +101,21 @@ public final class VertexProgramStrategy extends AbstractTraversalStrategy<Trave
// wrap all non-VertexComputing steps into a TraversalVertexProgramStep
currentStep = traversal.getStartStep();
- if (!(currentStep instanceof ReadWriting)) {
+ while (!(currentStep instanceof EmptyStep)) {
+ final Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>();
+ final Step<?, ?> firstLegalOLAPStep = getFirstLegalOLAPStep(currentStep);
+ final Step<?, ?> lastLegalOLAPStep = getLastLegalOLAPStep(currentStep);
+ if (!(firstLegalOLAPStep instanceof EmptyStep)) {
+ final int index = TraversalHelper.stepIndex(firstLegalOLAPStep, traversal);
+ TraversalHelper.removeToTraversal(firstLegalOLAPStep, lastLegalOLAPStep.getNextStep(), (Traversal.Admin) computerTraversal);
+ final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, computerTraversal);
+ traversal.addStep(index, traversalVertexProgramStep);
+ }
+ currentStep = traversal.getStartStep();
while (!(currentStep instanceof EmptyStep)) {
- final Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>();
- final Step<?, ?> firstLegalOLAPStep = getFirstLegalOLAPStep(currentStep);
- final Step<?, ?> lastLegalOLAPStep = getLastLegalOLAPStep(currentStep);
- if (!(firstLegalOLAPStep instanceof EmptyStep)) {
- final int index = TraversalHelper.stepIndex(firstLegalOLAPStep, traversal);
- TraversalHelper.removeToTraversal(firstLegalOLAPStep, lastLegalOLAPStep.getNextStep(), (Traversal.Admin) computerTraversal);
- final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, computerTraversal);
- traversal.addStep(index, traversalVertexProgramStep);
- }
- currentStep = traversal.getStartStep();
- while (!(currentStep instanceof EmptyStep)) {
- if (!(currentStep instanceof VertexComputing))
- break;
- currentStep = currentStep.getNextStep();
- }
+ if (!(currentStep instanceof VertexComputing))
+ break;
+ currentStep = currentStep.getNextStep();
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c97d747f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
index 1d4f40b..e5b3a6a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
@@ -18,8 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect;
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c97d747f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
index 8348410..af3db6e 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.sideEffect.HadoopIoStep;
import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
@@ -30,6 +31,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* The default implementation of the {@link IoStep} is a single threaded operation and doesn't properly take into
* account the method by which OLAP read/writes take place with Hadoop. This strategy removes that step and replaces
@@ -48,16 +54,20 @@ public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalS
@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
- // replace IoStep steps with hadoop specific one
- if (traversal.getStartStep() instanceof IoStep) {
- final ReadWriting readWriting = (ReadWriting) traversal.getStartStep();
- final HadoopIoStep hadoopIoStep = new HadoopIoStep(traversal, readWriting.getFile());
- hadoopIoStep.setMode(readWriting.getMode());
- readWriting.getParameters().getRaw().entrySet().forEach(kv ->
- hadoopIoStep.configure(kv.getKey(), kv.getValue().get(0))
- );
+ // VertexProgramStrategy should wrap up the IoStep in a TraversalVertexProgramStep. use that to grab the
+ // GraphComputer that was injected in there and push that in to the HadoopIoStep. this step pattern match
+ // is fairly specific and since you really can't chain together steps after io() this approach should work
+ if (traversal.getStartStep() instanceof TraversalVertexProgramStep) {
+ final TraversalVertexProgramStep tvp = (TraversalVertexProgramStep) traversal.getStartStep();
+ if (tvp.computerTraversal.get().getStartStep() instanceof ReadWriting) {
+ final ReadWriting readWriting = (ReadWriting) tvp.computerTraversal.get().getStartStep();
+ final HadoopIoStep hadoopIoStep = new HadoopIoStep(traversal, readWriting.getFile());
+ hadoopIoStep.setMode(readWriting.getMode());
+ hadoopIoStep.setComputer(tvp.getComputer());
+ readWriting.getParameters().getRaw().forEach((key, value) -> value.forEach(v -> hadoopIoStep.configure(key, v)));
- TraversalHelper.replaceStep((Step) readWriting, hadoopIoStep, traversal);
+ TraversalHelper.replaceStep(tvp, hadoopIoStep, traversal);
+ }
}
}
[34/50] tinkerpop git commit: Merge branch 'tp32' into tp33
Posted by sp...@apache.org.
Merge branch 'tp32' into tp33
Conflicts:
gremlin-server/src/main/bin/gremlin-server.sh
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/88b6e143
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/88b6e143
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/88b6e143
Branch: refs/heads/TINKERPOP-1967
Commit: 88b6e143dd81134074c17e5fa098f03972514273
Parents: 7985106 2b045f3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 07:16:27 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Jul 30 07:16:27 2018 -0400
----------------------------------------------------------------------
gremlin-server/src/main/bin/gremlin-server.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/88b6e143/gremlin-server/src/main/bin/gremlin-server.sh
----------------------------------------------------------------------
diff --cc gremlin-server/src/main/bin/gremlin-server.sh
index fec465c,06b065a..a09e10c
--- a/gremlin-server/src/main/bin/gremlin-server.sh
+++ b/gremlin-server/src/main/bin/gremlin-server.sh
@@@ -90,191 -49,14 +90,191 @@@ els
fi
# Set Java options
-if [ "$JAVA_OPTIONS" = "" ] ; then
+if [[ "$JAVA_OPTIONS" = "" ]] ; then
- JAVA_OPTIONS="-Xms32m -Xmx512m"
+ JAVA_OPTIONS="-Xms512m -Xmx4096m"
fi
-# Execute the application and return its exit code
-if [ "$1" = "-i" ]; then
- shift
- exec $JAVA -Dlog4j.configuration=conf/log4j-server.properties $JAVA_OPTIONS -cp $CP:$CLASSPATH org.apache.tinkerpop.gremlin.server.util.GremlinServerInstall "$@"
-else
- exec $JAVA -Dlog4j.configuration=conf/log4j-server.properties $JAVA_OPTIONS -cp $CP:$CLASSPATH org.apache.tinkerpop.gremlin.server.GremlinServer "$@"
-fi
+# Build Java CLASSPATH
+CP="$GREMLIN_HOME/conf/"
+CP="$CP":$( echo $GREMLIN_HOME/lib/*.jar . | sed 's/ /:/g')
+CP="$CP":$( find -L "$GREMLIN_HOME"/ext -mindepth 1 -maxdepth 1 -type d | \
+ sort | sed 's/$/\/plugin\/*/' | tr '\n' ':' )
+
+CLASSPATH="${CLASSPATH:-}:$CP"
+
+GREMLIN_SERVER_CMD=org.apache.tinkerpop.gremlin.server.GremlinServer
+GREMLIN_INSTALL_CMD=org.apache.tinkerpop.gremlin.server.util.GremlinServerInstall
+
+
+isRunning() {
+ if [[ -r "$PID_FILE" ]] ; then
+ PID=$(cat "$PID_FILE")
+ ps -p "$PID" &> /dev/null
+ return $?
+ else
+ return 1
+ fi
+}
+
+status() {
+ isRunning
+ RUNNING=$?
+ if [[ $RUNNING -gt 0 ]]; then
+ echo Server not running
+ else
+ echo Server running with PID $(cat "$PID_FILE")
+ fi
+}
+
+stop() {
+ isRunning
+ RUNNING=$?
+ if [[ $RUNNING -gt 0 ]]; then
+ echo Server not running
+ rm -f "$PID_FILE"
+ else
+ kill "$PID" &> /dev/null || { echo "Unable to kill server [$PID]"; exit 1; }
+ for i in $(seq 1 60); do
+ ps -p "$PID" &> /dev/null || { echo "Server stopped [$PID]"; rm -f "$PID_FILE"; return 0; }
+ [[ $i -eq 30 ]] && kill "$PID" &> /dev/null
+ sleep 1
+ done
+ echo "Unable to kill server [$PID]";
+ exit 1;
+ fi
+}
+
+start() {
+ isRunning
+ RUNNING=$?
+ if [[ $RUNNING -eq 0 ]]; then
+ echo Server already running with PID $(cat "$PID_FILE").
+ exit 1
+ fi
+
+ if [[ -z "$RUNAS" ]]; then
+
+ mkdir -p "$LOG_DIR" &>/dev/null
+ if [[ ! -d "$LOG_DIR" ]]; then
+ echo ERROR: LOG_DIR $LOG_DIR does not exist and could not be created.
+ exit 1
+ fi
+
+ mkdir -p "$PID_DIR" &>/dev/null
+ if [[ ! -d "$PID_DIR" ]]; then
+ echo ERROR: PID_DIR $PID_DIR does not exist and could not be created.
+ exit 1
+ fi
+
+ $JAVA -Dlog4j.configuration=$LOG4J_CONF $JAVA_OPTIONS -cp $CP:$CLASSPATH $GREMLIN_SERVER_CMD "$GREMLIN_YAML" >> "$LOG_FILE" 2>&1 &
+ PID=$!
+ disown $PID
+ echo $PID > "$PID_FILE"
+ else
+
+ su -c "mkdir -p $LOG_DIR &>/dev/null" "$RUNAS"
+ if [[ ! -d "$LOG_DIR" ]]; then
+ echo ERROR: LOG_DIR $LOG_DIR does not exist and could not be created.
+ exit 1
+ fi
+
+ su -c "mkdir -p $PID_DIR &>/dev/null" "$RUNAS"
+ if [[ ! -d "$PID_DIR" ]]; then
+ echo ERROR: PID_DIR $PID_DIR does not exist and could not be created.
+ exit 1
+ fi
+
+ su -c "$JAVA -Dlog4j.configuration=$LOG4J_CONF $JAVA_OPTIONS -cp $CP:$CLASSPATH $GREMLIN_SERVER_CMD \"$GREMLIN_YAML\" >> \"$LOG_FILE\" 2>&1 & echo \$! " "$RUNAS" > "$PID_FILE"
+ chown "$RUNAS" "$PID_FILE"
+ fi
+
+ isRunning
+ RUNNING=$?
+ if [[ $RUNNING -eq 0 ]]; then
+ echo Server started $(cat "$PID_FILE").
+ exit 0
+ else
+ echo Server failed
+ exit 1
+ fi
+
+}
+
+startForeground() {
+ isRunning
+ RUNNING=$?
+ if [[ $RUNNING -eq 0 ]]; then
+ echo Server already running with PID $(cat "$PID_FILE").
+ exit 1
+ fi
+
+ if [[ -z "$RUNAS" ]]; then
+ $JAVA -Dlog4j.configuration=$LOG4J_CONF $JAVA_OPTIONS -cp $CP:$CLASSPATH $GREMLIN_SERVER_CMD "$GREMLIN_YAML"
+ exit 0
+ else
+ echo Starting in foreground not supported with RUNAS
+ exit 1
+ fi
+
+}
+
+install() {
+
+ isRunning
+ RUNNING=$?
+ if [[ $RUNNING -eq 0 ]]; then
+ echo Server must be stopped before installing.
+ exit 1
+ fi
+
+ echo Installing dependency $@
+
+ DEPS="$@"
+ if [[ -z "$RUNAS" ]]; then
+ $JAVA -Dlog4j.configuration=$LOG4J_CONF $JAVA_OPTIONS -cp $CP:$CLASSPATH $GREMLIN_INSTALL_CMD $DEPS
+ else
+ su -c "$JAVA -Dlog4j.configuration=$LOG4J_CONF $JAVA_OPTIONS -cp $CP:$CLASSPATH $GREMLIN_INSTALL_CMD $DEPS " "$RUNAS"
+ fi
+
+}
+
+case "$1" in
+ status)
+ status
+ ;;
+ restart)
+ stop
+ start
+ ;;
+ start)
+ start
+ ;;
+ stop)
+ stop
+ ;;
+ -i)
+ shift
+ echo "Redirecting to 'install $@' (-i will be removed in a future release)"
+ install "$@"
+ ;;
+ install)
+ shift
+ install "$@"
+ ;;
+ console)
+ startForeground
+ ;;
+ *)
+ if [[ -n "$1" ]] ; then
+ if [[ -r "$1" ]]; then
+ GREMLIN_YAML="$1"
+ startForeground
+ elif [[ -r "$GREMLIN_HOME/$1" ]] ; then
+ GREMLIN_YAML="$GREMLIN_HOME/$1"
+ startForeground
+ fi
+ echo Configuration file not found.
+ fi
+ echo "Usage: $0 {start|stop|restart|status|console|install <group> <artifact> <version>|<conf file>}"; exit 1;
+ ;;
+esac
[26/50] tinkerpop git commit: TINKERPOP-1996 Added IoStep to list of
unsupported steps
Posted by sp...@apache.org.
TINKERPOP-1996 Added IoStep to list of unsupported steps
If this isn't there then GraphReader/Writer will blow up as it tries to mutate the graph. IoStep is an OLTP only step. For OLAP each graph implementation will need to add its own GraphComputer-ready step.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/e9ebacfc
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/e9ebacfc
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/e9ebacfc
Branch: refs/heads/TINKERPOP-1967
Commit: e9ebacfc84aca0e0ac30720bba68939f77c4c5d0
Parents: 23c71b6
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 15:08:45 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 15:08:45 2018 -0400
----------------------------------------------------------------------
.../strategy/verification/ComputerVerificationStrategy.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/e9ebacfc/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
index 2d076b6..5c2d299 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.Mutating;
import org.apache.tinkerpop.gremlin.process.traversal.step.PathProcessor;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.InjectStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
@@ -46,7 +47,7 @@ public final class ComputerVerificationStrategy extends AbstractTraversalStrateg
private static final ComputerVerificationStrategy INSTANCE = new ComputerVerificationStrategy();
private static final Set<Class<?>> UNSUPPORTED_STEPS = new HashSet<>(Arrays.asList(
- InjectStep.class, Mutating.class, SubgraphStep.class, ComputerResultStep.class
+ InjectStep.class, Mutating.class, SubgraphStep.class, ComputerResultStep.class, IoStep.class
));
private ComputerVerificationStrategy() {
[31/50] tinkerpop git commit: Merge branch 'tp32' into tp33
Posted by sp...@apache.org.
Merge branch 'tp32' into tp33
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/79851067
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/79851067
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/79851067
Branch: refs/heads/TINKERPOP-1967
Commit: 798510679b5af84ffc1ecdfa5ef86d52229cbc99
Parents: 9b693c0 3b8c828
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 27 16:29:37 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 27 16:29:37 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/element-existence.asciidoc | 91 ++++++++++++++++++++++++
docs/src/recipes/index.asciidoc | 2 +
2 files changed, 93 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/79851067/docs/src/recipes/index.asciidoc
----------------------------------------------------------------------
[43/50] tinkerpop git commit: Merged vtslab recipe for connected
components
Posted by sp...@apache.org.
Merged vtslab recipe for connected components
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/3298474a
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/3298474a
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/3298474a
Branch: refs/heads/TINKERPOP-1967
Commit: 3298474afc452de3ce9d2c04f6b8fbd3e3f6c744
Parents: 2e6b45a
Author: HadoopMarc <vt...@xs4all.nl>
Authored: Mon May 21 14:03:54 2018 +0200
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/connected-components.asciidoc | 123 +++++++++++++-------
1 file changed, 83 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/3298474a/docs/src/recipes/connected-components.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/connected-components.asciidoc b/docs/src/recipes/connected-components.asciidoc
index 70abdbd..edbeec5 100644
--- a/docs/src/recipes/connected-components.asciidoc
+++ b/docs/src/recipes/connected-components.asciidoc
@@ -14,17 +14,31 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
////
+
+// @author Daniel Kuppitz (anwer on gremlin user list)
+// @author Robert Dale (answer on gremlin user list)
+// @author Marc de Lignie
+
[[connected-components]]
== Connected Components
Gremlin can be used to find link:https://en.wikipedia.org/wiki/Connected_component_(graph_theory)[connected components]
-in a graph. As of TinkerPop 3.4.0, the process has been simplified to the `connectedComponent()`-step which is
-described in more detail in the link:
-link:http://tinkerpop.apache.org/docs/x.y.z/reference/#connectedcomponent-step[Reference Documentation].
+in a graph. In a directed graph like in TinkerPop, components can be weakly or strongly connected. This recipe is
+restricted to finding link:https://en.wikipedia.org/wiki/Directed_graph#Directed_graph_connectivity[weakly
+connected components], in which the direction of edges is not taken into account.
+
+Depending on the size of the graph, three solution regimes can be discriminated:
+
+1. Small graphs that fit in the memory of a single machine
+
+2. Medium graphs backed by storage for which a linear scan is still feasible. This regime is left to third party
+TinkerPop implementations, since TinkerPop itself has no storage-backed reference implementations. The idea is that
+component membership is stored in the graph, rather than in memory.
-The `connectedComponent()`-step replaces the original recipe described below from earlier versions of TinkerPop,
-however the algorithm from that old recipe remains interesting for educational purposes and has thus not been removed.
-The original recipe considers the following graph which has three connected components:
+3. Large graphs requiring an OLAP approach to yield results in a reasonable time.
+
+
+These regimes are discussed separately using the following graph with three weakly connected components:
image:connected-components.png[width=600]
@@ -41,46 +55,75 @@ g.addV().property(id, "A").as("a").
addE("link").from("d").to("e").iterate()
----
-One way to detect the various subgraphs would be to do something like this:
+
+===== Small graphs
+
+Connected components in a small graph can be determined with both an OLTP traversal and the OLAP
+`connectedComponent()`-step. The `connectedComponent()`-step is available as of TinkerPop 3.4.0 and is
+described in more detail in the
+link:http://tinkerpop.apache.org/docs/x.y.z/reference/#connectedcomponent-step[Reference Documentation].
+
+A straightforward way to detect the various subgraphs with an OLTP traversal is to do this:
[gremlin-groovy,existing]
----
-g.V().emit(cyclicPath().or().not(both())).repeat(both()).until(cyclicPath()). <1>
- path().aggregate("p"). <2>
- unfold().dedup(). <3>
- map(__.as("v").select("p").unfold(). <4>
- filter(unfold().where(eq("v"))).
- unfold().dedup().order().by(id).fold()).
- dedup() <5>
+g.V().emit(cyclicPath().or().not(both())). <1>
+ repeat(__.where(without('a')).store('a').both()).until(cyclicPath()). <2>
+ group().by(path().unfold().limit(1)). <3>
+ by(path().unfold().dedup().fold()). <4>
+ select(values).unfold() <5>
----
-<1> Iterate all vertices and repeatedly traverse over both incoming and outgoing edges (TinkerPop doesn't support
-unidirectional graphs directly so it must be simulated by ignoring the direction with `both`). Note the use of `emit`
-prior to `repeat` as this allows for return of a single length path.
-<2> Aggregate the `path()` of the emitted vertices to "p". It is within these paths that the list of connected
-components will be identified. Obviously the paths list are duplicative in the sense that they contains different
-paths traveled over the same vertices.
-<3> Unroll the elements in the path list with `unfold` and `dedup`.
-<4> Use the first vertex in each path to filter against the paths stored in "p". When a path is found that has the
-vertex in it, dedup the vertices in the path, order it by the identifier. Each path output from this `map` step
-represents a connected component.
-<5> The connected component list is duplicative given the nature of the paths in "p", but now that the vertices within
-the paths are ordered, a final `dedup` will make the list of connective components unique.
-
-NOTE: This is a nice example of where running smaller pieces of a large Gremlin statement make it easier to see what
-is happening at each step. Consider running this example one line at a time (or perhaps even in a step at a time) to
-see the output at each point.
-
-While the above approach returns results nicely, the traversal doesn't appear to work with OLAP. A less efficient
-approach, but one more suited for OLAP execution looks quite similar but does not use `dedup` as heavily (thus
-`GraphComputer` is forced to analyze far more paths):
+<1> The initial emit() step allows for output of isolated vertices, in addition to the discovery of
+components as described in (2).
+
+<2> The entire component to which the first returned vertex belongs, is visited. To allow for components of any
+structure, a repeat loop is applied that only stops for a particular branch of the component when it detects a cyclic
+path. Collection `'a'` is used to keep track of visited vertices, for both subtraversals within a component
+and new traversals resulting from the `g.V()` linear scan.
+
+<3> While `'a'` nicely keeps track of vertices already visited, the actual components need to be extracted from the
+path information of surviving traversers. The `path().unfold().limit(1)` closure provides the starting vertex
+of surviving traversers, which can be used to group the components.
+
+<4> This clause collects the unique vertices from all paths with the same starting vertex, thus from the same
+weak component.
+
+<5> The values of the groupby map contain the lists of vertices making up the requested components.
+
+This algorithm completes in linear time with the number of vertices and edges, because a traversal is started for each
+vertex and each edge with its associated out-vertex is visited exactly once.
+
+
+==== Large graphs
+
+Large graphs require an OLAP solution with a custom VertexProgram that can be run using a graph implementation's
+GraphComputer, in particular `SparkGraphComputer` on a `HadoopGraph`. The OLAP solution also runs faster for most
+medium-sized graphs, that is when these graph have a 'natural' structure with a limited maximum path length.
+
+The TinkerPop library of vertex programs contains the `WeakComponentsVertexProgram` which can be run in the same
+way as the link:http://tinkerpop.apache.org/docs/x.y.z/reference/#peerpressurevertexprogram[PeerPressureVertexProgram]:
[gremlin-groovy,existing]
----
-g.withComputer().V().emit(cyclicPath().or().not(both())).repeat(both()).until(cyclicPath()).
- aggregate("p").by(path()).cap("p").unfold().limit(local, 1).
- map(__.as("v").select("p").unfold().
- filter(unfold().where(eq("v"))).
- unfold().dedup().order().by(id).fold()
- ).toSet()
+result = g.getGraph().compute().
+ program(WeakComponentsVertexProgram.build().maxIterations(100).create()).
+ mapReduce(ClusterPopulationMapReduce.build().create()).
+ mapReduce(ClusterCountMapReduce.build().create()).
+ submit().get()
+result.memory().clusterPopulation
+gResult = result.graph().traversal()
+gResult.V().valueMap(true)
----
+
+The vertex program has interconnected vertices exchange id's and store the lowest id until no vertex receives a
+lower id. This algorithm is commonly applied in
+link:https://en.wikipedia.org/wiki/Bulk_synchronous_parallel[bulk synchronous parallel] systems, e.g. in
+link:https://spark.apache.org/graphx[Apache Spark GraphX].
+
+==== Scalability
+
+ToDo:
+ - limits and run time regime 1
+ - test of friendster graph regime 3
+ - discuss: link:http://www.vldb.org/pvldb/vol7/p1821-yan.pdf[http://www.vldb.org/pvldb/vol7/p1821-yan.pdf]
\ No newline at end of file
[29/50] tinkerpop git commit: TINKERPOP-1996 Fixed up typos in docs
Posted by sp...@apache.org.
TINKERPOP-1996 Fixed up typos in docs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/10478be0
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/10478be0
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/10478be0
Branch: refs/heads/TINKERPOP-1967
Commit: 10478be0c9cabb6b20723c3185464977767172bc
Parents: 38dc70d
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 27 07:10:41 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 27 07:10:41 2018 -0400
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 6 +++---
docs/src/upgrade/release-3.4.x.asciidoc | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/10478be0/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index c861d87..d096933 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1060,9 +1060,9 @@ job of the `io()`-step. By default, TinkerPop supports three formats for importi
NOTE: Additional documentation for TinkerPop IO formats can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/[IO Reference].
-By itself the `io()` step merely configures the kind of importing and exporting that is going
+By itself the `io()`-step merely configures the kind of importing and exporting that is going
to occur and it is the follow-on call to the `read()` or `write()` step that determines which of those actions will
-execute. Therefore, a typical usage of the `io()` step would look like this:
+execute. Therefore, a typical usage of the `io()`-step would look like this:
[source,java]
----
@@ -1073,7 +1073,7 @@ g.io(someOutputFile).write().iterate()
IMPORTANT: The commands above are still traversals and therefore require iteration to be executed, hence the use of
`iterate()` as a termination step.
-By default, the `io()` step will try to detect the right file format using the file name extension. To gain greater
+By default, the `io()`-step will try to detect the right file format using the file name extension. To gain greater
control of the format use the `with()` step modulator to provide further information to `io()`. For example:
[source,java]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/10478be0/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.4.x.asciidoc b/docs/src/upgrade/release-3.4.x.asciidoc
index 0bb3903..8d87397 100644
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@ -81,7 +81,7 @@ g.io(someInputFile).read().iterate()
g.io(someOutputFile).write().iterate()
----
-While `io()` step is still single-threaded for OLTP style loading, it can be utilized in conjunction with OLAP which
+While `io()`-step is still single-threaded for OLTP style loading, it can be utilized in conjunction with OLAP which
internally uses `CloneVertexProgram` and therefore any graph `InputFormat` or `OutputFormat` can be configured in
conjunction with this step for parallel loads of large datasets.
[20/50] tinkerpop git commit: TINKERPOP-1996 Undeprecated some Io
related classes
Posted by sp...@apache.org.
TINKERPOP-1996 Undeprecated some Io related classes
These classes still have use as part of IoRegistry which is still in use and I don't see a clear way to get rid of that easily. We'd have to change the whole system for serialization configuration to accomplish that so I guess this stuff stays for now.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/8fd3bf21
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/8fd3bf21
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/8fd3bf21
Branch: refs/heads/TINKERPOP-1967
Commit: 8fd3bf21efac092f619254245ba83614327c4dcb
Parents: c97d747
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 10:47:20 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 10:47:20 2018 -0400
----------------------------------------------------------------------
.../main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java | 3 ---
.../apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java | 3 ---
.../tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java | 3 ---
.../org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java | 3 ---
4 files changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8fd3bf21/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
index 0971e31..bae56c5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.structure.io;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.io.IOException;
@@ -32,9 +31,7 @@ import java.util.function.Consumer;
* internal {@link Mapper} (if the format has such capability).
*
* @author Stephen Mallette (http://stephen.genoprime.com)
- * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
-@Deprecated
public interface Io<R extends GraphReader.ReaderBuilder, W extends GraphWriter.WriterBuilder, M extends Mapper.Builder> {
/**
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8fd3bf21/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
index b6af646..88431bf 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.graphml;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
@@ -38,9 +37,7 @@ import java.util.function.Consumer;
* such things.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
- * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
-@Deprecated
public final class GraphMLIo implements Io<GraphMLReader.Builder, GraphMLWriter.Builder, GraphMLMapper.Builder> {
private final Graph graph;
private Optional<Consumer<Mapper.Builder>> onMapper;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8fd3bf21/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
index 7f8b835..a3923a1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.graphson;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
@@ -37,9 +36,7 @@ import java.util.function.Consumer;
* interfaces should see the {@link GraphSONMapper} for information on the expectations for the {@link IoRegistry}.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
- * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
-@Deprecated
public final class GraphSONIo implements Io<GraphSONReader.Builder, GraphSONWriter.Builder, GraphSONMapper.Builder> {
private final Graph graph;
private final Optional<Consumer<Mapper.Builder>> onMapper;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8fd3bf21/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
index a7dfe0d..29e63de 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
@@ -37,9 +36,7 @@ import java.util.function.Consumer;
* interfaces should see the {@link GryoMapper} for information on the expectations for the {@link IoRegistry}.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
- * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
-@Deprecated
public final class GryoIo implements Io<GryoReader.Builder, GryoWriter.Builder, GryoMapper.Builder> {
private final Graph graph;
[15/50] tinkerpop git commit: TINKERPOP-1996 Added support for
setting IoRegistries using with()
Posted by sp...@apache.org.
TINKERPOP-1996 Added support for setting IoRegistries using with()
IORegistry instances are important because they feed serializer information to the Reader/Writer instances. Of all the configuration options that one seemed like the most important to make possible using with().
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ff71c6ab
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ff71c6ab
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ff71c6ab
Branch: refs/heads/TINKERPOP-1967
Commit: ff71c6abee0b39d7ee95128c3d64906daad96a76
Parents: ae3b149
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 19 16:13:57 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 16:13:57 2018 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/process/traversal/IO.java | 7 ++
.../traversal/step/sideEffect/IoStep.java | 75 ++++++++++++--------
.../process/traversal/step/util/Parameters.java | 4 +-
.../Process/Traversal/GraphTraversal.cs | 21 ++----
.../src/Gremlin.Net/Process/Traversal/IO.cs | 2 +
.../gremlin-javascript/lib/process/traversal.js | 4 ++
.../jython/gremlin_python/process/traversal.py | 2 +
.../gremlin/structure/io/util/CustomId.java | 39 ++++++++++
.../step/sideEffect/TinkerGraphIoStepTest.java | 75 ++++++++++++++++++++
9 files changed, 181 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
index 6668cf1..67b4670 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
@@ -73,4 +74,10 @@ public class IO {
* the file extension provided to it.
*/
public static final String writer = Graph.Hidden.hide("tinkerpop.io.writer");
+
+ /**
+ * A key that identifies the fully qualified class names of {@link IoRegistry} instances to use. May be specified
+ * multiple times (i.e. once for each registry) using the {@link GraphTraversal#with(String, Object)} modulator.
+ */
+ public static final String registry = Graph.Hidden.hide("tinkerpop.io.registry");
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
index 9804333..1d4f40b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
@@ -31,10 +31,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementExce
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
@@ -46,6 +49,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
/**
* Handles read and write operations into the {@link Graph}.
@@ -138,15 +144,19 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
* extension or simply uses configurations provided by the user on the parameters given to the step.
*/
private GraphReader constructReader() {
- final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0);
+ final Object objectOrClass = parameters.get(IO.reader, this::detectFileType).get(0);
if (objectOrClass instanceof GraphReader)
return (GraphReader) objectOrClass;
else if (objectOrClass instanceof String) {
- if (objectOrClass.equals(IO.graphson))
- return GraphSONReader.build().create();
- else if (objectOrClass.equals(IO.gryo))
- return GryoReader.build().create();
- else if (objectOrClass.equals(IO.graphml))
+ if (objectOrClass.equals(IO.graphson)) {
+ final GraphSONMapper.Builder builder = GraphSONMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GraphSONReader.build().mapper(builder.create()).create();
+ } else if (objectOrClass.equals(IO.gryo)){
+ final GryoMapper.Builder builder = GryoMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GryoReader.build().mapper(builder.create()).create();
+ } else if (objectOrClass.equals(IO.graphml))
return GraphMLReader.build().create();
else {
try {
@@ -163,31 +173,24 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
}
}
- private GraphReader detectReader() {
- if (file.endsWith(".kryo"))
- return GryoReader.build().create();
- else if (file.endsWith(".json"))
- return GraphSONReader.build().create();
- else if (file.endsWith(".xml"))
- return GraphMLReader.build().create();
- else
- throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
- }
-
/**
* Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file
* extension or simply uses configurations provided by the user on the parameters given to the step.
*/
private GraphWriter constructWriter() {
- final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0);
+ final Object objectOrClass = parameters.get(IO.writer, this::detectFileType).get(0);
if (objectOrClass instanceof GraphWriter)
return (GraphWriter) objectOrClass;
else if (objectOrClass instanceof String) {
- if (objectOrClass.equals(IO.graphson))
- return GraphSONWriter.build().create();
- else if (objectOrClass.equals(IO.gryo))
- return GryoWriter.build().create();
- else if (objectOrClass.equals(IO.graphml))
+ if (objectOrClass.equals(IO.graphson)) {
+ final GraphSONMapper.Builder builder = GraphSONMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GraphSONWriter.build().mapper(builder.create()).create();
+ } else if (objectOrClass.equals(IO.gryo)){
+ final GryoMapper.Builder builder = GryoMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GryoWriter.build().mapper(builder.create()).create();
+ }else if (objectOrClass.equals(IO.graphml))
return GraphMLWriter.build().create();
else {
try {
@@ -204,21 +207,31 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
}
}
- private GraphWriter detectWriter() {
+ private String detectFileType() {
if (file.endsWith(".kryo"))
- return GryoWriter.build().create();
+ return IO.gryo;
else if (file.endsWith(".json"))
- return GraphSONWriter.build().create();
+ return IO.graphson;
else if (file.endsWith(".xml"))
- return GraphMLWriter.build().create();
+ return IO.graphml;
else
throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension");
}
- private Configuration getConfFromParameters() {
- final Configuration conf = new BaseConfiguration();
- parameters.getRaw().forEach((key, value) -> conf.setProperty(key.toString(), value.get(0)));
- return conf;
+ private List<IoRegistry> detectRegistries() {
+ final List<Object> k = parameters.get(IO.registry, Collections::emptyList);
+ return parameters.get(IO.registry, null).stream().map(cn -> {
+ try {
+ if (cn instanceof IoRegistry)
+ return (IoRegistry) cn;
+ else {
+ final Class<?> clazz = Class.forName(cn.toString());
+ return (IoRegistry) clazz.getMethod("instance").invoke(null);
+ }
+ } catch (Exception ex) {
+ throw new IllegalStateException(ex);
+ }
+ }).collect(Collectors.toList());
}
@Override
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
index 40d9330..eb57f4b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
@@ -100,11 +100,11 @@ public final class Parameters implements Cloneable, Serializable {
* Gets the value of a key and if that key isn't present returns the default value from the {@link Supplier}.
*
* @param key the key to retrieve
- * @param defaultValue the default value generator
+ * @param defaultValue the default value generator which if null will return an empty list
*/
public <E> List<E> get(final Object key, final Supplier<E> defaultValue) {
final List<E> list = (List<E>) this.parameters.get(key);
- return (null == list) ? Collections.singletonList(defaultValue.get()) : list;
+ return (null == list) ? (null == defaultValue ? Collections.emptyList() : Collections.singletonList(defaultValue.get())) : list;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
index 82d72c0..361b246 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
@@ -1256,20 +1256,20 @@ namespace Gremlin.Net.Process.Traversal
}
/// <summary>
- /// Adds the repeat step to this <see cref="GraphTraversal{SType, EType}" />.
+ /// Adds the read step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
- public GraphTraversal<S, E> Repeat (string loopName, ITraversal repeatTraversal)
+ public GraphTraversal<S, E> Read ()
{
- Bytecode.AddStep("repeat", loopName, repeatTraversal);
+ Bytecode.AddStep("read");
return Wrap<S, E>(this);
}
/// <summary>
- /// Adds the read step to this <see cref="GraphTraversal{SType, EType}" />.
+ /// Adds the repeat step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
- public GraphTraversal<S, E> Read ()
+ public GraphTraversal<S, E> Repeat (string loopName, ITraversal repeatTraversal)
{
- Bytecode.AddStep("read");
+ Bytecode.AddStep("repeat", loopName, repeatTraversal);
return Wrap<S, E>(this);
}
@@ -1713,15 +1713,6 @@ namespace Gremlin.Net.Process.Traversal
}
/// <summary>
- /// Adds the with step to this <see cref="GraphTraversal{SType, EType}" />.
- /// </summary>
- public GraphTraversal<S, E> With (string key, object value)
- {
- Bytecode.AddStep("with", key, value);
- return Wrap<S, E>(this);
- }
-
- /// <summary>
/// Adds the write step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
public GraphTraversal<S, E> Write ()
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
index 288f7e3..861b431 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
@@ -46,6 +46,8 @@ namespace Gremlin.Net.Process.Traversal
public const String reader = "~tinkerpop.io.reader";
+ public const String registry = "~tinkerpop.io.registry";
+
public const String writer = "~tinkerpop.io.writer";
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
----------------------------------------------------------------------
diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
index 3f69fb1..09aec91 100644
--- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
+++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
@@ -129,6 +129,10 @@ class IO {
return "~tinkerpop.io.reader"
}
+ static get registry() {
+ return "~tinkerpop.io.registry"
+ }
+
static get writer() {
return "~tinkerpop.io.writer"
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
index d9fb4d9..49bb7b1 100644
--- a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
+++ b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
@@ -319,6 +319,8 @@ class IO(object):
reader = "~tinkerpop.io.reader"
+ registry = "~tinkerpop.io.registry"
+
writer = "~tinkerpop.io.writer"
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
index d503ae8..0ab3b90 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
@@ -18,9 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.util;
+import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.Io;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException;
import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
import org.apache.tinkerpop.shaded.jackson.core.JsonParser;
@@ -31,11 +36,13 @@ import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer;
import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer;
import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer;
+import org.javatuples.Pair;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -66,6 +73,24 @@ public class CustomId {
}
@Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final CustomId customId = (CustomId) o;
+
+ if (!cluster.equals(customId.cluster)) return false;
+ return elementId.equals(customId.elementId);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = cluster.hashCode();
+ result = 31 * result + elementId.hashCode();
+ return result;
+ }
+
+ @Override
public String toString() {
return cluster + ":" + elementId;
}
@@ -219,4 +244,18 @@ public class CustomId {
return "simple";
}
}
+
+ public static class CustomIdIoRegistry extends AbstractIoRegistry {
+
+ private static final CustomIdIoRegistry INSTANCE = new CustomIdIoRegistry();
+
+ private CustomIdIoRegistry() {
+ register(GryoIo.class, CustomId.class, null);
+ register(GraphSONIo.class, null, new CustomIdTinkerPopJacksonModuleV3d0());
+ }
+
+ public static CustomIdIoRegistry instance() {
+ return INSTANCE;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
new file mode 100644
index 0000000..06c4db8
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.process.traversal.step.sideEffect;
+
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.util.CustomId;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * It was hard to test the {@link IO#registry} configuration as a generic test. Opted to test it as a bit of a
+ * standalone test with TinkerGraph.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class TinkerGraphIoStepTest {
+
+ private Graph graph;
+ private GraphTraversalSource g;
+
+ @Before
+ public void setup() {
+ graph = TinkerGraph.open();
+ g = graph.traversal();
+ }
+
+ @Test
+ public void shouldWriteReadWithCustomIoRegistryGryo() throws Exception {
+ final UUID uuid = UUID.randomUUID();
+ g.addV("person").property("name","stephen").property("custom", new CustomId("a", uuid)).iterate();
+
+ final File file = TestHelper.generateTempFile(TinkerGraphIoStepTest.class, "shouldWriteReadWithCustomIoRegistryGryo", ".kryo");
+ g.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.class.getName()).write().iterate();
+
+ final Graph emptyGraph = TinkerGraph.open();
+ final GraphTraversalSource emptyG = emptyGraph.traversal();
+
+ try {
+ emptyG.io(file.getAbsolutePath()).read().iterate();
+ fail("Can't read without a registry");
+ } catch (Exception ignored) {
+ // do nothing
+ }
+
+ emptyG.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.instance()).read().iterate();
+
+ assertEquals(1, emptyG.V().has("custom", new CustomId("a", uuid)).count().next().intValue());
+ }
+}
[42/50] tinkerpop git commit: Merge branch 'tp33'
Posted by sp...@apache.org.
Merge branch 'tp33'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/a2db39cc
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/a2db39cc
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/a2db39cc
Branch: refs/heads/TINKERPOP-1967
Commit: a2db39cc710dd2809bab3fb6b2db44257458dcc8
Parents: edd8234 c49c0cc
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Jul 31 08:35:34 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 08:35:34 2018 -0400
----------------------------------------------------------------------
docs/src/dev/developer/development-environment.asciidoc | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[47/50] tinkerpop git commit: TINKERPOP-1967 Added
connectedComponent() step
Posted by sp...@apache.org.
TINKERPOP-1967 Added connectedComponent() step
Deprecated the recipe for "Connected Components" but left the old content present as I felt it had educational value.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/2e6b45a3
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/2e6b45a3
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/2e6b45a3
Branch: refs/heads/TINKERPOP-1967
Commit: 2e6b45a3d4ce9d7953ca7818950e702604b11976
Parents: a2db39c
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu May 17 14:44:01 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/recipes/connected-components.asciidoc | 8 +-
docs/src/reference/the-graphcomputer.asciidoc | 6 +
docs/src/reference/the-traversal.asciidoc | 35 +++
docs/src/upgrade/release-3.4.x.asciidoc | 32 +++
.../tinkerpop/gremlin/jsr223/CoreImports.java | 4 +
.../ConnectedComponentVertexProgram.java | 234 +++++++++++++++++++
.../traversal/step/map/ConnectedComponent.java | 38 +++
.../ConnectedComponentVertexProgramStep.java | 98 ++++++++
.../gremlin/process/remote/RemoteGraph.java | 4 +
.../traversal/dsl/graph/GraphTraversal.java | 15 ++
.../traversal/dsl/graph/GraphTraversalTest.java | 2 +-
.../Process/Traversal/GraphTraversal.cs | 9 +
.../lib/process/graph-traversal.js | 10 +
.../gremlin_python/process/graph_traversal.py | 4 +
.../gremlin/process/ProcessComputerSuite.java | 2 +
.../step/map/ConnectedComponentTest.java | 117 ++++++++++
.../computer/SparkHadoopGraphProvider.java | 2 +
18 files changed, 619 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 430f52d..10255ba 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -35,6 +35,7 @@ This release also includes changes from <<release-3-3-3, 3.3.3>>.
* Replaced `Parameterizing.addPropertyMutations()` with `Configuring.configure()`.
* Changed interface hierarchy for `Parameterizing` and `Mutating` interfaces as they are tightly related.
* Introduced the `with()` step modulator which can supply configuration options to `Configuring` steps.
+* Added `connectedComponent()` step and related `VertexProgram`.
* Added `supportsUpsert()` option to `VertexFeatures` and `EdgeFeatures`.
* `min()` and `max()` now support all types implementing `Comparable`.
* Change the `toString()` of `Path` to be standardized as other graph elements are.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/docs/src/recipes/connected-components.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/connected-components.asciidoc b/docs/src/recipes/connected-components.asciidoc
index 46d61eb..70abdbd 100644
--- a/docs/src/recipes/connected-components.asciidoc
+++ b/docs/src/recipes/connected-components.asciidoc
@@ -18,7 +18,13 @@ limitations under the License.
== Connected Components
Gremlin can be used to find link:https://en.wikipedia.org/wiki/Connected_component_(graph_theory)[connected components]
-in a graph. Consider the following graph which has three connected components:
+in a graph. As of TinkerPop 3.4.0, the process has been simplified to the `connectedComponent()`-step which is
+described in more detail in the link:
+link:http://tinkerpop.apache.org/docs/x.y.z/reference/#connectedcomponent-step[Reference Documentation].
+
+The `connectedComponent()`-step replaces the original recipe described below from earlier versions of TinkerPop,
+however the algorithm from that old recipe remains interesting for educational purposes and has thus not been removed.
+The original recipe considers the following graph which has three connected components:
image:connected-components.png[width=600]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/docs/src/reference/the-graphcomputer.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-graphcomputer.asciidoc b/docs/src/reference/the-graphcomputer.asciidoc
index 1d7586c..9cd1c76 100644
--- a/docs/src/reference/the-graphcomputer.asciidoc
+++ b/docs/src/reference/the-graphcomputer.asciidoc
@@ -403,6 +403,12 @@ g.V().peerPressure().by('cluster').valueMap()
g.V().peerPressure().by(outE('knows')).by('cluster').valueMap()
----
+[[connectedcomponentvertexprogram]]
+=== ConnectedComponentVertexProgram
+
+The `ConnectedComponentVertexProgram` identifies link:https://en.wikipedia.org/wiki/Connected_component_(graph_theory)[Connected Component]
+instances in a graph. See <<connectedcomponent-step,`connectedComponent()`>>-step for more information.
+
[[bulkdumpervertexprogram]]
[[clonevertexprogram]]
=== CloneVertexProgram
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 7d385c8..4d5b48a 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -625,6 +625,41 @@ g.V().coin(1.0)
link:++http://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#coin-double-++[`coin(double)`]
+[[connectedcomponent-step]]
+=== ConnectedComponent Step
+
+The `connectedComponent()` step performs a computation to identify link:https://en.wikipedia.org/wiki/Connected_component_(graph_theory)[Connected Component]
+instances in a graph. When this step completes, the vertices will be labelled with a component identifier to denote
+the component to which they are associated.
+
+IMPORTANT: The `connectedComponent()`-step is a `VertexComputing`-step and as such, can only be used against a graph
+that supports `GraphComputer` (OLAP).
+
+[gremlin-groovy,modern]
+----
+g = graph.traversal().withComputer()
+g.V().
+ connectedComponent().
+ with(ConnectedComponent.propertyName, 'component').
+ project('name','component').
+ by('name').
+ by('component')
+g.V().hasLabel('person').
+ connectedComponent().
+ with(ConnectedComponent.propertyName, 'component').
+ with(ConnectedComponent.edges, outE('knows')).
+ project('name','component').
+ by('name').
+ by('component')
+----
+
+Note the use of the `with()` modulating step which provides configuration options to the algorithm. It takes
+configuration keys from the `ConnectedComponent` class and is automatically imported to the Gremlin Console.
+
+*Additional References*
+
+link:++http://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#connectedComponent--++[`connectedComponent()`]
+
[[constant-step]]
=== Constant Step
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.4.x.asciidoc b/docs/src/upgrade/release-3.4.x.asciidoc
index 9951024..85ba89b 100644
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@ -65,6 +65,38 @@ release where breaking changes are allowed.
See: link:https://issues.apache.org/jira/browse/TINKERPOP-1975[TINKERPOP-1975],
link:http://tinkerpop.apache.org/docs/3.4.0/reference/#with-step[Reference Documentation]
+==== connectedComponent() Step
+
+In prior version of TinkerPop, it was recommended that the identification of
+link:https://en.wikipedia.org/wiki/Connected_component_(graph_theory)[Connected Component] instances in a graph be
+computed by way of a reasonably complex bit of Gremlin that looked something like this:
+
+[source,groovy]
+----
+g.V().emit(cyclicPath().or().not(both())).repeat(both()).until(cyclicPath()).
+ path().aggregate("p").
+ unfold().dedup().
+ map(__.as("v").select("p").unfold().
+ filter(unfold().where(eq("v"))).
+ unfold().dedup().order().by(id).fold()).
+ dedup()
+----
+
+The above approach had a number of drawbacks that included a large execution cost as well as incompatibilities in OLAP.
+To simplify usage of this commonly use graph algorithm, TinkerPop 3.4.0 introduces the `connectedComponent()` step
+which reduces the above operation to:
+
+[source,groovy]
+----
+g.withComputer().V().connectedComponent()
+----
+
+It is important to note that this step does require the use of a `GraphComputer` to work, as it utilizes a
+`VertexProgram` behind the scenes.
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-1967[TINKERPOP-1967],
+link:http://tinkerpop.apache.org/docs/x.y.z/reference/#connectedcomponent-step[Reference Documentation]
+
==== io() Step
There have been some important changes to IO operations for reading and writing graph data. The use of `Graph.io()`
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
index 72ad47a..eb3b831 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/CoreImports.java
@@ -42,6 +42,7 @@ import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVerte
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.IncrementalBulkLoader;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.OneTimeBulkLoader;
import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.connected.ConnectedComponentVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.ClusterCountMapReduce;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.ClusterPopulationMapReduce;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
@@ -49,6 +50,7 @@ import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankMa
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ConnectedComponent;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.PageRank;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.PeerPressure;
import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
@@ -251,6 +253,8 @@ public final class CoreImports {
// graph computer
CLASS_IMPORTS.add(Computer.class);
CLASS_IMPORTS.add(ComputerResult.class);
+ CLASS_IMPORTS.add(ConnectedComponent.class);
+ CLASS_IMPORTS.add(ConnectedComponentVertexProgram.class);
CLASS_IMPORTS.add(GraphComputer.class);
CLASS_IMPORTS.add(Memory.class);
CLASS_IMPORTS.add(VertexProgram.class);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
new file mode 100644
index 0000000..de718f1
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.clustering.connected;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationUtils;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
+import org.apache.tinkerpop.gremlin.process.traversal.Operator;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * Identifies "Connected Component" instances in a graph by assigning a component identifier (the lexicographically
+ * least string value of the vertex in the component) to each vertex.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ * @author Daniel Kuppitz (http://gremlin.guru)
+ */
+public class ConnectedComponentVertexProgram implements VertexProgram<String> {
+
+ public static final String COMPONENT = "gremlin.connectedComponentVertexProgram.component";
+ private static final String PROPERTY = "gremlin.connectedComponentVertexProgram.property";
+ private static final String EDGE_TRAVERSAL = "gremlin.pageRankVertexProgram.edgeTraversal";
+ private static final String HAS_HALTED = "gremlin.connectedComponentVertexProgram.hasHalted";
+ private static final String VOTE_TO_HALT = "gremlin.connectedComponentVertexProgram.voteToHalt";
+
+ private static final Set<MemoryComputeKey> MEMORY_COMPUTE_KEYS = Collections.singleton(MemoryComputeKey.of(VOTE_TO_HALT, Operator.and, false, true));
+ private MessageScope.Local<?> scope = MessageScope.Local.of(__::bothE);
+ private Set<MessageScope> scopes;
+ private String property = COMPONENT;
+ private boolean hasHalted = false;
+ private PureTraversal<Vertex, Edge> edgeTraversal = null;
+ private Configuration configuration;
+
+ private ConnectedComponentVertexProgram() {}
+
+ @Override
+ public void loadState(final Graph graph, final Configuration config) {
+ configuration = new BaseConfiguration();
+ if (config != null) {
+ ConfigurationUtils.copy(config, configuration);
+ }
+
+ if (configuration.containsKey(EDGE_TRAVERSAL)) {
+ this.edgeTraversal = PureTraversal.loadState(configuration, EDGE_TRAVERSAL, graph);
+ this.scope = MessageScope.Local.of(() -> this.edgeTraversal.get().clone());
+ }
+
+ scopes = new HashSet<>(Collections.singletonList(scope));
+
+ this.property = configuration.getString(PROPERTY, COMPONENT);
+ this.hasHalted = configuration.getBoolean(HAS_HALTED, false);
+ }
+
+ @Override
+ public void storeState(final Configuration config) {
+ VertexProgram.super.storeState(config);
+ if (configuration != null) {
+ ConfigurationUtils.copy(configuration, config);
+ }
+ }
+
+ @Override
+ public void setup(final Memory memory) {
+ memory.set(VOTE_TO_HALT, true);
+ }
+
+ @Override
+ public void execute(final Vertex vertex, final Messenger<String> messenger, final Memory memory) {
+ if (memory.isInitialIteration()) {
+ // on the first pass, just initialize the component to its own id then pass it to all adjacent vertices
+ // for evaluation
+ vertex.property(VertexProperty.Cardinality.single, property, vertex.id().toString());
+
+ // no need to send messages from traversers that were filtered - happens for stuff like
+ // g.V().hasLabel('software').connectedComponent() (i.e. when there is a TraversalVertexProgram in the mix
+ // halting traversers. only want to send messages from traversers that are still hanging about after
+ // the filter. the unfiltered vertices can only react to messages sent to them. of course, this can
+ // lead to weirdness in results.
+ if (vertex.edges(Direction.BOTH).hasNext() && !(hasHalted && !vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent())) {
+ // since there was message passing we don't want to halt on the first round. this should only trigger
+ // a single pass finish if the graph is completely disconnected (technically, it won't even really
+ // work in cases where halted traversers come into play
+ messenger.sendMessage(scope, vertex.id().toString());
+ memory.add(VOTE_TO_HALT, false);
+ }
+ } else {
+ // by the second iteration all vertices that matter should have a component assigned
+ String currentComponent = vertex.value(property);
+ boolean different = false;
+
+ // iterate through messages received and determine if there is a component that has a lesser value than
+ // the currently assigned one
+ final Iterator<String> componentIterator = messenger.receiveMessages();
+ while(componentIterator.hasNext()) {
+ final String candidateComponent = componentIterator.next();
+ if (candidateComponent.compareTo(currentComponent) < 0) {
+ currentComponent = candidateComponent;
+ different = true;
+ }
+ }
+
+ // if there is a better component then assign it and notify adjacent vertices. triggering the message
+ // passing should not halt future executions
+ if (different) {
+ vertex.property(VertexProperty.Cardinality.single, property, currentComponent);
+ messenger.sendMessage(scope, currentComponent);
+ memory.add(VOTE_TO_HALT, false);
+ }
+ }
+ }
+
+ @Override
+ public Set<VertexComputeKey> getVertexComputeKeys() {
+ return new HashSet<>(Collections.singletonList(VertexComputeKey.of(property, false)));
+ }
+
+ @Override
+ public Set<MemoryComputeKey> getMemoryComputeKeys() {
+ return MEMORY_COMPUTE_KEYS;
+ }
+
+ @Override
+ public boolean terminate(final Memory memory) {
+ final boolean voteToHalt = memory.<Boolean>get(VOTE_TO_HALT);
+ if (voteToHalt) {
+ return true;
+ } else {
+ // it is basically always assumed that the program will want to halt, but if message passing occurs, the
+ // program will want to continue, thus reset false values to true for future iterations
+ memory.set(VOTE_TO_HALT, true);
+ return false;
+ }
+ }
+
+ @Override
+ public Set<MessageScope> getMessageScopes(final Memory memory) {
+ return scopes;
+ }
+
+ @Override
+ public GraphComputer.ResultGraph getPreferredResultGraph() {
+ return GraphComputer.ResultGraph.NEW;
+ }
+
+ @Override
+ public GraphComputer.Persist getPreferredPersist() {
+ return GraphComputer.Persist.VERTEX_PROPERTIES;
+ }
+
+
+ @Override
+ @SuppressWarnings("CloneDoesntCallSuperClone,CloneDoesntDeclareCloneNotSupportedException")
+ public ConnectedComponentVertexProgram clone() {
+ return this;
+ }
+
+ @Override
+ public Features getFeatures() {
+ return new Features() {
+ @Override
+ public boolean requiresLocalMessageScopes() {
+ return true;
+ }
+
+ @Override
+ public boolean requiresVertexPropertyAddition() {
+ return true;
+ }
+ };
+ }
+
+ public static ConnectedComponentVertexProgram.Builder build() {
+ return new ConnectedComponentVertexProgram.Builder();
+ }
+
+ public static final class Builder extends AbstractVertexProgramBuilder<ConnectedComponentVertexProgram.Builder> {
+
+ private Builder() {
+ super(ConnectedComponentVertexProgram.class);
+ }
+
+ public ConnectedComponentVertexProgram.Builder hasHalted(final boolean hasHalted) {
+ this.configuration.setProperty(HAS_HALTED, hasHalted);
+ return this;
+ }
+
+ public ConnectedComponentVertexProgram.Builder edges(final Traversal.Admin<Vertex, Edge> edgeTraversal) {
+ PureTraversal.storeState(this.configuration, EDGE_TRAVERSAL, edgeTraversal);
+ return this;
+ }
+
+ public ConnectedComponentVertexProgram.Builder property(final String key) {
+ this.configuration.setProperty(PROPERTY, key);
+ return this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
new file mode 100644
index 0000000..85558bc
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponent.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+
+/**
+ * Configuration options to be passed to the {@link GraphTraversal#with(String, Object)} step on
+ * {@link GraphTraversal#connectedComponent()} ()}.
+ */
+public class ConnectedComponent {
+ /**
+ * Configures the edge to traverse when calculating the pagerank.
+ */
+ public static final String edges = Graph.Hidden.hide("tinkerpop.connectedComponent.edges");
+
+ /**
+ * Configures the name of the property within which to store the pagerank value.
+ */
+ public static final String propertyName = Graph.Hidden.hide("tinkerpop.connectedComponent.propertyName");
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
new file mode 100644
index 0000000..edeb497
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.connected.ConnectedComponentVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Configuring;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
+import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public final class ConnectedComponentVertexProgramStep extends VertexProgramStep implements TraversalParent, Configuring {
+
+ private Parameters parameters = new Parameters();
+ private PureTraversal<Vertex, Edge> edgeTraversal;
+ private String clusterProperty = ConnectedComponentVertexProgram.COMPONENT;
+
+ public ConnectedComponentVertexProgramStep(final Traversal.Admin traversal) {
+ super(traversal);
+ this.configure(ConnectedComponent.edges, __.<Vertex>bothE());
+ }
+
+ @Override
+ public void configure(final Object... keyValues) {
+ if (keyValues[0].equals(ConnectedComponent.edges)) {
+ if (!(keyValues[1] instanceof Traversal))
+ throw new IllegalArgumentException("ConnectedComponent.edges requires a Traversal as its argument");
+ this.edgeTraversal = new PureTraversal<>(((Traversal<Vertex,Edge>) keyValues[1]).asAdmin());
+ this.integrateChild(this.edgeTraversal.get());
+ } else if (keyValues[0].equals(ConnectedComponent.propertyName)) {
+ if (!(keyValues[1] instanceof String))
+ throw new IllegalArgumentException("ConnectedComponent.propertyName requires a String as its argument");
+ this.clusterProperty = (String) keyValues[1];
+ } else {
+ this.parameters.set(this, keyValues);
+ }
+ }
+
+ @Override
+ public Parameters getParameters() {
+ return parameters;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() ^ this.clusterProperty.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.stepString(this, this.clusterProperty, new GraphFilter(this.computer));
+ }
+
+ @Override
+ public ConnectedComponentVertexProgram generateProgram(final Graph graph, final Memory memory) {
+ final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
+ detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
+ return ConnectedComponentVertexProgram.build().
+ hasHalted(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS)).
+ edges(detachedTraversal).
+ property(this.clusterProperty).create(graph);
+ }
+
+ @Override
+ public ConnectedComponentVertexProgramStep clone() {
+ return (ConnectedComponentVertexProgramStep) super.clone();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
index 2df5ecf..5edd0d5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
@@ -57,6 +57,10 @@ import java.util.Iterator;
method = "*",
reason = "https://issues.apache.org/jira/browse/TINKERPOP-1976")
@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ConnectedComponentTest",
+ method = "*",
+ reason = "hmmmm")
+@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest",
method = "*",
reason = "https://issues.apache.org/jira/browse/TINKERPOP-1976")
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
index a675ad1..4fd247f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
@@ -19,6 +19,8 @@
package org.apache.tinkerpop.gremlin.process.traversal.dsl.graph;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.connected.ConnectedComponentVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ConnectedComponentVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.PageRankVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.PeerPressureVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ProgramVertexProgramStep;
@@ -2452,6 +2454,18 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
/**
+ * Executes a Connected Component algorithm over the graph.
+ *
+ * @return the traversal with the appended {@link ConnectedComponentVertexProgram}
+ * @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#connectedcomponent-step" target="_blank">Reference Documentation - ConnectedComponent Step</a>
+ * @since 3.4.0
+ */
+ public default GraphTraversal<S, E> connectedComponent() {
+ this.asAdmin().getBytecode().addStep(Symbols.connectedComponent);
+ return this.asAdmin().addStep((Step<E, E>) new ConnectedComponentVertexProgramStep(this.asAdmin()));
+ }
+
+ /**
* Executes an arbitrary {@link VertexProgram} over the graph.
*
* @return the traversal with the appended {@link ProgramVertexProgramStep}
@@ -2882,6 +2896,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
public static final String pageRank = "pageRank";
public static final String peerPressure = "peerPressure";
+ public static final String connectedComponent = "connectedComponent";
public static final String program = "program";
public static final String by = "by";
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java
index 3d9a549..40ca312 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java
@@ -43,7 +43,7 @@ import static org.junit.Assert.assertEquals;
public class GraphTraversalTest {
private static final Logger logger = LoggerFactory.getLogger(GraphTraversalTest.class);
- private static Set<String> NO_GRAPH = new HashSet<>(Arrays.asList("asAdmin", "by", "read", "write", "with", "option", "iterate", "to", "from", "profile", "pageRank", "peerPressure", "program", "none"));
+ private static Set<String> NO_GRAPH = new HashSet<>(Arrays.asList("asAdmin", "by", "read", "write", "with", "option", "iterate", "to", "from", "profile", "pageRank", "connectedComponent", "peerPressure", "program", "none"));
private static Set<String> NO_ANONYMOUS = new HashSet<>(Arrays.asList("start", "__"));
private static Set<String> IGNORES_BYTECODE = new HashSet<>(Arrays.asList("asAdmin", "read", "write", "iterate", "mapValues", "mapKeys"));
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
index 361b246..e2fa3e1 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
@@ -401,6 +401,15 @@ namespace Gremlin.Net.Process.Traversal
}
/// <summary>
+ /// Adds the connectedComponent step to this <see cref="GraphTraversal{SType, EType}" />.
+ /// </summary>
+ public GraphTraversal<S, E> ConnectedComponent ()
+ {
+ Bytecode.AddStep("connectedComponent");
+ return Wrap<S, E>(this);
+ }
+
+ /// <summary>
/// Adds the constant step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
public GraphTraversal<S, E2> Constant<E2> (E2 e)
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/graph-traversal.js
----------------------------------------------------------------------
diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/graph-traversal.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/graph-traversal.js
index 4f39fa5..1c8d639 100644
--- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/graph-traversal.js
+++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/graph-traversal.js
@@ -353,6 +353,16 @@ class GraphTraversal extends Traversal {
}
/**
+ * Graph traversal connectedComponent method.
+ * @param {...Object} args
+ * @returns {GraphTraversal}
+ */
+ connectedComponent(...args) {
+ this.bytecode.addStep('connectedComponent', args);
+ return this;
+ }
+
+ /**
* Graph traversal constant method.
* @param {...Object} args
* @returns {GraphTraversal}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py
index 6d56c5c..dc0cbed 100644
--- a/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py
+++ b/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py
@@ -194,6 +194,10 @@ class GraphTraversal(Traversal):
self.bytecode.add_step("coin", *args)
return self
+ def connectedComponent(self, *args):
+ self.bytecode.add_step("connectedComponent", *args)
+ return self
+
def constant(self, *args):
self.bytecode.add_step("constant", *args)
return self
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
index eab562d..6466ae8 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
@@ -50,6 +50,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.filter.TailTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddEdgeTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.CoalesceTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ConnectedComponentTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ConstantTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.FlatMapTest;
@@ -143,6 +144,7 @@ public class ProcessComputerSuite extends AbstractGremlinSuite {
// map
CoalesceTest.Traversals.class,
+ ConnectedComponentTest.Traversals.class,
ConstantTest.Traversals.class,
CountTest.Traversals.class,
FlatMapTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
new file mode 100644
index 0000000..25e618a
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.connected.ConnectedComponentVertexProgram;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ConnectedComponent;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.junit.Test;
+
+import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.bothE;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public abstract class ConnectedComponentTest extends AbstractGremlinProcessTest {
+
+ public abstract Traversal<Vertex, Vertex> get_g_V_connectedComponent();
+
+ public abstract Traversal<Vertex, Vertex> get_g_V_hasLabelXsoftwareX_connectedComponent();
+
+ public abstract Traversal<Vertex, Vertex> get_g_V_connectedComponent_withXedges_bothEXknowsXX_withXpropertyName_clusterX();
+
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_connectedComponent() {
+ final Traversal<Vertex, Vertex> traversal = get_g_V_connectedComponent();
+ printTraversalForm(traversal);
+ int counter = 0;
+ while (traversal.hasNext()) {
+ final Vertex vertex = traversal.next();
+ counter++;
+ assertEquals("1", vertex.value(ConnectedComponentVertexProgram.COMPONENT));
+ }
+ assertEquals(6, counter);
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_hasLabelXsoftwareX_connectedComponent() {
+ final Traversal<Vertex, Vertex> traversal = get_g_V_hasLabelXsoftwareX_connectedComponent();
+ printTraversalForm(traversal);
+ int counter = 0;
+ while (traversal.hasNext()) {
+ final Vertex vertex = traversal.next();
+ final String name = vertex.value("name");
+ switch (name) {
+ case "lop":
+ case "ripple":
+ assertEquals("3", vertex.value(ConnectedComponentVertexProgram.COMPONENT));
+ break;
+ }
+ counter++;
+ }
+ assertEquals(2, counter);
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_connectedComponent_withXEDGES_bothEXknowsXX_withXPROPERTY_NAME_clusterX() {
+ final Traversal<Vertex, Vertex> traversal = get_g_V_connectedComponent_withXedges_bothEXknowsXX_withXpropertyName_clusterX();
+ printTraversalForm(traversal);
+ int counter = 0;
+ while (traversal.hasNext()) {
+ final Vertex vertex = traversal.next();
+ final String name = vertex.value("name");
+ switch (name) {
+ case "marko":
+ case "vadas":
+ case "josh":
+ assertEquals("1", vertex.value("cluster"));
+ break;
+ case "peter":
+ assertEquals("6", vertex.value("cluster"));
+ break;
+ }
+ counter++;
+ }
+ assertEquals(4, counter);
+ }
+
+ public static class Traversals extends ConnectedComponentTest {
+ @Override
+ public Traversal<Vertex, Vertex> get_g_V_connectedComponent() {
+ return g.V().connectedComponent();
+ }
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_V_hasLabelXsoftwareX_connectedComponent() {
+ return g.V().hasLabel("software").connectedComponent();
+ }
+ @Override
+ public Traversal<Vertex, Vertex> get_g_V_connectedComponent_withXedges_bothEXknowsXX_withXpropertyName_clusterX() {
+ return g.V().hasLabel("person").connectedComponent().with(ConnectedComponent.edges, bothE("knows")).with(ConnectedComponent.propertyName, "cluster");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2e6b45a3/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index c778c6d..2f727c8 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -39,6 +39,7 @@ import org.apache.tinkerpop.gremlin.process.computer.Computer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ConnectedComponentTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PeerPressureTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest;
@@ -115,6 +116,7 @@ public class SparkHadoopGraphProvider extends AbstractFileGraphProvider {
if (null != loadGraphWith &&
!test.equals(ProgramTest.Traversals.class) &&
!test.equals(PageRankTest.Traversals.class) &&
+ !test.equals(ConnectedComponentTest.Traversals.class) &&
!test.equals(PeerPressureTest.Traversals.class) &&
!test.equals(FileSystemStorageCheck.class) &&
!testMethodName.equals("shouldSupportJobChaining") && // GraphComputerTest.shouldSupportJobChaining
[02/50] tinkerpop git commit: TINKERPOP-1996 Removed OptOuts for
read()/write() tests
Posted by sp...@apache.org.
TINKERPOP-1996 Removed OptOuts for read()/write() tests
Not necessary because existing checks ignore these. For read() you can't write to a HadoopGraph directly (i.e. create vertices/edges) and for write() (and technically read()) it is ignored as it requires a GraphComputer to work.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/bd275a7f
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/bd275a7f
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/bd275a7f
Branch: refs/heads/TINKERPOP-1967
Commit: bd275a7ffa4f0d04634c830aa4f7577375c7944c
Parents: 9e4da01
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 13 11:37:21 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:09 2018 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/hadoop/structure/HadoopGraph.java | 10 ----------
1 file changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/bd275a7f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 5935ebf..14c5360 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -142,16 +142,6 @@ import java.util.stream.Stream;
method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
computers = {"ALL"})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals",
- method = "*",
- reason = "This body of tests is not configured to properly suit OLAP based testing and HadoopGraph is not designed to handle single-threaded OLTP reads/writes.",
- computers = {"ALL"})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals",
- method = "*",
- reason = "This body of tests is not configured to properly suit OLAP based testing and HadoopGraph is not designed to handle single-threaded OLTP reads/writes.",
- computers = {"ALL"})
public final class HadoopGraph implements Graph {
public static final Logger LOGGER = LoggerFactory.getLogger(HadoopGraph.class);
[03/50] tinkerpop git commit: TINKERPOP-1996 Fixed a bad method call
for Configuring steps
Posted by sp...@apache.org.
TINKERPOP-1996 Fixed a bad method call for Configuring steps
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/9e4da014
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/9e4da014
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/9e4da014
Branch: refs/heads/TINKERPOP-1967
Commit: 9e4da0149247a50277e2a468b0becf892426ce2e
Parents: ae79637
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 13 11:37:02 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:09 2018 -0400
----------------------------------------------------------------------
.../process/computer/traversal/strategy/HadoopIoStrategy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/9e4da014/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
index 89ee04b..6d3899e 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -55,7 +55,7 @@ public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalS
final HadoopIoStep hadoopIoStep = new HadoopIoStep(traversal, readWriting.getFile());
hadoopIoStep.setMode(readWriting.getMode());
readWriting.getParameters().getRaw().entrySet().forEach(kv ->
- hadoopIoStep.configure(null, kv.getKey(), kv.getValue())
+ hadoopIoStep.configure(kv.getKey(), kv.getValue())
);
TraversalHelper.replaceStep((Step) readWriting, hadoopIoStep, traversal);
[28/50] tinkerpop git commit: TINKERPOP-1996 Fixed up typos in docs
Posted by sp...@apache.org.
TINKERPOP-1996 Fixed up typos in docs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/38dc70df
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/38dc70df
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/38dc70df
Branch: refs/heads/TINKERPOP-1967
Commit: 38dc70df6db105595784a93ecf5f5726201a962d
Parents: fdb35c6
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 27 07:07:17 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 27 07:07:17 2018 -0400
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/38dc70df/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index c3b9300..c861d87 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1086,9 +1086,9 @@ g.io(someOutputFile).
write().iterate()
----
-The `IO` class is a helper for the `io()` step that provides expressions that can be used to help configure it
+The `IO` class is a helper for the `io()`-step that provides expressions that can be used to help configure it
and in this case it allows direct specification of the "reader" or "writer" to use. The "reader" actually refers to
-a `GraphReader` implementation and the `writer" refers to a `GraphWriter` implementation. The implementations of
+a `GraphReader` implementation and the "writer" refers to a `GraphWriter` implementation. The implementations of
those interfaces provided by default are the standard TinkerPop implementations.
That default is an important point to consider for users. The default TinkerPop implementations are not designed with
@@ -1100,7 +1100,7 @@ writing perspective is not that different in there are no parallel operations in
to disk requires a single pass of the data without high memory requirements for larger datasets.
In general, TinkerPop recommends that users examine the native bulk import/export tools of the graph implementation
-that they choose. Those tools will often outperform the `io()` step and perhaps be easier to use with a greater
+that they choose. Those tools will often outperform the `io()`-step and perhaps be easier to use with a greater
feature set. That said, graph providers do have the option to optimize `io()` to back it with their own
import/export utilities and therefore the default behavior provided by TinkerPop described above might be overridden
by the graph.
@@ -1111,7 +1111,7 @@ functionality internally using <<clonevertexprogram,CloneVertexProgram>>. With t
can be imported/exported assuming that there is a Hadoop `InputFormat` or `OutputFormat` to support it.
IMPORTANT: Remote Gremlin Console users or Gremlin Language Variant (GLV) users (e.g. gremlin-python) who utilize
-the `io()` step should recall that their `read()` or `write()` operation will occur on the server and not locally
+the `io()`-step should recall that their `read()` or `write()` operation will occur on the server and not locally
and therefore the file specified for import/export must be something accessible by the server.
GraphSON and Gryo formats are extensible allowing users and graph providers to extend supported serialization options.
[46/50] tinkerpop git commit: Extended the connected-components recipe
Posted by sp...@apache.org.
Extended the connected-components recipe
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/3ef1661c
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/3ef1661c
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/3ef1661c
Branch: refs/heads/TINKERPOP-1967
Commit: 3ef1661cf330b4c4bd610aec4f1337f7ac5e30d9
Parents: 3298474
Author: HadoopMarc <vt...@xs4all.nl>
Authored: Sun Jun 10 15:17:17 2018 +0200
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/connected-components.asciidoc | 94 ++++++++++++--------
docs/static/images/cc-scale-ratio.png | Bin 0 -> 14393 bytes
docs/static/images/cc-scale-size.png | Bin 0 -> 12220 bytes
3 files changed, 58 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/3ef1661c/docs/src/recipes/connected-components.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/connected-components.asciidoc b/docs/src/recipes/connected-components.asciidoc
index edbeec5..850c31f 100644
--- a/docs/src/recipes/connected-components.asciidoc
+++ b/docs/src/recipes/connected-components.asciidoc
@@ -31,11 +31,11 @@ Depending on the size of the graph, three solution regimes can be discriminated:
1. Small graphs that fit in the memory of a single machine
-2. Medium graphs backed by storage for which a linear scan is still feasible. This regime is left to third party
+2. Medium-sized graphs backed by storage for which an OLTP linear scan is still feasible. This regime is left to third party
TinkerPop implementations, since TinkerPop itself has no storage-backed reference implementations. The idea is that
component membership is stored in the graph, rather than in memory.
-3. Large graphs requiring an OLAP approach to yield results in a reasonable time.
+3. Large graphs requiring an approach with `HadoopGraph` and `SparkGraphComputer` to yield results in a reasonable time.
These regimes are discussed separately using the following graph with three weakly connected components:
@@ -55,16 +55,21 @@ g.addV().property(id, "A").as("a").
addE("link").from("d").to("e").iterate()
----
+==== Small graph traversals
-===== Small graphs
-
-Connected components in a small graph can be determined with both an OLTP traversal and the OLAP
+Connected components in a small graph can be determined with either an OLTP traversal or the OLAP
`connectedComponent()`-step. The `connectedComponent()`-step is available as of TinkerPop 3.4.0 and is
described in more detail in the
link:http://tinkerpop.apache.org/docs/x.y.z/reference/#connectedcomponent-step[Reference Documentation].
+The traversal looks like:
+[gremlin-groovy,existing]
+----
+g.withComputer().V().connectedComponent().
+ group().by('gremlin.connectedComponentVertexProgram.component').
+ select(values).unfold()
+----
A straightforward way to detect the various subgraphs with an OLTP traversal is to do this:
-
[gremlin-groovy,existing]
----
g.V().emit(cyclicPath().or().not(both())). <1>
@@ -73,7 +78,6 @@ g.V().emit(cyclicPath().or().not(both())). <1
by(path().unfold().dedup().fold()). <4>
select(values).unfold() <5>
----
-
<1> The initial emit() step allows for output of isolated vertices, in addition to the discovery of
components as described in (2).
@@ -83,7 +87,7 @@ path. Collection `'a'` is used to keep track of visited vertices, for both subt
and new traversals resulting from the `g.V()` linear scan.
<3> While `'a'` nicely keeps track of vertices already visited, the actual components need to be extracted from the
-path information of surviving traversers. The `path().unfold().limit(1)` closure provides the starting vertex
+path information. The `path().unfold().limit(1)` closure provides the starting vertex
of surviving traversers, which can be used to group the components.
<4> This clause collects the unique vertices from all paths with the same starting vertex, thus from the same
@@ -91,39 +95,57 @@ weak component.
<5> The values of the groupby map contain the lists of vertices making up the requested components.
-This algorithm completes in linear time with the number of vertices and edges, because a traversal is started for each
-vertex and each edge with its associated out-vertex is visited exactly once.
-
-==== Large graphs
-Large graphs require an OLAP solution with a custom VertexProgram that can be run using a graph implementation's
-GraphComputer, in particular `SparkGraphComputer` on a `HadoopGraph`. The OLAP solution also runs faster for most
-medium-sized graphs, that is when these graph have a 'natural' structure with a limited maximum path length.
+==== Small graph scalability
-The TinkerPop library of vertex programs contains the `WeakComponentsVertexProgram` which can be run in the same
-way as the link:http://tinkerpop.apache.org/docs/x.y.z/reference/#peerpressurevertexprogram[PeerPressureVertexProgram]:
+The scalability of the OLTP traversal and the `connectedComponent()`-step for in-memory graphs is shown in the figures
+below.
-[gremlin-groovy,existing]
-----
-result = g.getGraph().compute().
- program(WeakComponentsVertexProgram.build().maxIterations(100).create()).
- mapReduce(ClusterPopulationMapReduce.build().create()).
- mapReduce(ClusterCountMapReduce.build().create()).
- submit().get()
-result.memory().clusterPopulation
-gResult = result.graph().traversal()
-gResult.V().valueMap(true)
-----
+[[cc-scale-size]]
+.Run times for finding connected components in a randomly generated graph with 10 components of equal size and with an edge/vertex ratio of 6
+image::cc-scale-size.png[width=600, side=bottom]
-The vertex program has interconnected vertices exchange id's and store the lowest id until no vertex receives a
-lower id. This algorithm is commonly applied in
+In general, the `connectedComponent()`-step is almost a factor two faster than the OLTP traversal. Only, for very
+small graphs the overhead of running the ConnectedComponentVertexProgram is larger than that of the OLTP traversal.
+The vertex program works by having interconnected vertices exchange id's and store the lowest id until no vertex
+receives a lower id. This algorithm is commonly applied in
link:https://en.wikipedia.org/wiki/Bulk_synchronous_parallel[bulk synchronous parallel] systems, e.g. in
-link:https://spark.apache.org/graphx[Apache Spark GraphX].
+link:https://spark.apache.org/graphx[Apache Spark GraphX]. Overhead for the vertex program arises because it has to run
+as many cycles as the largest length of the shortest paths between any two vertices in a component of the graph. In
+every cycle each vertex has to be checked for being
+"halted". Overhead of the OLTP traversal consists of each traverser having to carry complete path information. For
+pure depth-first-search or breadth-first-search implementations, connected-component algotithms should scale
+as [.big]##O##(V+E). For the traversals in the figure above this is almost the case.
+
+
+[[cc-scale-ratio]]
+.Run times for finding connected components in a randomly generated graph with 10 components, each consisting of 6400 vertices
+image::cc-scale-ratio.png[width=600]
+
+The random graphs used for the scalability tests can be modulated with the edge/vertex ratio. For small ratios the
+components generated are more lint-like and harder to process by the `connectedComponent()`-step. For high ratios
+the components are more mesh-like and the ConnectedComponentVertexProgram needs few cycles to process the graph. These
+characteristics show clearly from the graph. Indeed, for a given number of vertices, the run time of the
+`connectedComponent()`-step does not depend on the number of edges, but rather on the maximum shortest path length in
+the graph.
+
+
+==== Large graphs
+
+Large graphs in TinkerPop require distributed processing by `SparkGraphComputer` to get results in a reasonable time (OLAP
+approach). This means that the graph must be available as `HadoopGraph` (third party TinkerPop implementations often
+allow to make a graph available as an `HadoopGraph` by providing an Hadoop `InputFormat`). Running the
+`connectedComponent()`-step on
+an `HadoopGraph` works the same as for a small graph, provided that `SparkGraphComputer` is specified as the graph computer,
+either with the `gremlin.hadoop.defaultGraphComputer` property or as part of the `withComputer()`-step.
+
+Scalability of the the `connectedComponent()`-step with `SparkGraphComputer` is high, but note that:
+
+* the graph should fit in the memory of the Spark cluster to allow the VertexProgram to run its cycles without spilling
+intermediate results to disk and loosing most of the gains from the distributed processing
-==== Scalability
+* as discussed for small graphs, the BSP algorithm does not play well with graphs having a large shortest path between
+any pair of vertices. Overcoming this limitation is still a
+link:http://www.vldb.org/pvldb/vol7/p1821-yan.pdf[subject of academic research].
-ToDo:
- - limits and run time regime 1
- - test of friendster graph regime 3
- - discuss: link:http://www.vldb.org/pvldb/vol7/p1821-yan.pdf[http://www.vldb.org/pvldb/vol7/p1821-yan.pdf]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/3ef1661c/docs/static/images/cc-scale-ratio.png
----------------------------------------------------------------------
diff --git a/docs/static/images/cc-scale-ratio.png b/docs/static/images/cc-scale-ratio.png
new file mode 100644
index 0000000..33a842d
Binary files /dev/null and b/docs/static/images/cc-scale-ratio.png differ
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/3ef1661c/docs/static/images/cc-scale-size.png
----------------------------------------------------------------------
diff --git a/docs/static/images/cc-scale-size.png b/docs/static/images/cc-scale-size.png
new file mode 100644
index 0000000..2b08a89
Binary files /dev/null and b/docs/static/images/cc-scale-size.png differ
[44/50] tinkerpop git commit: TINKERPOP-1967 fixed up halted
traversers
Posted by sp...@apache.org.
TINKERPOP-1967 fixed up halted traversers
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/d34106d4
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/d34106d4
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/d34106d4
Branch: refs/heads/TINKERPOP-1967
Commit: d34106d4f29e3e050d2fceccbf06dba0e68d209d
Parents: 669d9fc
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 10:51:35 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
.../ConnectedComponentVertexProgram.java | 42 +++++++++++++++-----
.../ConnectedComponentVertexProgramStep.java | 26 ++++++++++--
.../step/map/ConnectedComponentTest.java | 2 +-
3 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d34106d4/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
index de718f1..82907eb 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/connected/ConnectedComponentVertexProgram.java
@@ -32,7 +32,10 @@ import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexPr
import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.IndexedTraverserSet;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -40,6 +43,8 @@ import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
@@ -61,12 +66,14 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
private static final String VOTE_TO_HALT = "gremlin.connectedComponentVertexProgram.voteToHalt";
private static final Set<MemoryComputeKey> MEMORY_COMPUTE_KEYS = Collections.singleton(MemoryComputeKey.of(VOTE_TO_HALT, Operator.and, false, true));
+
private MessageScope.Local<?> scope = MessageScope.Local.of(__::bothE);
private Set<MessageScope> scopes;
private String property = COMPONENT;
- private boolean hasHalted = false;
private PureTraversal<Vertex, Edge> edgeTraversal = null;
private Configuration configuration;
+ private TraverserSet<Vertex> haltedTraversers;
+ private IndexedTraverserSet<Vertex, Vertex> haltedTraversersIndex;
private ConnectedComponentVertexProgram() {}
@@ -85,7 +92,12 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
scopes = new HashSet<>(Collections.singletonList(scope));
this.property = configuration.getString(PROPERTY, COMPONENT);
- this.hasHalted = configuration.getBoolean(HAS_HALTED, false);
+
+ this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
+ this.haltedTraversersIndex = new IndexedTraverserSet<>(v -> v);
+ for (final Traverser.Admin<Vertex> traverser : this.haltedTraversers) {
+ this.haltedTraversersIndex.add(traverser.split());
+ }
}
@Override
@@ -104,6 +116,8 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
@Override
public void execute(final Vertex vertex, final Messenger<String> messenger, final Memory memory) {
if (memory.isInitialIteration()) {
+ copyHaltedTraversersFromMemory(vertex);
+
// on the first pass, just initialize the component to its own id then pass it to all adjacent vertices
// for evaluation
vertex.property(VertexProperty.Cardinality.single, property, vertex.id().toString());
@@ -113,7 +127,7 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
// halting traversers. only want to send messages from traversers that are still hanging about after
// the filter. the unfiltered vertices can only react to messages sent to them. of course, this can
// lead to weirdness in results.
- if (vertex.edges(Direction.BOTH).hasNext() && !(hasHalted && !vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent())) {
+ if (vertex.edges(Direction.BOTH).hasNext()) {
// since there was message passing we don't want to halt on the first round. this should only trigger
// a single pass finish if the graph is completely disconnected (technically, it won't even really
// work in cases where halted traversers come into play
@@ -148,7 +162,9 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
@Override
public Set<VertexComputeKey> getVertexComputeKeys() {
- return new HashSet<>(Collections.singletonList(VertexComputeKey.of(property, false)));
+ return new HashSet<>(Arrays.asList(
+ VertexComputeKey.of(property, false),
+ VertexComputeKey.of(TraversalVertexProgram.HALTED_TRAVERSERS, false)));
}
@Override
@@ -158,6 +174,10 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
@Override
public boolean terminate(final Memory memory) {
+ if (memory.isInitialIteration() && this.haltedTraversersIndex != null) {
+ this.haltedTraversersIndex.clear();
+ }
+
final boolean voteToHalt = memory.<Boolean>get(VOTE_TO_HALT);
if (voteToHalt) {
return true;
@@ -206,6 +226,15 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
};
}
+ private void copyHaltedTraversersFromMemory(final Vertex vertex) {
+ final Collection<Traverser.Admin<Vertex>> traversers = this.haltedTraversersIndex.get(vertex);
+ if (traversers != null) {
+ final TraverserSet<Vertex> newHaltedTraversers = new TraverserSet<>();
+ newHaltedTraversers.addAll(traversers);
+ vertex.property(VertexProperty.Cardinality.single, TraversalVertexProgram.HALTED_TRAVERSERS, newHaltedTraversers);
+ }
+ }
+
public static ConnectedComponentVertexProgram.Builder build() {
return new ConnectedComponentVertexProgram.Builder();
}
@@ -216,11 +245,6 @@ public class ConnectedComponentVertexProgram implements VertexProgram<String> {
super(ConnectedComponentVertexProgram.class);
}
- public ConnectedComponentVertexProgram.Builder hasHalted(final boolean hasHalted) {
- this.configuration.setProperty(HAS_HALTED, hasHalted);
- return this;
- }
-
public ConnectedComponentVertexProgram.Builder edges(final Traversal.Admin<Vertex, Edge> edgeTraversal) {
PureTraversal.storeState(this.configuration, EDGE_TRAVERSAL, edgeTraversal);
return this;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d34106d4/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
index edeb497..c222cfa 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ConnectedComponentVertexProgramStep.java
@@ -29,11 +29,16 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.step.Configuring;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import org.apache.tinkerpop.gremlin.util.Serializer;
+
+import java.io.IOException;
+import java.util.Base64;
/**
* @author Stephen Mallette (http://stephen.genoprime.com)
@@ -84,10 +89,25 @@ public final class ConnectedComponentVertexProgramStep extends VertexProgramStep
public ConnectedComponentVertexProgram generateProgram(final Graph graph, final Memory memory) {
final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
- return ConnectedComponentVertexProgram.build().
- hasHalted(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS)).
+
+ final ConnectedComponentVertexProgram.Builder builder = ConnectedComponentVertexProgram.build().
edges(detachedTraversal).
- property(this.clusterProperty).create(graph);
+ property(this.clusterProperty);
+
+ if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS)) {
+ final TraverserSet<?> haltedTraversers = memory.get(TraversalVertexProgram.HALTED_TRAVERSERS);
+ if (!haltedTraversers.isEmpty()) {
+ Object haltedTraversersValue;
+ try {
+ haltedTraversersValue = Base64.getEncoder().encodeToString(Serializer.serializeObject(haltedTraversers));
+ } catch (final IOException ignored) {
+ haltedTraversersValue = haltedTraversers;
+ }
+ builder.configure(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversersValue);
+ }
+ }
+
+ return builder.create(graph);
}
@Override
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d34106d4/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
index 25e618a..8b1904f 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ConnectedComponentTest.java
@@ -67,7 +67,7 @@ public abstract class ConnectedComponentTest extends AbstractGremlinProcessTest
switch (name) {
case "lop":
case "ripple":
- assertEquals("3", vertex.value(ConnectedComponentVertexProgram.COMPONENT));
+ assertEquals("1", vertex.value(ConnectedComponentVertexProgram.COMPONENT));
break;
}
counter++;
[23/50] tinkerpop git commit: TINKERPOP-1996 No need to assert io()
against VertexProgramStrategy
Posted by sp...@apache.org.
TINKERPOP-1996 No need to assert io() against VertexProgramStrategy
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7f1bf178
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7f1bf178
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7f1bf178
Branch: refs/heads/TINKERPOP-1967
Commit: 7f1bf1783efb8a7eca17d0367af66c6289455fd8
Parents: e6e4413
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 11:52:13 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 11:52:13 2018 -0400
----------------------------------------------------------------------
.../strategy/decoration/VertexProgramStrategyTest.java | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7f1bf178/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
index d3bb6ef..8ceef48 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java
@@ -43,7 +43,6 @@ import static org.junit.Assert.assertEquals;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
- * @author Stephen Mallette (http://stephen.genoprime.com)
*/
@RunWith(Parameterized.class)
public class VertexProgramStrategyTest {
@@ -68,12 +67,7 @@ public class VertexProgramStrategyTest {
public static Iterable<Object[]> generateTestParameters() {
final ComputerResultStep computerResultStep = new ComputerResultStep(EmptyTraversal.instance());
-
- // The tests for io() need to verify that there is no change i.e. we don't want the step getting wrapped up in
- // traversalvertexprogramstep stuff or else it won't execute properly in OLAP
return Arrays.asList(new Traversal[][]{
- { EmptyGraph.instance().traversal().io("blah.json"), EmptyGraph.instance().traversal().io("blah.json")},
- { EmptyGraph.instance().traversal().io("blah.json"), EmptyGraph.instance().traversal().io("blah.json")},
{__.V().out().count(), start().addStep(traversal(__.V().out().count())).addStep(computerResultStep)},
{__.V().pageRank().out().count(), start().pageRank().asAdmin().addStep(traversal(__.V().out().count())).addStep(computerResultStep)},
{__.V().out().pageRank(), start().addStep(traversal(__.V().out())).pageRank().asAdmin().addStep(traversal(__.identity())).addStep(computerResultStep)},
[49/50] tinkerpop git commit: TINKERPOP-1967 Minor text cleanup for
connectedComponent() docs
Posted by sp...@apache.org.
TINKERPOP-1967 Minor text cleanup for connectedComponent() docs
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7fd7afd6
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7fd7afd6
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7fd7afd6
Branch: refs/heads/TINKERPOP-1967
Commit: 7fd7afd6da206d69cead5401a7e06484e84d3ff2
Parents: 3ef1661
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jun 15 08:24:22 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/connected-components.asciidoc | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7fd7afd6/docs/src/recipes/connected-components.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/connected-components.asciidoc b/docs/src/recipes/connected-components.asciidoc
index 850c31f..e6d0f7a 100644
--- a/docs/src/recipes/connected-components.asciidoc
+++ b/docs/src/recipes/connected-components.asciidoc
@@ -37,7 +37,6 @@ component membership is stored in the graph, rather than in memory.
3. Large graphs requiring an approach with `HadoopGraph` and `SparkGraphComputer` to yield results in a reasonable time.
-
These regimes are discussed separately using the following graph with three weakly connected components:
image:connected-components.png[width=600]
@@ -70,6 +69,7 @@ g.withComputer().V().connectedComponent().
----
A straightforward way to detect the various subgraphs with an OLTP traversal is to do this:
+
[gremlin-groovy,existing]
----
g.V().emit(cyclicPath().or().not(both())). <1>
@@ -95,8 +95,6 @@ weak component.
<5> The values of the groupby map contain the lists of vertices making up the requested components.
-
-
==== Small graph scalability
The scalability of the OLTP traversal and the `connectedComponent()`-step for in-memory graphs is shown in the figures
@@ -118,7 +116,6 @@ every cycle each vertex has to be checked for being
pure depth-first-search or breadth-first-search implementations, connected-component algotithms should scale
as [.big]##O##(V+E). For the traversals in the figure above this is almost the case.
-
[[cc-scale-ratio]]
.Run times for finding connected components in a randomly generated graph with 10 components, each consisting of 6400 vertices
image::cc-scale-ratio.png[width=600]
@@ -130,7 +127,6 @@ characteristics show clearly from the graph. Indeed, for a given number of verti
`connectedComponent()`-step does not depend on the number of edges, but rather on the maximum shortest path length in
the graph.
-
==== Large graphs
Large graphs in TinkerPop require distributed processing by `SparkGraphComputer` to get results in a reasonable time (OLAP
@@ -142,10 +138,8 @@ either with the `gremlin.hadoop.defaultGraphComputer` property or as part of the
Scalability of the the `connectedComponent()`-step with `SparkGraphComputer` is high, but note that:
-* the graph should fit in the memory of the Spark cluster to allow the VertexProgram to run its cycles without spilling
-intermediate results to disk and loosing most of the gains from the distributed processing
-
-* as discussed for small graphs, the BSP algorithm does not play well with graphs having a large shortest path between
+* The graph should fit in the memory of the Spark cluster to allow the VertexProgram to run its cycles without spilling
+intermediate results to disk and loosing most of the gains from the distributed processing.
+* As discussed for small graphs, the BSP algorithm does not play well with graphs having a large shortest path between
any pair of vertices. Overcoming this limitation is still a
-link:http://www.vldb.org/pvldb/vol7/p1821-yan.pdf[subject of academic research].
-
+link:http://www.vldb.org/pvldb/vol7/p1821-yan.pdf[subject of academic research].
\ No newline at end of file
[04/50] tinkerpop git commit: TINKERPOP-1996 Deprecated Graph.io()
and related infrastructure.
Posted by sp...@apache.org.
TINKERPOP-1996 Deprecated Graph.io() and related infrastructure.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ae796378
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ae796378
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ae796378
Branch: refs/heads/TINKERPOP-1967
Commit: ae796378e07925f9385f3ec65c10022b59aab8b5
Parents: 328737a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 12 14:33:03 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:09 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 ++
.../main/java/org/apache/tinkerpop/gremlin/structure/Graph.java | 3 +++
.../main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java | 3 +++
.../java/org/apache/tinkerpop/gremlin/structure/io/IoCore.java | 3 +++
.../apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java | 3 +++
.../tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java | 3 +++
.../org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java | 3 +++
7 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index c0cd27d..bc78bbe 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -27,6 +27,8 @@ This release also includes changes from <<release-3-3-3, 3.3.3>>.
* Bumped to Netty 4.1.25.
* Bumped to Spark 2.3.1.
+* Added the `io()` start step and `read()` and `write()` termination steps to the Gremlin language.
+* Deprecated `Graph.io()` and related infrastructure.
* Moved `Parameterizing` interface to the `org.apache.tinkerpop.gremlin.process.traversal.step` package with other marker interfaces of its type.
* Replaced `Parameterizing.addPropertyMutations()` with `Configuring.configure()`.
* Changed interface hierarchy for `Parameterizing` and `Mutating` interfaces as they are tightly related.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
index dc14cc6..f1fc54a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
@@ -300,7 +300,10 @@ public interface Graph extends AutoCloseable, Host {
* For those graphs that do not need to register any custom serializers, the default implementation should suffice.
* If the default is overridden, take care to register the current graph via the
* {@link org.apache.tinkerpop.gremlin.structure.io.Io.Builder#graph(Graph)} method.
+ *
+ * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
+ @Deprecated
public default <I extends Io> I io(final Io.Builder<I> builder) {
return (I) builder.graph(this).create();
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
index bae56c5..0971e31 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/Io.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.io.IOException;
@@ -31,7 +32,9 @@ import java.util.function.Consumer;
* internal {@link Mapper} (if the format has such capability).
*
* @author Stephen Mallette (http://stephen.genoprime.com)
+ * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
+@Deprecated
public interface Io<R extends GraphReader.ReaderBuilder, W extends GraphWriter.WriterBuilder, M extends Mapper.Builder> {
/**
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoCore.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoCore.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoCore.java
index 9d9ad60..a357184 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoCore.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoCore.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLIo;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
@@ -27,7 +28,9 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
* methods statically.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
+ * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
+@Deprecated
public final class IoCore {
private IoCore() {}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
index 88431bf..b6af646 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphml/GraphMLIo.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.graphml;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
@@ -37,7 +38,9 @@ import java.util.function.Consumer;
* such things.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
+ * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
+@Deprecated
public final class GraphMLIo implements Io<GraphMLReader.Builder, GraphMLWriter.Builder, GraphMLMapper.Builder> {
private final Graph graph;
private Optional<Consumer<Mapper.Builder>> onMapper;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
index a3923a1..7f8b835 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONIo.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.graphson;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
@@ -36,7 +37,9 @@ import java.util.function.Consumer;
* interfaces should see the {@link GraphSONMapper} for information on the expectations for the {@link IoRegistry}.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
+ * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
+@Deprecated
public final class GraphSONIo implements Io<GraphSONReader.Builder, GraphSONWriter.Builder, GraphSONMapper.Builder> {
private final Graph graph;
private final Optional<Consumer<Mapper.Builder>> onMapper;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ae796378/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
index 29e63de..a7dfe0d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoIo.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
@@ -36,7 +37,9 @@ import java.util.function.Consumer;
* interfaces should see the {@link GryoMapper} for information on the expectations for the {@link IoRegistry}.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
+ * @deprecated As of release 3.4.0, replaced by {@link GraphTraversalSource#io(String)}.
*/
+@Deprecated
public final class GryoIo implements Io<GryoReader.Builder, GryoWriter.Builder, GryoMapper.Builder> {
private final Graph graph;
[30/50] tinkerpop git commit: Added "get or create" or "upsert"
recipe CTR
Posted by sp...@apache.org.
Added "get or create" or "upsert" recipe CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/3b8c8280
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/3b8c8280
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/3b8c8280
Branch: refs/heads/TINKERPOP-1967
Commit: 3b8c8280cab1e7a6fd3b273ef0d2cf2c6d00f650
Parents: dde73e4
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 27 16:29:11 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 27 16:29:11 2018 -0400
----------------------------------------------------------------------
docs/src/recipes/element-existence.asciidoc | 91 ++++++++++++++++++++++++
docs/src/recipes/index.asciidoc | 2 +
2 files changed, 93 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/3b8c8280/docs/src/recipes/element-existence.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/element-existence.asciidoc b/docs/src/recipes/element-existence.asciidoc
new file mode 100644
index 0000000..be6e517
--- /dev/null
+++ b/docs/src/recipes/element-existence.asciidoc
@@ -0,0 +1,91 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+////
+[[element-existence]]
+== Element Existence
+
+Checking for whether or not a graph element is present in the graph is simple:
+
+[gremlin-groovy,modern]
+----
+g.V().has('person','name','marko').hasNext()
+g.V().has('person','name','stephen').hasNext()
+----
+
+Knowing that an element exists or not is usually a common point of decision in determining the appropriate path of code
+to take. In the example above, the check is for vertex existence and a typical reason to check for existence is to
+determine whether or not to add a new vertex or to return the one that exists (i.e. "get or create" pattern). This
+entire operation can occur in a single traversal.
+
+[gremlin-groovy,modern]
+----
+g.V().has('person','name','marko').
+ fold().
+ coalesce(unfold(),
+ addV('person').
+ property('name','marko').
+ property('age',29))
+g.V().has('person','name','stephen').
+ fold().
+ coalesce(unfold(),
+ addV('person').
+ property('name','stephen').
+ property('age',34))
+----
+
+This use of `coalesce()` shown above is the basis for this pattern. Note that at the end of `has()`-step there is
+either a vertex or not. By using `fold()`, "existence" or "not existence" is reduced to a `List` with the vertex or
+a `List` with no values. With a `List` as the traverser flowing into `coalesce()` the first child traversal to return
+something will execute. If the `List` has a vertex then it will `unfold()` and return the existing one. If it is empty,
+then the vertex does not exist and it is added and returned.
+
+This "get or create" logic can be expanded to be "upsert" like functionality as follows:
+
+[gremlin-groovy,modern]
+----
+g.V().has('person','name','marko').
+ fold().
+ coalesce(unfold(),
+ addV('person').property('name','marko')).
+ property('age',29)
+g.V().has('person','name','stephen').
+ fold().
+ coalesce(unfold(),
+ addV('person').property('name','stephen')).
+ property('age',34)
+----
+
+By moving the `property()`-step that set the "age" value outside of `coalesce()`, the property is then set for both
+newly created vertices and for existing ones.
+
+WARNING: Always consider the specific nature of the graph implementation in use when considering these patterns. Some
+graph databases may not treat these traversals as true "upsert" operations and may do a "read before write" in their
+execution.
+
+It is possible to do similar sorts of operations with edges using the same pattern:
+
+[gremlin-groovy,modern]
+----
+g.V().has('person','name','vadas').as('v').
+ V().has('software','name','ripple').
+ coalesce(__.inE('created').where(outV().as('v')),
+ addE('created').from('v').property('weight',0.5))
+----
+
+In this case, the adjacent vertices of the edge are retrieved first and within the `coalesce()`, the existence of
+the edge is checked with `where()` using a matching pattern on the "v" label and returned if found. If the edge is not
+found between these two vertices, then it is created as part of the second traversal given to `coalesce()`.
+
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/3b8c8280/docs/src/recipes/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/recipes/index.asciidoc b/docs/src/recipes/index.asciidoc
index c59cc89..8886289 100644
--- a/docs/src/recipes/index.asciidoc
+++ b/docs/src/recipes/index.asciidoc
@@ -50,6 +50,8 @@ include::duplicate-vertex.asciidoc[]
include::edge-move.asciidoc[]
+include::element-existence.asciidoc[]
+
include::if-then-based-grouping.asciidoc[]
include::pagination.asciidoc[]
[40/50] tinkerpop git commit: Added dev notes for python environment
CTR
Posted by sp...@apache.org.
Added dev notes for python environment CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/e3018fb2
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/e3018fb2
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/e3018fb2
Branch: refs/heads/TINKERPOP-1967
Commit: e3018fb2e98df62a21272446f63cea5ee550408e
Parents: 00a085a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Jul 31 08:35:06 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 08:35:06 2018 -0400
----------------------------------------------------------------------
docs/src/dev/developer/development-environment.asciidoc | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/e3018fb2/docs/src/dev/developer/development-environment.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc
index cde81c1..779f642 100644
--- a/docs/src/dev/developer/development-environment.asciidoc
+++ b/docs/src/dev/developer/development-environment.asciidoc
@@ -74,13 +74,14 @@ The site will be generated to the `target/site/home` directory.
[[python-environment]]
=== Python Environment
-As of TinkerPop 3.2.2, the build optionally requires link:https://www.python.org/[Python 2.x] to work with the
-`gremlin-python` module. If Python is not installed, TinkerPop will still build with Maven, but native Python tests and
+As of TinkerPop 3.2.2, the build optionally requires link:https://www.python.org/[Python] to build the `gremlin-python`
+module. If Python is not installed, TinkerPop will still build with Maven, but native Python tests and
Java tests that require Python code will be skipped. Developers should also install link:https://pypi.python.org/pypi/pip[pip]
and link:https://virtualenv.pypa.io/en/stable/[virtualenv] (version 15.0.2 - older versions may cause build failures).
-Once the Python environment is established, the full building and testing of `gremlin-python` may commence. It can be
-done manually from the command line with:
+The build expects two Python executables `python` and `python3` where `python` maps to 2.7.6 and `python3` is 3.4.3 or
+higher. Once the Python environment is established, the full building and testing of `gremlin-python` may commence. It
+can be done manually from the command line with:
[source,text]
mvn clean install -Pglv-python
[36/50] tinkerpop git commit: Added a better reason for some OptOut
on RemoteGraph CTR
Posted by sp...@apache.org.
Added a better reason for some OptOut on RemoteGraph CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/00a085aa
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/00a085aa
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/00a085aa
Branch: refs/heads/TINKERPOP-1967
Commit: 00a085aa15cfc877222712f8572c89f1c096597c
Parents: 2b045f3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 11:40:52 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Jul 30 11:40:52 2018 -0400
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/00a085aa/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
index 8a1ab02..4b23475 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
@@ -55,11 +55,11 @@ import java.util.Iterator;
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PeerPressureTest",
method = "*",
- reason = "hmmmm")
+ reason = "https://issues.apache.org/jira/browse/TINKERPOP-1976")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest",
method = "*",
- reason = "hmmmm")
+ reason = "https://issues.apache.org/jira/browse/TINKERPOP-1976")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.TranslationStrategyProcessTest",
method = "*",
[45/50] tinkerpop git commit: TINKERPOP-1967 Referenced
TINKERPOP-1976 in OptOut for connectedComponent on RemoteGraph
Posted by sp...@apache.org.
TINKERPOP-1967 Referenced TINKERPOP-1976 in OptOut for connectedComponent on RemoteGraph
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/f54d836d
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/f54d836d
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/f54d836d
Branch: refs/heads/TINKERPOP-1967
Commit: f54d836db169cf1e7c0655abb711f9bf55c201dd
Parents: d34106d
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 11:22:25 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 13:07:31 2018 -0400
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f54d836d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
index 5edd0d5..57600e1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/RemoteGraph.java
@@ -59,7 +59,7 @@ import java.util.Iterator;
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ConnectedComponentTest",
method = "*",
- reason = "hmmmm")
+ reason = "https://issues.apache.org/jira/browse/TINKERPOP-1976")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest",
method = "*",
[18/50] tinkerpop git commit: TINKERPOP-1996 Pass configurations from
with() through to Hadoop
Posted by sp...@apache.org.
TINKERPOP-1996 Pass configurations from with() through to Hadoop
This will allow users to override or add to the Hadoop/Spark/OLAP configuration as needed
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4d979cf8
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4d979cf8
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4d979cf8
Branch: refs/heads/TINKERPOP-1967
Commit: 4d979cf8dc4482d574191a76a0c60e281f887563
Parents: 51dc821
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 07:35:48 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 07:35:48 2018 -0400
----------------------------------------------------------------------
.../traversal/step/sideEffect/HadoopIoStep.java | 29 ++++++++++++++++++--
1 file changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4d979cf8/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
index ca369b6..2e96276 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
@@ -37,7 +38,9 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
/**
* An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the
- * {@link CloneVertexProgram} for its implementation.
+ * {@link CloneVertexProgram} for its implementation. Standard Hadoop OLAP configurations can be passed using the
+ * {@link GraphTraversal#with(String, Object)} step modulator as all options aside from those in {@link IO} will be
+ * transferred.
*
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
@@ -69,7 +72,6 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
@Override
public void configure(final Object... keyValues) {
- // TODO: probably should write to the Configuration selectively - no need for actual Parameters?????????
this.parameters.set(null, keyValues);
}
@@ -121,6 +123,8 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
+
+ addParametersToConfiguration(graph);
}
private void configureForWrite(final Graph graph) {
@@ -137,6 +141,27 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
+
+ addParametersToConfiguration(graph);
+ }
+
+ /**
+ * Overwrites all configurations from values passed using {@link GraphTraversal#with(String, Object)}.
+ */
+ private void addParametersToConfiguration(final Graph graph) {
+ parameters.getRaw(IO.writer, IO.writer, IO.registry).entrySet().forEach(kv -> {
+ if (kv.getValue().size() == 1)
+ graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(0));
+ else {
+ // reset the default configuration with the first option then add to that for List options
+ for (int ix = 0; ix < kv.getValue().size(); ix++) {
+ if (ix == 0)
+ graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(ix));
+ else
+ graph.configuration().addProperty(kv.getKey().toString(), kv.getValue().get(ix));
+ }
+ }
+ });
}
private String detectReader() {
[10/50] tinkerpop git commit: TINKERPOP-1996 Added support/testing
for io() in GLVs
Posted by sp...@apache.org.
TINKERPOP-1996 Added support/testing for io() in GLVs
Had to revert to using iterate() and stop read/write() from terminating the traversal. Kinda stinks, but we rely on iterate() quite heavily and for remoting allowing read()/write() to terminate means that the traversal will execute during traversal construction in the translator (which is early and potentially bad).
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/048ea21c
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/048ea21c
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/048ea21c
Branch: refs/heads/TINKERPOP-1967
Commit: 048ea21c0adf9297233d06222e75c64dbe1fa1ca
Parents: f8e3b8a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 19 13:30:02 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:41:01 2018 -0400
----------------------------------------------------------------------
.../traversal/dsl/graph/GraphTraversal.java | 11 +--
gremlin-dotnet/glv/IO.template | 46 +++++++++++
gremlin-dotnet/glv/generate.groovy | 8 ++
.../src/Gremlin.Net/Process/Traversal/IO.cs | 54 +++++++++++++
.../Gherkin/TraversalEvaluation/IOParameter.cs | 82 +++++++++++++++++++
.../TraversalEvaluation/TraversalParser.cs | 7 ++
gremlin-javascript/glv/TraversalSource.template | 9 +++
gremlin-javascript/glv/generate.groovy | 4 +
.../gremlin-javascript/lib/process/traversal.js | 25 ++++++
.../test/cucumber/feature-steps.js | 2 +
gremlin-python/glv/TraversalSource.template | 11 +++
gremlin-python/glv/generate.groovy | 4 +
.../jython/gremlin_python/process/traversal.py | 19 +++++
.../src/main/jython/radish/feature_steps.py | 5 +-
gremlin-test/features/sideEffect/Read.feature | 84 ++++++++++++++++++++
gremlin-test/features/sideEffect/Write.feature | 60 ++++++++++++++
.../gremlin/AbstractGraphProvider.java | 2 +-
.../process/traversal/step/map/ReadTest.java | 57 ++++++++++---
18 files changed, 471 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
index 20f8996..a675ad1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
@@ -124,6 +124,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCount
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.InjectStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.LambdaSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SackValueStep;
@@ -2734,13 +2735,13 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
////
- ///////////////////// IO TERMINATOR STEPS /////////////////////
+ ///////////////////// IO STEPS /////////////////////
/**
* This step is technically a step modulator for the the {@link GraphTraversalSource#io(String)} step which
* instructs the step to perform a read with its given configuration.
*
- * @return the traversal that has been iterated with the read IO action executed
+ * @return the traversal with the {@link IoStep} modulated to read
* @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#io-step" target="_blank">Reference Documentation - IO Step</a>
* @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#read-step" target="_blank">Reference Documentation - Read Step</a>
* @since 3.4.0
@@ -2748,14 +2749,14 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
public default GraphTraversal<S,E> read() {
this.asAdmin().getBytecode().addStep(Symbols.read);
((ReadWriting) this.asAdmin().getEndStep()).setMode(ReadWriting.Mode.READING);
- return this.iterate();
+ return this;
}
/**
* This step is technically a step modulator for the the {@link GraphTraversalSource#io(String)} step which
* instructs the step to perform a write with its given configuration.
*
- * @return the traversal that has been iterated with the write IO action executed
+ * @return the traversal with the {@link IoStep} modulated to write
* @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#io-step" target="_blank">Reference Documentation - IO Step</a>
* @see <a href="http://tinkerpop.apache.org/docs/${project.version}/reference/#write-step" target="_blank">Reference Documentation - Write Step</a>
* @since 3.4.0
@@ -2763,7 +2764,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
public default GraphTraversal<S,E> write() {
this.asAdmin().getBytecode().addStep(Symbols.write);
((ReadWriting) this.asAdmin().getEndStep()).setMode(ReadWriting.Mode.WRITING);
- return this.iterate();
+ return this;
}
/**
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-dotnet/glv/IO.template
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/glv/IO.template b/gremlin-dotnet/glv/IO.template
new file mode 100644
index 0000000..7b25c88
--- /dev/null
+++ b/gremlin-dotnet/glv/IO.template
@@ -0,0 +1,46 @@
+#region License
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#endregion
+
+// THIS IS A GENERATED FILE - DO NOT MODIFY THIS FILE DIRECTLY - see pom.xml
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+namespace Gremlin.Net.Process.Traversal
+{
+#pragma warning disable 1591
+
+ /// <summary>
+ /// <see cref="IO" /> keeps configuration options for the io() step.
+ /// </summary>
+ public class IO
+ {
+ <% io.each {k,v -> %>
+ public const String <%= k %> = "<%= v %>";
+ <% } %>
+ }
+
+#pragma warning restore 1591
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-dotnet/glv/generate.groovy
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/glv/generate.groovy b/gremlin-dotnet/glv/generate.groovy
index bd2d9d7..58a5026 100644
--- a/gremlin-dotnet/glv/generate.groovy
+++ b/gremlin-dotnet/glv/generate.groovy
@@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource
import org.apache.tinkerpop.gremlin.process.traversal.P
+import org.apache.tinkerpop.gremlin.process.traversal.IO
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__
import org.apache.tinkerpop.gremlin.structure.Direction
import java.lang.reflect.Modifier
@@ -303,6 +304,9 @@ def binding = ["pmethods": P.class.getMethods().
def graphTraversalT2 = getGraphTraversalT2ForT2(t2)
return ["methodName": javaMethod.name, "t2":t2, "tParam":tParam, "parameters":parameters, "paramNames":paramNames, "callGenericTypeArg":callGenericTypeArg, "graphTraversalT2":graphTraversalT2]
},
+ "io": IO.class.getFields().
+ sort{ a, b -> a.name <=> b.name }.
+ collectEntries{ f -> [(f.name) : f.get(null)]},
"toCSharpMethodName": toCSharpMethodName]
def engine = new groovy.text.GStringTemplateEngine()
@@ -322,6 +326,10 @@ def pTemplate = engine.createTemplate(new File("${projectBaseDir}/glv/P.template
def pFile = new File("${projectBaseDir}/src/Gremlin.Net/Process/Traversal/P.cs")
pFile.newWriter().withWriter{ it << pTemplate }
+def ioTemplate = engine.createTemplate(new File("${projectBaseDir}/glv/IO.template")).make(binding)
+def ioFile = new File("${projectBaseDir}/src/Gremlin.Net/Process/Traversal/IO.cs")
+ioFile.newWriter().withWriter{ it << ioTemplate }
+
def createEnum = { enumClass ->
def b = ["enumClass": enumClass,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
new file mode 100644
index 0000000..288f7e3
--- /dev/null
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
@@ -0,0 +1,54 @@
+#region License
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#endregion
+
+// THIS IS A GENERATED FILE - DO NOT MODIFY THIS FILE DIRECTLY - see pom.xml
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+namespace Gremlin.Net.Process.Traversal
+{
+#pragma warning disable 1591
+
+ /// <summary>
+ /// <see cref="IO" /> keeps configuration options for the io() step.
+ /// </summary>
+ public class IO
+ {
+
+ public const String graphml = "graphml";
+
+ public const String graphson = "graphson";
+
+ public const String gryo = "gryo";
+
+ public const String reader = "~tinkerpop.io.reader";
+
+ public const String writer = "~tinkerpop.io.writer";
+
+ }
+
+#pragma warning restore 1591
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/IOParameter.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/IOParameter.cs b/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/IOParameter.cs
new file mode 100644
index 0000000..513e589
--- /dev/null
+++ b/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/IOParameter.cs
@@ -0,0 +1,82 @@
+#region License
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Dynamic;
+using System.Linq;
+using System.Reflection;
+using Gremlin.Net.Process.Traversal;
+
+namespace Gremlin.Net.IntegrationTest.Gherkin.TraversalEvaluation
+{
+ /// <summary>
+ /// Represents a parameter for the io() step - (e.g. IO.graphml)
+ /// </summary>
+ internal class IOParameter : ITokenParameter, IEquatable<IOParameter>
+ {
+ private readonly string _text;
+ private readonly string _value;
+
+ public IOParameter(string text)
+ {
+ _text = text;
+ var separatorIndex = text.IndexOf('.');
+ _value = text.Substring(separatorIndex + 1);
+ }
+
+ public bool Equals(IOParameter other)
+ {
+ return _text == other._text;
+ }
+
+ public override bool Equals(object obj)
+ {
+ if (ReferenceEquals(null, obj)) return false;
+ if (ReferenceEquals(this, obj)) return true;
+ if (obj.GetType() != GetType()) return false;
+ return Equals((IOParameter) obj);
+ }
+
+ public override int GetHashCode()
+ {
+ return _text.GetHashCode();
+ }
+
+ public object GetValue()
+ {
+ var field = typeof(IO).GetField(_value, BindingFlags.Static | BindingFlags.Public);
+ return field.GetValue(null);
+ }
+
+ public void SetContextParameterValues(IDictionary<string, object> parameterValues)
+ {
+
+ }
+
+ public Type GetParameterType()
+ {
+ return typeof(String);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/TraversalParser.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/TraversalParser.cs b/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/TraversalParser.cs
index e3f6a3f..f8e4095 100644
--- a/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/TraversalParser.cs
+++ b/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Gherkin/TraversalEvaluation/TraversalParser.cs
@@ -44,6 +44,8 @@ namespace Gremlin.Net.IntegrationTest.Gherkin.TraversalEvaluation
private static readonly Regex RegexEnum = new Regex(@"\w+\.\w+", RegexOptions.Compiled);
+ private static readonly Regex RegexIO = new Regex(@"IO.\w+", RegexOptions.Compiled);
+
private static readonly Regex RegexParam = new Regex(@"\w+", RegexOptions.Compiled);
private static readonly HashSet<Type> NumericTypes = new HashSet<Type>
@@ -424,6 +426,11 @@ namespace Gremlin.Net.IntegrationTest.Gherkin.TraversalEvaluation
i += parameterText.Length - 1;
return LiteralParameter.Create(Convert.ToBoolean(parameterText));
}
+ if (RegexIO.IsMatch(parameterText))
+ {
+ i += parameterText.Length - 1;
+ return new IOParameter(parameterText);
+ }
if (RegexEnum.IsMatch(parameterText))
{
i += parameterText.Length - 1;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-javascript/glv/TraversalSource.template
----------------------------------------------------------------------
diff --git a/gremlin-javascript/glv/TraversalSource.template b/gremlin-javascript/glv/TraversalSource.template
index 6965110..321956c 100644
--- a/gremlin-javascript/glv/TraversalSource.template
+++ b/gremlin-javascript/glv/TraversalSource.template
@@ -111,6 +111,14 @@ class Traversal {
};
}
+class IO {
+<% io.each {k,v -> %>
+ static get <%= k %>() {
+ return "<%= v %>"
+ }
+<% } %>
+}
+
class P {
/**
* Represents an operation.
@@ -190,6 +198,7 @@ class EnumValue {
module.exports = {
EnumValue,
P,
+ IO,
Traversal,
TraversalSideEffects,
Traverser<%
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-javascript/glv/generate.groovy
----------------------------------------------------------------------
diff --git a/gremlin-javascript/glv/generate.groovy b/gremlin-javascript/glv/generate.groovy
index 8778e89..a339689 100644
--- a/gremlin-javascript/glv/generate.groovy
+++ b/gremlin-javascript/glv/generate.groovy
@@ -24,6 +24,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource
import org.apache.tinkerpop.gremlin.process.traversal.P
+import org.apache.tinkerpop.gremlin.process.traversal.IO
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__
import java.lang.reflect.Modifier
@@ -88,6 +89,9 @@ def binding = ["enums": CoreImports.getClassImports()
collect { it.name }.
unique().
sort { a, b -> a <=> b },
+ "io": IO.class.getFields().
+ sort{ a, b -> a.name <=> b.name }.
+ collectEntries{ f -> [(f.name) : f.get(null)]},
"toJs": toJs,
"version": determineVersion(),
"decapitalize": decapitalize]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
----------------------------------------------------------------------
diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
index d39ccf0..3f69fb1 100644
--- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
+++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
@@ -111,6 +111,30 @@ class Traversal {
};
}
+class IO {
+
+ static get graphml() {
+ return "graphml"
+ }
+
+ static get graphson() {
+ return "graphson"
+ }
+
+ static get gryo() {
+ return "gryo"
+ }
+
+ static get reader() {
+ return "~tinkerpop.io.reader"
+ }
+
+ static get writer() {
+ return "~tinkerpop.io.writer"
+ }
+
+}
+
class P {
/**
* Represents an operation.
@@ -250,6 +274,7 @@ class EnumValue {
module.exports = {
EnumValue,
P,
+ IO,
Traversal,
TraversalSideEffects,
Traverser,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-javascript/src/main/javascript/gremlin-javascript/test/cucumber/feature-steps.js
----------------------------------------------------------------------
diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/test/cucumber/feature-steps.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/test/cucumber/feature-steps.js
index 16aae78..320d3f4 100644
--- a/gremlin-javascript/src/main/javascript/gremlin-javascript/test/cucumber/feature-steps.js
+++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/test/cucumber/feature-steps.js
@@ -187,6 +187,7 @@ function getSandbox(g, parameters) {
},
Order: traversalModule.order,
P: traversalModule.P,
+ IO: traversalModule.IO,
Pick: traversalModule.pick,
Pop: traversalModule.pop,
Scope: traversalModule.scope,
@@ -207,6 +208,7 @@ function translate(traversalText) {
// Change according to naming convention
traversalText = traversalText.replace(/\.in\(/g, '.in_(');
traversalText = traversalText.replace(/\.from\(/g, '.from_(');
+ traversalText = traversalText.replace(/\.with\(/g, '.with_(');
return traversalText;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-python/glv/TraversalSource.template
----------------------------------------------------------------------
diff --git a/gremlin-python/glv/TraversalSource.template b/gremlin-python/glv/TraversalSource.template
index b6f5b9f..3ca2786 100644
--- a/gremlin-python/glv/TraversalSource.template
+++ b/gremlin-python/glv/TraversalSource.template
@@ -135,6 +135,17 @@ statics.add_static('<%= method %>',<%= method %>)
<% } %>
'''
+IO
+'''
+
+
+class IO(object):
+<% io.each {k,v -> %>
+ <%= k %> = "<%= v %>"
+<% } %>
+
+
+'''
TRAVERSER
'''
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-python/glv/generate.groovy
----------------------------------------------------------------------
diff --git a/gremlin-python/glv/generate.groovy b/gremlin-python/glv/generate.groovy
index 57cc9c9..f810e10 100644
--- a/gremlin-python/glv/generate.groovy
+++ b/gremlin-python/glv/generate.groovy
@@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource
import org.apache.tinkerpop.gremlin.process.traversal.P
+import org.apache.tinkerpop.gremlin.process.traversal.IO
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__
import java.lang.reflect.Modifier
// this is a bit of a copy of what's in SymbolHelper - no way around it because this code generation task occurs
@@ -78,6 +79,9 @@ def binding = ["enums": CoreImports.getClassImports()
collect { toPython(it.name) }.
unique().
sort { a, b -> a <=> b },
+ "io": IO.class.getFields().
+ sort{ a, b -> a.name <=> b.name }.
+ collectEntries{ f -> [(f.name) : f.get(null)]},
"toPython": toPython,
"toJava": toJava]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
index 068c865..d9fb4d9 100644
--- a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
+++ b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
@@ -305,6 +305,25 @@ statics.add_static('without',without)
'''
+IO
+'''
+
+
+class IO(object):
+
+ graphml = "graphml"
+
+ graphson = "graphson"
+
+ gryo = "gryo"
+
+ reader = "~tinkerpop.io.reader"
+
+ writer = "~tinkerpop.io.writer"
+
+
+
+'''
TRAVERSER
'''
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-python/src/main/jython/radish/feature_steps.py
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/jython/radish/feature_steps.py b/gremlin-python/src/main/jython/radish/feature_steps.py
index 5067d1b..99022a0 100644
--- a/gremlin-python/src/main/jython/radish/feature_steps.py
+++ b/gremlin-python/src/main/jython/radish/feature_steps.py
@@ -21,7 +21,7 @@ import json
import re
from gremlin_python.structure.graph import Graph, Path
from gremlin_python.process.graph_traversal import __
-from gremlin_python.process.traversal import Barrier, Cardinality, P, Pop, Scope, Column, Order, Direction, T, Pick, Operator
+from gremlin_python.process.traversal import Barrier, Cardinality, P, Pop, Scope, Column, Order, Direction, T, Pick, Operator, IO
from radish import given, when, then
from hamcrest import *
@@ -34,6 +34,7 @@ regex_in = re.compile(r"([(.,\s])in\(")
regex_is = re.compile(r"([(.,\s])is\(")
regex_not = re.compile(r"([(.,\s])not\(")
regex_or = re.compile(r"([(.,\s])or\(")
+regex_with = re.compile(r"([(.,\s])with\(")
regex_true = re.compile(r"(true)")
regex_false = re.compile(r"(false)")
@@ -241,6 +242,7 @@ def _translate(traversal):
replaced = regex_not.sub(r"\1not_(", replaced)
replaced = regex_or.sub(r"\1or_(", replaced)
replaced = regex_in.sub(r"\1in_(", replaced)
+ replaced = regex_with.sub(r"\1with_(", replaced)
replaced = regex_true.sub(r"True", replaced)
return regex_false.sub(r"False", replaced)
@@ -254,6 +256,7 @@ def _make_traversal(g, traversal_string, params):
"Direction": Direction,
"Order": Order,
"P": P,
+ "IO": IO,
"Pick": Pick,
"Pop": Pop,
"Scope": Scope,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-test/features/sideEffect/Read.feature
----------------------------------------------------------------------
diff --git a/gremlin-test/features/sideEffect/Read.feature b/gremlin-test/features/sideEffect/Read.feature
new file mode 100644
index 0000000..ae96102
--- /dev/null
+++ b/gremlin-test/features/sideEffect/Read.feature
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+Feature: Step - read()
+
+ Scenario: g_io_readXkryoX
+ Given the empty graph
+ And the traversal of
+ """
+ g.io("data/tinkerpop-modern.kryo").read()
+ """
+ When iterated to list
+ Then the result should be empty
+ And the graph should return 6 for count of "g.V()"
+ And the graph should return 6 for count of "g.E()"
+
+ Scenario: g_io_read_withXreader_gryoX
+ Given the empty graph
+ And the traversal of
+ """
+ g.io("data/tinkerpop-modern.kryo").with(IO.reader, IO.gryo).read()
+ """
+ When iterated to list
+ Then the result should be empty
+ And the graph should return 6 for count of "g.V()"
+ And the graph should return 6 for count of "g.E()"
+
+ Scenario: g_io_readXgraphsonX
+ Given the empty graph
+ And the traversal of
+ """
+ g.io("data/tinkerpop-modern.json").read()
+ """
+ When iterated to list
+ Then the result should be empty
+ And the graph should return 6 for count of "g.V()"
+ And the graph should return 6 for count of "g.E()"
+
+ Scenario: g_io_read_withXreader_graphsonX
+ Given the empty graph
+ And the traversal of
+ """
+ g.io("data/tinkerpop-modern.json").with(IO.reader, IO.graphson).read()
+ """
+ When iterated to list
+ Then the result should be empty
+ And the graph should return 6 for count of "g.V()"
+ And the graph should return 6 for count of "g.E()"
+
+ Scenario: g_io_readXgraphmlX
+ Given the empty graph
+ And the traversal of
+ """
+ g.io("data/tinkerpop-modern.xml").read()
+ """
+ When iterated to list
+ Then the result should be empty
+ And the graph should return 6 for count of "g.V()"
+ And the graph should return 6 for count of "g.E()"
+
+ Scenario: g_io_read_withXreader_graphmlX
+ Given the empty graph
+ And the traversal of
+ """
+ g.io("data/tinkerpop-modern.xml").with(IO.reader, IO.graphml).read()
+ """
+ When iterated to list
+ Then the result should be empty
+ And the graph should return 6 for count of "g.V()"
+ And the graph should return 6 for count of "g.E()"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-test/features/sideEffect/Write.feature
----------------------------------------------------------------------
diff --git a/gremlin-test/features/sideEffect/Write.feature b/gremlin-test/features/sideEffect/Write.feature
new file mode 100644
index 0000000..9a774e8
--- /dev/null
+++ b/gremlin-test/features/sideEffect/Write.feature
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+Feature: Step - write()
+
+ Scenario: g_io_writeXkryoX
+ Given an unsupported test
+ Then nothing should happen because
+ """
+ We don't have a nice way to assert the remotely written file with this framework - just testing compilation.
+ """
+
+ Scenario: g_io_write_withXwriter_gryoX
+ Given an unsupported test
+ Then nothing should happen because
+ """
+ We don't have a nice way to assert the remotely written file with this framework - just testing compilation.
+ """
+
+ Scenario: g_io_writeXgraphsonX
+ Given an unsupported test
+ Then nothing should happen because
+ """
+ We don't have a nice way to assert the remotely written file with this framework - just testing compilation.
+ """
+
+ Scenario: g_io_write_withXwriter_graphsonX
+ Given an unsupported test
+ Then nothing should happen because
+ """
+ We don't have a nice way to assert the remotely written file with this framework - just testing compilation.
+ """
+
+ Scenario: g_io_writeXgraphmlX
+ Given an unsupported test
+ Then nothing should happen because
+ """
+ We don't have a nice way to assert the remotely written file with this framework - just testing compilation.
+ """
+
+ Scenario: g_io_write_withXwriter_graphmlX
+ Given an unsupported test
+ Then nothing should happen because
+ """
+ We don't have a nice way to assert the remotely written file with this framework - just testing compilation.
+ """
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
index b6fc43c..9774411 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
@@ -145,6 +145,6 @@ public abstract class AbstractGraphProvider implements GraphProvider {
protected void readIntoGraph(final Graph graph, final String path) throws IOException {
final String dataFile = TestHelper.generateTempFileFromResource(graph.getClass(),
GryoResourceAccess.class, path.substring(path.lastIndexOf(File.separator) + 1), "", false).getAbsolutePath();
- graph.traversal().io(dataFile).read();
+ graph.traversal().io(dataFile).read().iterate();
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/048ea21c/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
index ab59194..24bacbb 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
+import org.apache.tinkerpop.gremlin.process.remote.RemoteGraph;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -34,8 +35,10 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import java.io.IOException;
+import java.util.function.Supplier;
import static org.apache.tinkerpop.gremlin.structure.Graph.Features.GraphFeatures.FEATURE_IO_READ;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
/**
@@ -62,9 +65,14 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXkryoX(fileToRead);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
- IoTest.assertModernGraph(graph, false, true);
+ if (graph instanceof RemoteGraph) {
+ assertEquals(6L, g.V().count().next().longValue());
+ assertEquals(6L, g.E().count().next().longValue());
+ } else {
+ IoTest.assertModernGraph(graph, false, true);
+ }
}
@Test
@@ -73,9 +81,14 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_gryoX(fileToRead);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
- IoTest.assertModernGraph(graph, false, true);
+ if (graph instanceof RemoteGraph) {
+ assertEquals(6L, g.V().count().next().longValue());
+ assertEquals(6L, g.E().count().next().longValue());
+ } else {
+ IoTest.assertModernGraph(graph, false, true);
+ }
}
@Test
@@ -84,9 +97,14 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXjsonX(fileToRead);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
- IoTest.assertModernGraph(graph, false, true);
+ if (graph instanceof RemoteGraph) {
+ assertEquals(6L, g.V().count().next().longValue());
+ assertEquals(6L, g.E().count().next().longValue());
+ } else {
+ IoTest.assertModernGraph(graph, false, true);
+ }
}
@Test
@@ -95,9 +113,14 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphsonX(fileToRead);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
- IoTest.assertModernGraph(graph, false, true);
+ if (graph instanceof RemoteGraph) {
+ assertEquals(6L, g.V().count().next().longValue());
+ assertEquals(6L, g.E().count().next().longValue());
+ } else {
+ IoTest.assertModernGraph(graph, false, true);
+ }
}
@Test
@@ -106,9 +129,14 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXxmlX(fileToRead);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
- IoTest.assertModernGraph(graph, false, true);
+ if (graph instanceof RemoteGraph) {
+ assertEquals(6L, g.V().count().next().longValue());
+ assertEquals(6L, g.E().count().next().longValue());
+ } else {
+ IoTest.assertModernGraph(graph, false, true);
+ }
}
@Test
@@ -117,9 +145,14 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphmlX(fileToRead);
printTraversalForm(traversal);
- assertFalse(traversal.hasNext());
+ traversal.iterate();
- IoTest.assertModernGraph(graph, false, true);
+ if (graph instanceof RemoteGraph) {
+ assertEquals(6L, g.V().count().next().longValue());
+ assertEquals(6L, g.E().count().next().longValue());
+ } else {
+ IoTest.assertModernGraph(graph, false, true);
+ }
}
public static class Traversals extends ReadTest {
[13/50] tinkerpop git commit: TINKERPOP-1996 Enabled feature coverage
checks for GLV tests on read()/write()
Posted by sp...@apache.org.
TINKERPOP-1996 Enabled feature coverage checks for GLV tests on read()/write()
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/a580b6fd
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/a580b6fd
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/a580b6fd
Branch: refs/heads/TINKERPOP-1967
Commit: a580b6fda232ce6dd035e4c261a53d0f0dc69f83
Parents: 048ea21
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 19 14:08:25 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 14:08:25 2018 -0400
----------------------------------------------------------------------
.../apache/tinkerpop/gremlin/process/FeatureCoverageTest.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/a580b6fd/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/process/FeatureCoverageTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/process/FeatureCoverageTest.java b/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/process/FeatureCoverageTest.java
index 503df77..5739629 100644
--- a/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/process/FeatureCoverageTest.java
+++ b/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/process/FeatureCoverageTest.java
@@ -56,11 +56,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest;
@@ -158,10 +160,12 @@ public class FeatureCoverageTest {
GroupCountTest.class,
GroupTest.class,
InjectTest.class,
+ ReadTest.class,
SackTest.class,
SideEffectCapTest.class,
//SideEffectTest.class,
- StoreTest.class);
+ StoreTest.class,
+ WriteTest.class);
// SubgraphTest.class,
// TreeTest.class);
[06/50] tinkerpop git commit: TINKERPOP-1996 Moved IoStep
implementations to sideEffect package
Posted by sp...@apache.org.
TINKERPOP-1996 Moved IoStep implementations to sideEffect package
These steps really aren't quite sideEffects and not quite map steps either but they seem to fit better as sideEffect. meh
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/6d05805a
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/6d05805a
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/6d05805a
Branch: refs/heads/TINKERPOP-1967
Commit: 6d05805ada657bcb3f50a60aa0c313c29d4611bb
Parents: 62175c2
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Sat Jul 14 06:23:54 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:10 2018 -0400
----------------------------------------------------------------------
.../dsl/graph/GraphTraversalSource.java | 2 +-
.../process/traversal/step/map/IoStep.java | 241 -------------------
.../traversal/step/sideEffect/IoStep.java | 241 +++++++++++++++++++
.../traversal/step/map/HadoopIoStep.java | 170 -------------
.../traversal/step/sideEffect/HadoopIoStep.java | 163 +++++++++++++
.../traversal/strategy/HadoopIoStrategy.java | 5 +-
6 files changed, 407 insertions(+), 415 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java
index 7357418..df1b108 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java
@@ -31,7 +31,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddEdgeStartStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddVertexStartStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.IoStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.InjectStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.RequirementsStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java
deleted file mode 100644
index 668b3dc..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.process.traversal.step.map;
-
-import org.apache.tinkerpop.gremlin.process.traversal.IO;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser;
-import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
-import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
-import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
-import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.lang.reflect.Method;
-
-/**
- * Handles read and write operations into the {@link Graph}.
- *
- * @author Stephen Mallette (http://stephen.genoprime.com)
- */
-public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
-
- private enum Format {
- GRYO,
- GRAPHSON,
- GRAPHML
- }
-
- private Parameters parameters = new Parameters();
- private boolean first = true;
- private String file;
- private Mode mode = Mode.UNSET;
-
- public IoStep(final Traversal.Admin traversal, final String file) {
- super(traversal);
-
- if (null == file || file.isEmpty())
- throw new IllegalArgumentException("file cannot be null or empty");
-
- this.file = file;
- }
-
- @Override
- public void setMode(final Mode mode) {
- this.mode = mode;
- }
-
- @Override
- public Mode getMode() {
- return mode;
- }
-
- @Override
- public String getFile() {
- return file;
- }
-
- @Override
- public Parameters getParameters() {
- return this.parameters;
- }
-
- @Override
- public void configure(final Object... keyValues) {
- this.parameters.set(null, keyValues);
- }
-
- @Override
- protected Traverser.Admin<S> processNextStart() {
- if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()");
- if (!this.first) throw FastNoSuchElementException.instance();
-
- this.first = false;
- final File file = new File(this.file);
-
- if (mode == Mode.READING) {
- if (!file.exists()) throw new IllegalStateException(this.file + " does not exist");
- return read(file);
- } else if (mode == Mode.WRITING) {
- return write(file);
- } else {
- throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name());
- }
- }
-
- private Traverser.Admin<S> write(final File file) {
- try (final OutputStream stream = new FileOutputStream(file)) {
- final Graph graph = (Graph) this.traversal.getGraph().get();
- constructWriter().writeGraph(stream, graph);
-
- return EmptyTraverser.instance();
- } catch (IOException ioe) {
- throw new IllegalStateException(String.format("Could not write file %s from graph", this.file), ioe);
- }
- }
-
- private Traverser.Admin<S> read(final File file) {
- try (final InputStream stream = new FileInputStream(file)) {
- final Graph graph = (Graph) this.traversal.getGraph().get();
- constructReader().readGraph(stream, graph);
-
- return EmptyTraverser.instance();
- } catch (IOException ioe) {
- throw new IllegalStateException(String.format("Could not read file %s into graph", this.file), ioe);
- }
- }
-
- /**
- * Builds a {@link GraphReader} instance to use. Attempts to detect the file format to be read using the file
- * extension or simply uses configurations provided by the user on the parameters given to the step.
- */
- private GraphReader constructReader() {
- final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0);
- if (objectOrClass instanceof GraphReader)
- return (GraphReader) objectOrClass;
- else if (objectOrClass instanceof String) {
- if (objectOrClass.equals(IO.graphson))
- return GraphSONReader.build().create();
- else if (objectOrClass.equals(IO.gryo))
- return GryoReader.build().create();
- else if (objectOrClass.equals(IO.graphml))
- return GraphMLReader.build().create();
- else {
- try {
- final Class<?> graphReaderClazz = Class.forName((String) objectOrClass);
- final Method build = graphReaderClazz.getMethod("build");
- final GraphReader.ReaderBuilder builder = (GraphReader.ReaderBuilder) build.invoke(null);
- return builder.create();
- } catch (Exception ex) {
- throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex);
- }
- }
- } else {
- throw new IllegalStateException("GraphReader could not be determined");
- }
- }
-
- private GraphReader detectReader() {
- if (file.endsWith(".kryo"))
- return GryoReader.build().create();
- else if (file.endsWith(".json"))
- return GraphSONReader.build().create();
- else if (file.endsWith(".xml"))
- return GraphMLReader.build().create();
- else
- throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
- }
-
- /**
- * Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file
- * extension or simply uses configurations provided by the user on the parameters given to the step.
- */
- private GraphWriter constructWriter() {
- final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0);
- if (objectOrClass instanceof GraphWriter)
- return (GraphWriter) objectOrClass;
- else if (objectOrClass instanceof String) {
- if (objectOrClass.equals(IO.graphson))
- return GraphSONWriter.build().create();
- else if (objectOrClass.equals(IO.gryo))
- return GryoWriter.build().create();
- else if (objectOrClass.equals(IO.graphml))
- return GraphMLWriter.build().create();
- else {
- try {
- final Class<?> graphWriterClazz = Class.forName((String) objectOrClass);
- final Method build = graphWriterClazz.getMethod("build");
- final GraphWriter.WriterBuilder builder = (GraphWriter.WriterBuilder) build.invoke(null);
- return builder.create();
- } catch (Exception ex) {
- throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex);
- }
- }
- } else {
- throw new IllegalStateException("GraphReader could not be determined");
- }
- }
-
- private GraphWriter detectWriter() {
- if (file.endsWith(".kryo"))
- return GryoWriter.build().create();
- else if (file.endsWith(".json"))
- return GraphSONWriter.build().create();
- else if (file.endsWith(".xml"))
- return GraphMLWriter.build().create();
- else
- throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension");
- }
-
- @Override
- public int hashCode() {
- final int hash = super.hashCode() ^ this.parameters.hashCode();
- return (null != this.file) ? (hash ^ file.hashCode()) : hash;
- }
-
- @Override
- public String toString() {
- return StringFactory.stepString(this, file, this.parameters);
- }
-
- @Override
- public IoStep clone() {
- final IoStep clone = (IoStep) super.clone();
- clone.parameters = this.parameters.clone();
- clone.file = this.file;
- clone.mode = this.mode;
- return clone;
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
new file mode 100644
index 0000000..74b295b
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect;
+
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser;
+import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Method;
+
+/**
+ * Handles read and write operations into the {@link Graph}.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
+
+ private enum Format {
+ GRYO,
+ GRAPHSON,
+ GRAPHML
+ }
+
+ private Parameters parameters = new Parameters();
+ private boolean first = true;
+ private String file;
+ private Mode mode = Mode.UNSET;
+
+ public IoStep(final Traversal.Admin traversal, final String file) {
+ super(traversal);
+
+ if (null == file || file.isEmpty())
+ throw new IllegalArgumentException("file cannot be null or empty");
+
+ this.file = file;
+ }
+
+ @Override
+ public void setMode(final Mode mode) {
+ this.mode = mode;
+ }
+
+ @Override
+ public Mode getMode() {
+ return mode;
+ }
+
+ @Override
+ public String getFile() {
+ return file;
+ }
+
+ @Override
+ public Parameters getParameters() {
+ return this.parameters;
+ }
+
+ @Override
+ public void configure(final Object... keyValues) {
+ this.parameters.set(null, keyValues);
+ }
+
+ @Override
+ protected Traverser.Admin<S> processNextStart() {
+ if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()");
+ if (!this.first) throw FastNoSuchElementException.instance();
+
+ this.first = false;
+ final File file = new File(this.file);
+
+ if (mode == Mode.READING) {
+ if (!file.exists()) throw new IllegalStateException(this.file + " does not exist");
+ return read(file);
+ } else if (mode == Mode.WRITING) {
+ return write(file);
+ } else {
+ throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name());
+ }
+ }
+
+ private Traverser.Admin<S> write(final File file) {
+ try (final OutputStream stream = new FileOutputStream(file)) {
+ final Graph graph = (Graph) this.traversal.getGraph().get();
+ constructWriter().writeGraph(stream, graph);
+
+ return EmptyTraverser.instance();
+ } catch (IOException ioe) {
+ throw new IllegalStateException(String.format("Could not write file %s from graph", this.file), ioe);
+ }
+ }
+
+ private Traverser.Admin<S> read(final File file) {
+ try (final InputStream stream = new FileInputStream(file)) {
+ final Graph graph = (Graph) this.traversal.getGraph().get();
+ constructReader().readGraph(stream, graph);
+
+ return EmptyTraverser.instance();
+ } catch (IOException ioe) {
+ throw new IllegalStateException(String.format("Could not read file %s into graph", this.file), ioe);
+ }
+ }
+
+ /**
+ * Builds a {@link GraphReader} instance to use. Attempts to detect the file format to be read using the file
+ * extension or simply uses configurations provided by the user on the parameters given to the step.
+ */
+ private GraphReader constructReader() {
+ final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0);
+ if (objectOrClass instanceof GraphReader)
+ return (GraphReader) objectOrClass;
+ else if (objectOrClass instanceof String) {
+ if (objectOrClass.equals(IO.graphson))
+ return GraphSONReader.build().create();
+ else if (objectOrClass.equals(IO.gryo))
+ return GryoReader.build().create();
+ else if (objectOrClass.equals(IO.graphml))
+ return GraphMLReader.build().create();
+ else {
+ try {
+ final Class<?> graphReaderClazz = Class.forName((String) objectOrClass);
+ final Method build = graphReaderClazz.getMethod("build");
+ final GraphReader.ReaderBuilder builder = (GraphReader.ReaderBuilder) build.invoke(null);
+ return builder.create();
+ } catch (Exception ex) {
+ throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex);
+ }
+ }
+ } else {
+ throw new IllegalStateException("GraphReader could not be determined");
+ }
+ }
+
+ private GraphReader detectReader() {
+ if (file.endsWith(".kryo"))
+ return GryoReader.build().create();
+ else if (file.endsWith(".json"))
+ return GraphSONReader.build().create();
+ else if (file.endsWith(".xml"))
+ return GraphMLReader.build().create();
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
+
+ /**
+ * Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file
+ * extension or simply uses configurations provided by the user on the parameters given to the step.
+ */
+ private GraphWriter constructWriter() {
+ final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0);
+ if (objectOrClass instanceof GraphWriter)
+ return (GraphWriter) objectOrClass;
+ else if (objectOrClass instanceof String) {
+ if (objectOrClass.equals(IO.graphson))
+ return GraphSONWriter.build().create();
+ else if (objectOrClass.equals(IO.gryo))
+ return GryoWriter.build().create();
+ else if (objectOrClass.equals(IO.graphml))
+ return GraphMLWriter.build().create();
+ else {
+ try {
+ final Class<?> graphWriterClazz = Class.forName((String) objectOrClass);
+ final Method build = graphWriterClazz.getMethod("build");
+ final GraphWriter.WriterBuilder builder = (GraphWriter.WriterBuilder) build.invoke(null);
+ return builder.create();
+ } catch (Exception ex) {
+ throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex);
+ }
+ }
+ } else {
+ throw new IllegalStateException("GraphReader could not be determined");
+ }
+ }
+
+ private GraphWriter detectWriter() {
+ if (file.endsWith(".kryo"))
+ return GryoWriter.build().create();
+ else if (file.endsWith(".json"))
+ return GraphSONWriter.build().create();
+ else if (file.endsWith(".xml"))
+ return GraphMLWriter.build().create();
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension");
+ }
+
+ @Override
+ public int hashCode() {
+ final int hash = super.hashCode() ^ this.parameters.hashCode();
+ return (null != this.file) ? (hash ^ file.hashCode()) : hash;
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.stepString(this, file, this.parameters);
+ }
+
+ @Override
+ public IoStep clone() {
+ final IoStep clone = (IoStep) super.clone();
+ clone.parameters = this.parameters.clone();
+ clone.file = this.file;
+ clone.mode = this.mode;
+ return clone;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
deleted file mode 100644
index 97fdea4..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.IO;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
-import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
-import java.lang.reflect.Method;
-
-/**
- * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the
- * {@link CloneVertexProgram} for its implementation.
- *
- * @author Stephen Mallette (http://stephen.genoprime.com)
- */
-public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
-
- private Parameters parameters = new Parameters();
- private Mode mode = Mode.UNSET;
- private String file;
-
- public HadoopIoStep(final Traversal.Admin traversal, final String file) {
- super(traversal);
- this.file = file;
- }
-
- @Override
- public void setMode(final Mode mode) {
- this.mode = mode;
- }
-
- @Override
- public Mode getMode() {
- return mode;
- }
-
- @Override
- public String getFile() {
- return file;
- }
-
- @Override
- public void configure(final Object... keyValues) {
- // TODO: probably should write to the Configuration selectively - no need for actual Parameters?????????
- this.parameters.set(null, keyValues);
- }
-
- @Override
- public Parameters getParameters() {
- return parameters;
- }
-
- @Override
- public String toString() {
- return StringFactory.stepString(this, new GraphFilter(this.computer));
- }
-
- @Override
- public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) {
- if (mode == Mode.UNSET)
- throw new IllegalStateException("IO mode was not set to read() or write()");
- else if (mode == Mode.READING)
- configureForRead(graph);
- else if (mode == Mode.WRITING)
- configureForWrite(graph);
- else
- throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name());
-
- return CloneVertexProgram.build().create(graph);
- }
-
- @Override
- public HadoopIoStep clone() {
- return (HadoopIoStep) super.clone();
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
-
- private void configureForRead(final Graph graph) {
- final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0);
- String inputFormatClassName;
- if (inputFormatClassNameOrKeyword.equals(IO.graphson))
- inputFormatClassName = GraphSONInputFormat.class.getName();
- else if (inputFormatClassNameOrKeyword.equals(IO.gryo))
- inputFormatClassName = GryoInputFormat.class.getName();
- else if (inputFormatClassNameOrKeyword.equals(IO.graphml))
- throw new IllegalStateException("GraphML is not a supported file format for OLAP");
- else
- inputFormatClassName = inputFormatClassNameOrKeyword;
-
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
- }
-
- private void configureForWrite(final Graph graph) {
- final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0);
- String outputFormatClassName;
- if (outputFormatClassNameOrKeyword.equals(IO.graphson))
- outputFormatClassName = GraphSONOutputFormat.class.getName();
- else if (outputFormatClassNameOrKeyword.equals(IO.gryo))
- outputFormatClassName = GryoOutputFormat.class.getName();
- else if (outputFormatClassNameOrKeyword.equals(IO.graphml))
- throw new IllegalStateException("GraphML is not a supported file format for OLAP");
- else
- outputFormatClassName = outputFormatClassNameOrKeyword;
-
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
- }
-
- private String detectReader() {
- if (file.endsWith(".kryo"))
- return GryoInputFormat.class.getName();
- else if (file.endsWith(".json"))
- return GraphSONInputFormat.class.getName();
- else if (file.endsWith(".xml"))
- throw new IllegalStateException("GraphML is not a supported file format for OLAP");
- else
- throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
- }
-
- private String detectWriter() {
- if (file.endsWith(".kryo"))
- return GryoOutputFormat.class.getName();
- else if (file.endsWith(".json"))
- return GraphSONOutputFormat.class.getName();
- else if (file.endsWith(".xml"))
- throw new IllegalStateException("GraphML is not a supported file format for OLAP");
- else
- throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
new file mode 100644
index 0000000..ca369b6
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.sideEffect;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+/**
+ * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the
+ * {@link CloneVertexProgram} for its implementation.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
+
+ private Parameters parameters = new Parameters();
+ private Mode mode = Mode.UNSET;
+ private String file;
+
+ public HadoopIoStep(final Traversal.Admin traversal, final String file) {
+ super(traversal);
+ this.file = file;
+ }
+
+ @Override
+ public void setMode(final Mode mode) {
+ this.mode = mode;
+ }
+
+ @Override
+ public Mode getMode() {
+ return mode;
+ }
+
+ @Override
+ public String getFile() {
+ return file;
+ }
+
+ @Override
+ public void configure(final Object... keyValues) {
+ // TODO: probably should write to the Configuration selectively - no need for actual Parameters?????????
+ this.parameters.set(null, keyValues);
+ }
+
+ @Override
+ public Parameters getParameters() {
+ return parameters;
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.stepString(this, new GraphFilter(this.computer));
+ }
+
+ @Override
+ public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) {
+ if (mode == Mode.UNSET)
+ throw new IllegalStateException("IO mode was not set to read() or write()");
+ else if (mode == Mode.READING)
+ configureForRead(graph);
+ else if (mode == Mode.WRITING)
+ configureForWrite(graph);
+ else
+ throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name());
+
+ return CloneVertexProgram.build().create(graph);
+ }
+
+ @Override
+ public HadoopIoStep clone() {
+ return (HadoopIoStep) super.clone();
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ private void configureForRead(final Graph graph) {
+ final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0);
+ String inputFormatClassName;
+ if (inputFormatClassNameOrKeyword.equals(IO.graphson))
+ inputFormatClassName = GraphSONInputFormat.class.getName();
+ else if (inputFormatClassNameOrKeyword.equals(IO.gryo))
+ inputFormatClassName = GryoInputFormat.class.getName();
+ else if (inputFormatClassNameOrKeyword.equals(IO.graphml))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ inputFormatClassName = inputFormatClassNameOrKeyword;
+
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
+ }
+
+ private void configureForWrite(final Graph graph) {
+ final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0);
+ String outputFormatClassName;
+ if (outputFormatClassNameOrKeyword.equals(IO.graphson))
+ outputFormatClassName = GraphSONOutputFormat.class.getName();
+ else if (outputFormatClassNameOrKeyword.equals(IO.gryo))
+ outputFormatClassName = GryoOutputFormat.class.getName();
+ else if (outputFormatClassNameOrKeyword.equals(IO.graphml))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ outputFormatClassName = outputFormatClassNameOrKeyword;
+
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
+ }
+
+ private String detectReader() {
+ if (file.endsWith(".kryo"))
+ return GryoInputFormat.class.getName();
+ else if (file.endsWith(".json"))
+ return GraphSONInputFormat.class.getName();
+ else if (file.endsWith(".xml"))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
+
+ private String detectWriter() {
+ if (file.endsWith(".kryo"))
+ return GryoOutputFormat.class.getName();
+ else if (file.endsWith(".json"))
+ return GraphSONOutputFormat.class.getName();
+ else if (file.endsWith(".xml"))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
index cbc9b07..8348410 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -19,15 +19,14 @@
package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map.HadoopIoStep;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.sideEffect.HadoopIoStep;
import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
-import org.apache.tinkerpop.gremlin.process.traversal.step.filter.NoneStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.IoStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
[41/50] tinkerpop git commit: Merge branch 'tp32' into tp33
Posted by sp...@apache.org.
Merge branch 'tp32' into tp33
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/c49c0ccb
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/c49c0ccb
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/c49c0ccb
Branch: refs/heads/TINKERPOP-1967
Commit: c49c0ccb3bce76caabb4b0a9c7efe9df43801b39
Parents: ddc6694 e3018fb
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Jul 31 08:35:24 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jul 31 08:35:24 2018 -0400
----------------------------------------------------------------------
docs/src/dev/developer/development-environment.asciidoc | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c49c0ccb/docs/src/dev/developer/development-environment.asciidoc
----------------------------------------------------------------------
[33/50] tinkerpop git commit: Bumped default Xmx on Gremlin Server
Posted by sp...@apache.org.
Bumped default Xmx on Gremlin Server
Seems like the default is too small for anything but the toy graphs and people getting started who aren't JVM experts keep running into problems CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/2b045f30
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/2b045f30
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/2b045f30
Branch: refs/heads/TINKERPOP-1967
Commit: 2b045f30db0577464f1ff353464c7cf5d3c41746
Parents: 3b8c828
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 30 07:14:42 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Jul 30 07:14:42 2018 -0400
----------------------------------------------------------------------
gremlin-server/src/main/bin/gremlin-server.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2b045f30/gremlin-server/src/main/bin/gremlin-server.sh
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/bin/gremlin-server.sh b/gremlin-server/src/main/bin/gremlin-server.sh
index fcede2b..06b065a 100755
--- a/gremlin-server/src/main/bin/gremlin-server.sh
+++ b/gremlin-server/src/main/bin/gremlin-server.sh
@@ -50,7 +50,7 @@ fi
# Set Java options
if [ "$JAVA_OPTIONS" = "" ] ; then
- JAVA_OPTIONS="-Xms32m -Xmx512m"
+ JAVA_OPTIONS="-Xms512m -Xmx4096m"
fi
# Execute the application and return its exit code
[25/50] tinkerpop git commit: TINKERPOP-1996 Fixed verification on
io()
Posted by sp...@apache.org.
TINKERPOP-1996 Fixed verification on io()
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/23c71b60
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/23c71b60
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/23c71b60
Branch: refs/heads/TINKERPOP-1967
Commit: 23c71b60cc34c14e9defc6a9a7ab6c57c090460b
Parents: ae3f685
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 20 13:22:47 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Jul 20 13:22:47 2018 -0400
----------------------------------------------------------------------
.../verification/StandardVerificationStrategy.java | 6 ++++--
.../StandardVerificationStrategyTest.java | 16 +++++++++++-----
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/23c71b60/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
index 258d345..7d88ed9 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategy.java
@@ -84,8 +84,10 @@ public final class StandardVerificationStrategy extends AbstractTraversalStrateg
throw new VerificationException("The profile()-Step cannot be specified multiple times.", traversal);
}
- if (traversal.getStartStep() instanceof ReadWriting && traversal.getSteps().size() != 1) {
- if (TraversalHelper.getStepsOfClass(NoneStep.class, traversal).size() != traversal.getSteps().size() - 1)
+ if (traversal.getStartStep() instanceof ReadWriting && !endStep.equals(traversal.getStartStep())) {
+ final int total = TraversalHelper.getStepsOfClass(NoneStep.class, traversal).size() +
+ TraversalHelper.getStepsOfClass(RequirementsStep.class, traversal).size() + 1;
+ if (total != traversal.getSteps().size())
throw new VerificationException("The io() step must be the first and only step in the traversal - it cannot be used with other steps", traversal);
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/23c71b60/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
index deec002..96a68b2 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/StandardVerificationStrategyTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.strategy.verification;
+import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
@@ -29,6 +30,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.File;
import java.util.Arrays;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.out;
@@ -44,18 +46,22 @@ import static org.junit.Assert.fail;
@RunWith(Parameterized.class)
public class StandardVerificationStrategyTest {
+
@Parameterized.Parameters(name = "{0}")
- public static Iterable<Object[]> data() {
+ public static Iterable<Object[]> data() throws Exception {
+
+ final String file = TestHelper.generateTempFile(StandardVerificationStrategyTest.class, "shouldBeVerified", ".kryo").getAbsolutePath();
+
return Arrays.asList(new Object[][]{
// traversals that should fail verification
{"__.repeat(out().fold().unfold()).times(2)", repeat(out().fold().unfold()).times(2), false},
{"__.repeat(sum()).times(2)", repeat(sum()).times(2), false},
{"__.repeat(out().count())", repeat(out().count()), false},
- {"__.io().read().V()", EmptyGraph.instance().traversal().io("junk.kryo").read().V(), false},
- {"__.io().write().V()", EmptyGraph.instance().traversal().io("junk.kryo").write().V(), false},
+ {"__.io().read().V()", EmptyGraph.instance().traversal().io(file).read().V(), false},
+ {"__.io().write().V()", EmptyGraph.instance().traversal().io(file).write().V(), false},
// traversals that should pass verification
- {"__.io().read().V().none()", EmptyGraph.instance().traversal().io("junk.kryo").read().none(), true},
- {"__.io().write().V().none()", EmptyGraph.instance().traversal().io("junk.kryo").write().none(), true},
+ {"__.io().read()", EmptyGraph.instance().traversal().io(file).read(), true},
+ {"__.io().write()", EmptyGraph.instance().traversal().io(file).write(), true},
{"__.V().profile()",
__.V().profile(), true},
{"__.V().profile('metrics').cap('metrics')",
[12/50] tinkerpop git commit: TINKERPOP-1996 Used g.io() in tests by
default
Posted by sp...@apache.org.
TINKERPOP-1996 Used g.io() in tests by default
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/81870168
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/81870168
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/81870168
Branch: refs/heads/TINKERPOP-1967
Commit: 8187016886b8b699ff107c9b6a7dfe95deb7e4a1
Parents: 5bf19e2
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Jul 16 12:10:02 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:41:01 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 5 ++--
.../gremlin/AbstractGraphProvider.java | 8 +++----
.../apache/tinkerpop/gremlin/TestHelper.java | 24 ++++++++++++++------
3 files changed, 24 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/81870168/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 1add03c..1848421 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -25,11 +25,12 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
This release also includes changes from <<release-3-3-3, 3.3.3>>.
-* Bumped to Netty 4.1.25.
-* Bumped to Spark 2.3.1.
+* `AbstractGraphProvider` uses `g.io()` for loading test data.
* Added the `io()` start step and `read()` and `write()` termination steps to the Gremlin language.
* Added `GraphFeatures.supportsIoRead()` and `GraphFeatures.supportsIoWrite()`.
* Deprecated `Graph.io()` and related infrastructure.
+* Bumped to Netty 4.1.25.
+* Bumped to Spark 2.3.1.
* Moved `Parameterizing` interface to the `org.apache.tinkerpop.gremlin.process.traversal.step` package with other marker interfaces of its type.
* Replaced `Parameterizing.addPropertyMutations()` with `Configuring.configure()`.
* Changed interface hierarchy for `Parameterizing` and `Mutating` interfaces as they are tightly related.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/81870168/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
index 75d033b..b6fc43c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,9 +143,8 @@ public abstract class AbstractGraphProvider implements GraphProvider {
* @param path the path to the file to load into the graph
*/
protected void readIntoGraph(final Graph graph, final String path) throws IOException {
- final GraphReader reader = GryoReader.build().create();
- try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream(path)) {
- reader.readGraph(stream, graph);
- }
+ final String dataFile = TestHelper.generateTempFileFromResource(graph.getClass(),
+ GryoResourceAccess.class, path.substring(path.lastIndexOf(File.separator) + 1), "", false).getAbsolutePath();
+ graph.traversal().io(dataFile).read();
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/81870168/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
index 38d9a25..cda32e2 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/TestHelper.java
@@ -152,17 +152,27 @@ public final class TestHelper {
* {@link TestHelper#makeTestDataPath} in a subdirectory called {@code temp/resources}.
*/
public static File generateTempFileFromResource(final Class graphClass, final Class resourceClass, final String resourceName, final String extension) throws IOException {
+ return generateTempFileFromResource(graphClass, resourceClass, resourceName, extension, true);
+ }
+
+ /**
+ * Copies a file stored as part of a resource to the file system in the path returned from
+ * {@link TestHelper#makeTestDataPath} in a subdirectory called {@code temp/resources}.
+ */
+ public static File generateTempFileFromResource(final Class graphClass, final Class resourceClass, final String resourceName, final String extension, final boolean overwrite) throws IOException {
final File temp = makeTestDataPath(graphClass, "resources");
if (!temp.exists()) temp.mkdirs();
final File tempFile = new File(temp, resourceName + extension);
- final FileOutputStream outputStream = new FileOutputStream(tempFile);
- int data;
- final InputStream inputStream = resourceClass.getResourceAsStream(resourceName);
- while ((data = inputStream.read()) != -1) {
- outputStream.write(data);
+ if (!tempFile.exists() || overwrite) {
+ try (final FileOutputStream outputStream = new FileOutputStream(tempFile)) {
+ int data;
+ try (final InputStream inputStream = resourceClass.getResourceAsStream(resourceName)) {
+ while ((data = inputStream.read()) != -1) {
+ outputStream.write(data);
+ }
+ }
+ }
}
- outputStream.close();
- inputStream.close();
return tempFile;
}