You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2016/05/26 17:50:27 UTC
[01/27] incubator-tinkerpop git commit: Fixes a problem where
cardinality is better respected with subgraph step. [Forced Update!]
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1308 f3a555fd6 -> 6d3f18276 (forced update)
Fixes a problem where cardinality is better respected with subgraph step.
SubgraphStep now consults the parent graph features to determine cardinality of a property.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/636b2e5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/636b2e5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/636b2e5f
Branch: refs/heads/TINKERPOP-1308
Commit: 636b2e5fc22ae2b92a96c5bda8c5979a756779d5
Parents: 0a34d2d
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri May 13 13:36:01 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon May 16 09:04:50 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../traversal/step/sideEffect/SubgraphStep.java | 19 +++++-----
.../step/sideEffect/GroovySubgraphTest.groovy | 5 +++
.../traversal/step/sideEffect/SubgraphTest.java | 38 ++++++++++++++++++--
4 files changed, 52 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/636b2e5f/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 1e33187..a0d5c70 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -29,6 +29,7 @@ TinkerPop 3.1.3 (NOT OFFICIALLY RELEASED YET)
* Fixed a bug in `BulkSet.equals()` which made itself apparent when using `store()` and `aggregate()` with labeled `cap()`.
* Ensured that all asserts of vertex and edge counts were being applied properly in the test suite.
* Fixed bug in `gremlin-driver` where certain channel-level errors would not allow the driver to reconnect.
+* `SubgraphStep` now consults the parent graph features to determine cardinality of a property.
* Use of `Ctrl-C` in Gremlin Console now triggers closing of open remotes.
* Bumped SLF4J to 1.7.21 as previous versions suffered from a memory leak.
* Fixed a bug in `Neo4jGraphStepStrategy` where it wasn't defined properly as a `ProviderOptimizationStrategy`.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/636b2e5f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphStep.java
index 7290bdb..c7bcd57 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphStep.java
@@ -43,6 +43,7 @@ public final class SubgraphStep extends SideEffectStep<Edge> implements SideEffe
private Graph subgraph;
private String sideEffectKey;
+ private Graph.Features.VertexFeatures parentGraphFeatures;
private static final Map<String, Object> DEFAULT_CONFIGURATION = new HashMap<String, Object>() {{
put(Graph.GRAPH, "org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph"); // hard coded because TinkerGraph is not part of gremlin-core
@@ -56,12 +57,13 @@ public final class SubgraphStep extends SideEffectStep<Edge> implements SideEffe
@Override
protected void sideEffect(final Traverser.Admin<Edge> traverser) {
+ parentGraphFeatures = ((Graph) traversal.getGraph().get()).features().vertex();
if (null == this.subgraph) {
this.subgraph = traverser.sideEffects(this.sideEffectKey);
if (!this.subgraph.features().vertex().supportsUserSuppliedIds() || !this.subgraph.features().edge().supportsUserSuppliedIds())
throw new IllegalArgumentException("The provided subgraph must support user supplied ids for vertices and edges: " + this.subgraph);
}
- SubgraphStep.addEdgeToSubgraph(this.subgraph, traverser.get());
+ addEdgeToSubgraph(traverser.get());
}
@Override
@@ -93,24 +95,25 @@ public final class SubgraphStep extends SideEffectStep<Edge> implements SideEffe
///
- private static Vertex getOrCreate(final Graph subgraph, final Vertex vertex) {
+ private Vertex getOrCreate(final Vertex vertex) {
final Iterator<Vertex> vertexIterator = subgraph.vertices(vertex.id());
if (vertexIterator.hasNext()) return vertexIterator.next();
final Vertex subgraphVertex = subgraph.addVertex(T.id, vertex.id(), T.label, vertex.label());
vertex.properties().forEachRemaining(vertexProperty -> {
- final VertexProperty<?> subgraphVertexProperty = subgraphVertex.property(vertexProperty.key(), vertexProperty.value(), T.id, vertexProperty.id());
- vertexProperty.properties().forEachRemaining(property -> subgraphVertexProperty.<Object>property(property.key(), property.value()));
+ final VertexProperty.Cardinality cardinality = parentGraphFeatures.getCardinality(vertexProperty.key());
+ final VertexProperty<?> subgraphVertexProperty = subgraphVertex.property(cardinality, vertexProperty.key(), vertexProperty.value(), T.id, vertexProperty.id());
+ vertexProperty.properties().forEachRemaining(property -> subgraphVertexProperty.property(property.key(), property.value()));
});
return subgraphVertex;
}
- private static void addEdgeToSubgraph(final Graph subgraph, final Edge edge) {
+ private void addEdgeToSubgraph(final Edge edge) {
final Iterator<Edge> edgeIterator = subgraph.edges(edge.id());
if (edgeIterator.hasNext()) return;
final Iterator<Vertex> vertexIterator = edge.vertices(Direction.BOTH);
- final Vertex subGraphOutVertex = getOrCreate(subgraph, vertexIterator.next());
- final Vertex subGraphInVertex = getOrCreate(subgraph, vertexIterator.next());
+ final Vertex subGraphOutVertex = getOrCreate(vertexIterator.next());
+ final Vertex subGraphInVertex = getOrCreate(vertexIterator.next());
final Edge subGraphEdge = subGraphOutVertex.addEdge(edge.label(), subGraphInVertex, T.id, edge.id());
- edge.properties().forEachRemaining(property -> subGraphEdge.<Object>property(property.key(), property.value()));
+ edge.properties().forEachRemaining(property -> subGraphEdge.property(property.key(), property.value()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/636b2e5f/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
index 1918b00..c3ae74a 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
@@ -41,5 +41,10 @@ public abstract class GroovySubgraphTest {
final Graph subgraph) {
TraversalScriptHelper.compute("g.withSideEffect('sg') { subgraph }.V.repeat(__.bothE('created').subgraph('sg').outV).times(5).name.dedup", g, "subgraph", subgraph)
}
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_withSideEffectXsgX_V_hasXname_danielX_outE_subgraphXsgX_inV(final Graph subgraph) {
+ TraversalScriptHelper.compute("g.withSideEffect('sg') { subgraph }.V.has('name','daniel').outE.subgraph('sg').inV", g, "subgraph", subgraph);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/636b2e5f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
index ad4b0cc..dc55685 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
@@ -30,16 +30,19 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import java.util.Arrays;
+import java.util.List;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
+import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.CREW;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.bothE;
import static org.apache.tinkerpop.gremlin.structure.Graph.Features.EdgeFeatures.FEATURE_ADD_EDGES;
import static org.apache.tinkerpop.gremlin.structure.Graph.Features.ElementFeatures.FEATURE_USER_SUPPLIED_IDS;
import static org.apache.tinkerpop.gremlin.structure.Graph.Features.VertexFeatures.FEATURE_ADD_VERTICES;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-
/**
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
@@ -49,6 +52,8 @@ public abstract class SubgraphTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, String> get_g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup(final Graph subgraph);
+ public abstract Traversal<Vertex, Vertex> get_g_withSideEffectXsgX_V_hasXname_danielX_outE_subgraphXsgX_inV(final Graph subgraph);
+
@Test
@LoadGraphWith(MODERN)
@FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = FEATURE_ADD_VERTICES)
@@ -90,17 +95,39 @@ public abstract class SubgraphTest extends AbstractGremlinProcessTest {
public void g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup() throws Exception {
final Configuration config = graphProvider.newGraphConfiguration("subgraph", this.getClass(), name.getMethodName(), MODERN);
graphProvider.clear(config);
- final Graph subgraph = graphProvider.openTestGraph(config);
+ Graph subgraph = graphProvider.openTestGraph(config);
/////
final Traversal<Vertex, String> traversal = get_g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup(subgraph);
printTraversalForm(traversal);
checkResults(Arrays.asList("marko", "josh", "peter"), traversal);
- final Graph subGraph = traversal.asAdmin().getSideEffects().<Graph>get("sg").get();
+ subgraph = traversal.asAdmin().getSideEffects().<Graph>get("sg").get();
assertVertexEdgeCounts(subgraph, 5, 4);
graphProvider.clear(subgraph, config);
}
+ @Test
+ @LoadGraphWith(CREW)
+ @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = FEATURE_ADD_VERTICES)
+ @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = FEATURE_ADD_EDGES)
+ @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = FEATURE_USER_SUPPLIED_IDS)
+ @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = FEATURE_USER_SUPPLIED_IDS)
+ public void g_withSideEffectXsgX_V_hasXname_danielXout_capXsgX() throws Exception {
+ final Configuration config = graphProvider.newGraphConfiguration("subgraph", this.getClass(), name.getMethodName(), CREW);
+ graphProvider.clear(config);
+ final Graph subgraph = graphProvider.openTestGraph(config);
+ /////
+ final Traversal<Vertex, Vertex> traversal = get_g_withSideEffectXsgX_V_hasXname_danielX_outE_subgraphXsgX_inV(subgraph);
+ printTraversalForm(traversal);
+ traversal.iterate();
+ assertVertexEdgeCounts(subgraph, 3, 2);
+
+ final List<String> locations = subgraph.traversal().V().has("name", "daniel").<String>values("location").toList();
+ assertThat(locations, contains("spremberg", "kaiserslautern", "aachen"));
+
+ graphProvider.clear(subgraph, config);
+ }
+
public static class Traversals extends SubgraphTest {
@Override
@@ -112,5 +139,10 @@ public abstract class SubgraphTest extends AbstractGremlinProcessTest {
public Traversal<Vertex, String> get_g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup(final Graph subgraph) {
return g.withSideEffect("sg", () -> subgraph).V().repeat(bothE("created").subgraph("sg").outV()).times(5).<String>values("name").dedup();
}
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_withSideEffectXsgX_V_hasXname_danielX_outE_subgraphXsgX_inV(final Graph subgraph) {
+ return g.withSideEffect("sg", () -> subgraph).V().has("name","daniel").outE().subgraph("sg").inV();
+ }
}
}
[06/27] incubator-tinkerpop git commit: made a PageRank test a bit
more robust. running full integration over night.
Posted by sp...@apache.org.
made a PageRank test a bit more robust. running full integration over night.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6107ff95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6107ff95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6107ff95
Branch: refs/heads/TINKERPOP-1308
Commit: 6107ff95e47129d6266a68410c01c17b04af95cf
Parents: 7d29ef3
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 18 19:24:57 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 18 19:24:57 2016 -0600
----------------------------------------------------------------------
.../process/traversal/step/map/GroovyPageRankTest.groovy | 2 +-
.../gremlin/process/traversal/step/map/PageRankTest.java | 10 ++++++----
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6107ff95/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
index 8608c4d..13787cc 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
@@ -71,7 +71,7 @@ public abstract class GroovyPageRankTest {
}
@Override
- public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX() {
+ public Traversal<Vertex, Map<String, List<Object>>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.out('created').pageRank().by(bothE()).by('projectRank').times(0).valueMap('name','projectRank')")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6107ff95/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
index 1e18b48..57a3b3f 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
@@ -49,7 +49,7 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Vertex> get_g_V_pageRank();
- public abstract Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX();
+ public abstract Traversal<Vertex, Map<String, List<Object>>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX();
public abstract Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name();
@@ -82,10 +82,12 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
public void g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
- final Traversal<Vertex, Map<String, Object>> traversal = get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX();
+ final Traversal<Vertex, Map<String, List<Object>>> traversal = get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX();
printTraversalForm(traversal);
+ final List<Map<String, List<Object>>> result = traversal.toList();
+ assertEquals(4, result.size());
final Map<String, Double> map = new HashMap<>();
- traversal.forEachRemaining(m -> map.put(((List<String>) m.get("name")).get(0), ((List<Double>) m.get("projectRank")).get(0)));
+ result.forEach(m -> map.put((String) m.get("name").get(0), (Double) m.get("projectRank").get(0)));
assertEquals(2, map.size());
assertTrue(map.containsKey("lop"));
assertTrue(map.containsKey("ripple"));
@@ -244,7 +246,7 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
}
@Override
- public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX() {
+ public Traversal<Vertex, Map<String, List<Object>>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX() {
return g.V().out("created").pageRank().by(__.bothE()).by("projectRank").times(0).valueMap("name", "projectRank");
}
[23/27] incubator-tinkerpop git commit: Renamed gryo method for
better consistency with graphson.
Posted by sp...@apache.org.
Renamed gryo method for better consistency with graphson.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ace64ded
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ace64ded
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ace64ded
Branch: refs/heads/TINKERPOP-1308
Commit: ace64ded32539ae7377068bfb6c481a46f7e7200
Parents: f811a8a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri May 20 13:09:02 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu May 26 13:06:07 2016 -0400
----------------------------------------------------------------------
.../gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java | 6 +++---
.../gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ace64ded/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
index 4190d96..9198ae1 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
@@ -80,8 +80,8 @@ public abstract class AbstractGryoMessageSerializerV1d0 extends AbstractMessageS
* Called from the {@link #configure(Map, Map)} method right before the call to create the builder. Sub-classes
* can choose to alter the builder or completely replace it.
*/
- public GryoMapper.Builder beforeCreateMapper(final GryoMapper.Builder builder, final Map<String, Object> config,
- final Map<String, Graph> graphs) {
+ public GryoMapper.Builder configureBuilder(final GryoMapper.Builder builder, final Map<String, Object> config,
+ final Map<String, Graph> graphs) {
return builder;
}
@@ -115,7 +115,7 @@ public abstract class AbstractGryoMessageSerializerV1d0 extends AbstractMessageS
this.serializeToString = Boolean.parseBoolean(config.getOrDefault(TOKEN_SERIALIZE_RESULT_TO_STRING, "false").toString());
this.bufferSize = Integer.parseInt(config.getOrDefault(TOKEN_BUFFER_SIZE, "4096").toString());
- this.gryoMapper = beforeCreateMapper(builder, config, graphs).create();
+ this.gryoMapper = configureBuilder(builder, config, graphs).create();
}
private void addClassResolverSupplier(final Map<String, Object> config, final GryoMapper.Builder builder) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ace64ded/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
index a4ff38b..dab7c58 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
@@ -63,8 +63,8 @@ public class GryoLiteMessageSerializerV1d0 extends AbstractGryoMessageSerializer
}
@Override
- public GryoMapper.Builder beforeCreateMapper(final GryoMapper.Builder builder, final Map<String, Object> config,
- final Map<String, Graph> graphs) {
+ public GryoMapper.Builder configureBuilder(final GryoMapper.Builder builder, final Map<String, Object> config,
+ final Map<String, Graph> graphs) {
return overrideWithLite(builder);
}
[02/27] incubator-tinkerpop git commit: fixed HALTED_TRAVERSER bug
where traversers were not at their 'resting location'. This only shows up in
multi-OLAP job chains. HALTED_TRAVERESERS (master travesal) are now
propagated via Configuration and NOT via s
Posted by sp...@apache.org.
fixed HALTED_TRAVERSER bug where traversers were not at their 'resting location'. This only shows up in multi-OLAP job chains. HALTED_TRAVERESERS (master travesal) are now propagated via Configuration and NOT via sideEffects. This is more elegant and there are now helper methods in TraversalVertexProgram to make getting master halted traversers easy. Updated the-travesal.asciidoc to reflect how to use ProgramStep with it. Had to change a method signature in VertexComputing to account for this... its slight (breaking), but no one is using this right now, I'm sure of it. And if they are, the chaneg is trivial, just add a new argument to a method signature and ignore it to make it be like how it is in 3.2.0. ImmutablePath.TailPath was not serializable and this caused issues for HALTED_TRAVERSERS as well. Everything else fixed up.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4ee30289
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4ee30289
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4ee30289
Branch: refs/heads/TINKERPOP-1308
Commit: 4ee302899da37a0ad0ebab6bc4954ed39b69d6d9
Parents: c61095f
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 18 16:15:40 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 18 16:15:40 2016 -0600
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 23 +++++++--
.../process/computer/GiraphGraphComputer.java | 6 +++
.../traversal/MemoryTraversalSideEffects.java | 1 -
.../traversal/TraversalVertexProgram.java | 50 +++++++++++++++++---
.../computer/traversal/TraverserExecutor.java | 21 +++++---
.../traversal/step/VertexComputing.java | 8 +++-
.../step/map/PageRankVertexProgramStep.java | 7 +--
.../step/map/PeerPressureVertexProgramStep.java | 11 ++++-
.../step/map/ProgramVertexProgramStep.java | 10 +++-
.../step/map/TraversalVertexProgramStep.java | 13 +++--
.../traversal/step/map/VertexProgramStep.java | 21 +++-----
.../optimization/GraphFilterStrategy.java | 3 +-
.../traversal/step/util/ImmutablePath.java | 3 +-
.../step/map/GroovyPageRankTest.groovy | 5 ++
.../traversal/step/map/PageRankTest.java | 22 +++++++++
.../process/traversal/step/map/ProgramTest.java | 38 ++++++++-------
.../process/computer/SparkGraphComputer.java | 7 ++-
.../optimization/SparkInterceptorStrategy.java | 5 +-
.../SparkSingleIterationStrategy.java | 4 +-
19 files changed, 185 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 71708e2..3394e90 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1436,6 +1436,8 @@ are provided below.
[source,java]
----
+private TraverserSet<Object> haltedTraversers;
+
public void loadState(final Graph graph, final Configuration configuration) {
VertexProgram.super.loadState(graph, configuration);
this.traversal = PureTraversal.loadState(configuration, VertexProgramStep.ROOT_TRAVERSAL, graph);
@@ -1444,18 +1446,29 @@ public void loadState(final Graph graph, final Configuration configuration) {
this.memoryComputeKeys.addAll(MemoryTraversalSideEffects.getMemoryComputeKeys(this.traversal.get()));
// if master-traversal traversers may be propagated, create a memory compute key
this.memoryComputeKeys.add(MemoryComputeKey.of(TraversalVertexProgram.HALTED_TRAVERSERS, Operator.addAll, false, false));
+ // returns an empty traverser set if there are no halted traversers
+ this.haltedTraversers = TraversalVertexProgram.getHaltedTraversers(configuration);
+}
+
+public void storeState(final Configuration configuration) {
+ VertexProgram.super.storeState(configuration);
+ // if halted traversers is null or empty, it does nothing
+ TraversalVertexProgram.storeHaltedTraversers(configuration, this.haltedTraversers);
}
public void setup(final Memory memory) {
- if(sideEffects.exists(TraversalVertexProgram.HALTED_TRAVERSERS)) {
- // haltedTraversers in setup() represent master-traversal traversers
- // for example, from a traversal of the form g.V().groupCount().program(...)
- TraverserSet<Object> haltedTraversers = sideEffects.get(TraversalVertexProgram.HALTED_TRAVERSERS);
- // do as you please with this information
+ if(null != this.haltedTraversers) {
+ // do what you like with the halted master traversal traversers
}
+ // once used, no need to keep that information around (master)
+ if(null != this.haltedTraversers)
+ this.haltedTraversers.clear()
}
public void execute(final Vertex vertex, final Messenger messenger, final Memory memory) {
+ // once used, no need to keep that information around (workers)
+ if(null != this.haltedTraversers)
+ this.haltedTraversers.clear()
if(vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent()) {
// haltedTraversers in execute() represent worker-traversal traversers
// for example, from a traversal of the form g.V().out().program(...)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index 6bd2a04..012b9fc 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -58,9 +58,12 @@ import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
import org.apache.tinkerpop.gremlin.util.Gremlin;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.File;
import java.io.NotSerializableException;
+import java.util.HashSet;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
@@ -74,6 +77,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
protected GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
private MapMemory memory = new MapMemory();
private boolean useWorkerThreadsInConfiguration;
+ private Set<String> vertexProgramConfigurationKeys = new HashSet<>();
public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
super(hadoopGraph);
@@ -112,6 +116,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
final BaseConfiguration apacheConfiguration = new BaseConfiguration();
apacheConfiguration.setDelimiterParsingDisabled(true);
vertexProgram.storeState(apacheConfiguration);
+ IteratorUtils.fill(apacheConfiguration.getKeys(), this.vertexProgramConfigurationKeys);
ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, this.giraphConfiguration);
this.vertexProgram.getMessageCombiner().ifPresent(combiner -> this.giraphConfiguration.setMessageCombinerClass(GiraphMessageCombiner.class));
return this;
@@ -139,6 +144,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
// clear properties that should not be propagated in an OLAP chain
apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_GRAPH_FILTER);
apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR);
+ this.vertexProgramConfigurationKeys.forEach(apacheConfiguration::clearProperty); // clear out vertex program specific configurations
return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), this.memory.asImmutable());
}, exec);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
index 1327a7f..23d33f1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
@@ -192,7 +192,6 @@ public final class MemoryTraversalSideEffects implements TraversalSideEffects {
traversal.getSideEffects();
sideEffects.keys().
stream().
- filter(key -> !key.equals(TraversalVertexProgram.HALTED_TRAVERSERS)).
forEach(key -> keys.add(MemoryComputeKey.of(key, sideEffects.getReducer(key), true, false)));
return keys;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index 4df5189..211a5e5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -33,6 +33,7 @@ import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.Computer
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
+import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramHelper;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
@@ -66,6 +67,7 @@ import org.apache.tinkerpop.gremlin.util.iterator.EmptyIterator;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
@@ -102,6 +104,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
private PureTraversal<?, ?> traversal;
private TraversalMatrix<?, ?> traversalMatrix;
private final Set<MapReduce> mapReducers = new HashSet<>();
+ private TraverserSet<Object> haltedTraversers;
private boolean returnHaltedTraversers = false;
private TraversalVertexProgram() {
@@ -116,6 +119,32 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
return this.traversal;
}
+ public static <R> TraverserSet<R> loadHaltedTraversers(final Configuration configuration) {
+ if (!configuration.containsKey(HALTED_TRAVERSERS))
+ return new TraverserSet<>();
+
+ final Object object = configuration.getProperty(HALTED_TRAVERSERS) instanceof String ?
+ VertexProgramHelper.deserialize(configuration, HALTED_TRAVERSERS) :
+ configuration.getProperty(HALTED_TRAVERSERS);
+ if (object instanceof Traverser.Admin)
+ return new TraverserSet<>((Traverser.Admin<R>) object);
+ else {
+ final TraverserSet<R> traverserSet = new TraverserSet<>();
+ traverserSet.addAll((Collection) object);
+ return traverserSet;
+ }
+ }
+
+ public static <R> void storeHaltedTraversers(final Configuration configuration, final TraverserSet<R> haltedTraversers) {
+ if (null != haltedTraversers && !haltedTraversers.isEmpty()) {
+ try {
+ VertexProgramHelper.serialize(haltedTraversers, configuration, HALTED_TRAVERSERS);
+ } catch (final Exception e) {
+ configuration.setProperty(HALTED_TRAVERSERS, haltedTraversers);
+ }
+ }
+ }
+
@Override
public void loadState(final Graph graph, final Configuration configuration) {
if (!configuration.containsKey(TRAVERSAL))
@@ -125,6 +154,8 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
this.traversal.get().applyStrategies();
/// traversal is compiled and ready to be introspected
this.traversalMatrix = new TraversalMatrix<>(this.traversal.get());
+ // get any master-traversal halted traversers
+ this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
// if results will be serialized out, don't save halted traversers across the cluster
this.returnHaltedTraversers =
(this.traversal.get().getParent().asStep().getNextStep() instanceof ComputerResultStep || // if its just going to stream it out, don't distribute
@@ -158,6 +189,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
public void storeState(final Configuration configuration) {
VertexProgram.super.storeState(configuration);
this.traversal.storeState(configuration, TRAVERSAL);
+ TraversalVertexProgram.storeHaltedTraversers(configuration, this.haltedTraversers);
}
@Override
@@ -170,19 +202,17 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
memory.set(MUTATED_MEMORY_KEYS, new HashSet<>());
memory.set(COMPLETED_BARRIERS, new HashSet<>());
// if halted traversers are being sent from a previous VertexProgram in an OLAP chain (non-distributed traversers), get them into the stream
- if (sideEffects.exists(TraversalVertexProgram.HALTED_TRAVERSERS)) {
- final TraverserSet<Object> haltedTraversers = sideEffects.get(TraversalVertexProgram.HALTED_TRAVERSERS);
+ if (null != this.haltedTraversers) {
final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
- IteratorUtils.removeOnNext(haltedTraversers.iterator()).forEachRemaining(traverser -> {
+ IteratorUtils.removeOnNext(this.haltedTraversers.iterator()).forEachRemaining(traverser -> {
traverser.setStepId(this.traversal.get().getStartStep().getId());
toProcessTraversers.add(traverser);
});
assert haltedTraversers.isEmpty(); // TODO: this should be empty
final TraverserSet<Object> remoteActiveTraversers = new TraverserSet<>();
- MasterExecutor.processTraversers(this.traversal, this.traversalMatrix, toProcessTraversers, remoteActiveTraversers, haltedTraversers);
- memory.set(HALTED_TRAVERSERS, haltedTraversers);
+ MasterExecutor.processTraversers(this.traversal, this.traversalMatrix, toProcessTraversers, remoteActiveTraversers, this.haltedTraversers);
+ memory.set(HALTED_TRAVERSERS, this.haltedTraversers);
memory.set(ACTIVE_TRAVERSERS, remoteActiveTraversers);
- sideEffects.remove(TraversalVertexProgram.HALTED_TRAVERSERS);
} else {
memory.set(HALTED_TRAVERSERS, new TraverserSet<>());
memory.set(ACTIVE_TRAVERSERS, new TraverserSet<>());
@@ -196,6 +226,9 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
@Override
public void execute(final Vertex vertex, final Messenger<TraverserSet<Object>> messenger, final Memory memory) {
+ // if any global halted traversers, simply don't use them as they were handled by master setup()
+ if (null != this.haltedTraversers)
+ this.haltedTraversers = null;
// memory is distributed
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.EXECUTE);
// if a barrier was completed in another worker, it is also completed here (ensure distributed barriers are synchronized)
@@ -365,6 +398,11 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
super(TraversalVertexProgram.class);
}
+ public Builder haltedTraversers(final TraverserSet<Object> haltedTraversers) {
+ TraversalVertexProgram.storeHaltedTraversers(this.configuration, haltedTraversers);
+ return this;
+ }
+
public Builder traversal(final TraversalSource traversalSource, final String scriptEngine, final String traversalScript, final Object... bindings) {
return this.traversal(new ScriptTraversal<>(traversalSource, scriptEngine, traversalScript, bindings));
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
index 167924f..f5facdb 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
@@ -87,9 +87,16 @@ public final class TraverserExecutor {
while (traversers.hasNext()) {
final Traverser.Admin<Object> traverser = traversers.next();
traversers.remove();
- traverser.attach(Attachable.Method.get(vertex));
- traverser.setSideEffects(traversalSideEffects);
- toProcessTraversers.add(traverser);
+ if (traverser.isHalted()) {
+ if (returnHaltedTraversers)
+ memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
+ else
+ haltedTraversers.add(traverser);
+ } else {
+ traverser.attach(Attachable.Method.get(vertex));
+ traverser.setSideEffects(traversalSideEffects);
+ toProcessTraversers.add(traverser);
+ }
}
}
@@ -152,10 +159,10 @@ public final class TraverserExecutor {
final TraverserSet<Object> barrierSet = barrier.nextBarrier();
IteratorUtils.removeOnNext(barrierSet.iterator()).forEachRemaining(traverser -> {
traverser.addLabels(step.getLabels()); // this might need to be generalized for working with global barriers too
- if (traverser.isHalted()) {
+ if (traverser.isHalted() && ((!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) || getHostingVertex(traverser.get()).equals(vertex))) {
traverser.detach();
if (returnHaltedTraversers)
- memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser.split()));
+ memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
else
haltedTraversers.add(traverser);
} else {
@@ -174,10 +181,10 @@ public final class TraverserExecutor {
}
} else { // LOCAL PROCESSING
step.forEachRemaining(traverser -> {
- if (traverser.isHalted()) {
+ if (traverser.isHalted() && ((!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) || getHostingVertex(traverser.get()).equals(vertex))) {
traverser.detach();
if (returnHaltedTraversers)
- memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser.split()));
+ memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
else
haltedTraversers.add(traverser);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
index 57e1a3e..b599c42 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
@@ -21,9 +21,12 @@ package org.apache.tinkerpop.gremlin.process.computer.traversal.step;
import org.apache.tinkerpop.gremlin.process.computer.Computer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import java.util.Optional;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -48,10 +51,11 @@ public interface VertexComputing {
/**
* Generate the {@link VertexProgram}.
*
- * @param graph the {@link Graph} that the program will be executed over.
+ * @param graph the {@link Graph} that the program will be executed over.
+ * @param memoryOptional a reference to a {@link Memory} from the previous OLAP job if it exists.
* @return the generated vertex program instance.
*/
- public VertexProgram generateProgram(final Graph graph);
+ public VertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional);
/**
* @deprecated As of release 3.2.1. Please use {@link VertexComputing#getComputer()}.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
index 5d10e67..660f5b8 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
@@ -19,8 +19,9 @@
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.lambda.HaltedTraversersCountTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
@@ -37,8 +38,8 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
-import java.util.function.Function;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -83,7 +84,7 @@ public final class PageRankVertexProgramStep extends VertexProgramStep implement
}
@Override
- public PageRankVertexProgram generateProgram(final Graph graph) {
+ public PageRankVertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
final PageRankVertexProgram.Builder builder = PageRankVertexProgram.build()
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
index f42ce93..2ae36a5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
@@ -19,7 +19,7 @@
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
@@ -36,6 +36,7 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
/**
@@ -84,9 +85,15 @@ public final class PeerPressureVertexProgramStep extends VertexProgramStep imple
}
@Override
- public PeerPressureVertexProgram generateProgram(final Graph graph) {
+ public PeerPressureVertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
+ /*
+ memoryOptional.ifPresent(memory -> {
+ if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
+ builder.configure(TraversalVertexProgram.HALTED_TRAVERSERS, memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
+ });
+ */
return PeerPressureVertexProgram.build()
.property(this.clusterProperty)
.maxIterations(this.times)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
index a1342b6..d060c99 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
@@ -20,7 +20,9 @@
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
import org.apache.commons.configuration.MapConfiguration;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
@@ -29,6 +31,7 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -48,11 +51,16 @@ public final class ProgramVertexProgramStep extends VertexProgramStep {
}
@Override
- public VertexProgram generateProgram(final Graph graph) {
+ public VertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
final MapConfiguration base = new MapConfiguration(this.configuration);
base.setDelimiterParsingDisabled(true);
PureTraversal.storeState(base, ROOT_TRAVERSAL, TraversalHelper.getRootTraversal(this.getTraversal()).clone());
base.setProperty(STEP_ID, this.getId());
+ memoryOptional.ifPresent(memory -> {
+ if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS)) {
+ TraversalVertexProgram.storeHaltedTraversers(base, memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
+ }
+ });
return VertexProgram.createVertexProgram(graph, base);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
index 64b9097..5851f35 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
@@ -20,6 +20,7 @@
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
@@ -32,6 +33,7 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
/**
@@ -62,15 +64,18 @@ public final class TraversalVertexProgramStep extends VertexProgramStep implemen
}
@Override
- public TraversalVertexProgram generateProgram(final Graph graph) {
+ public TraversalVertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
final Traversal.Admin<?, ?> computerSpecificTraversal = this.computerTraversal.getPure();
computerSpecificTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()).clone());
this.getTraversal().getStrategies().toList().forEach(computerSpecificTraversal.getStrategies()::addStrategies);
computerSpecificTraversal.setSideEffects(new MemoryTraversalSideEffects(this.getTraversal().getSideEffects()));
computerSpecificTraversal.setParent(this);
- return TraversalVertexProgram.build()
- .traversal(computerSpecificTraversal)
- .create(graph);
+ final TraversalVertexProgram.Builder builder = TraversalVertexProgram.build().traversal(computerSpecificTraversal);
+ memoryOptional.ifPresent(memory -> {
+ if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
+ builder.haltedTraversers(memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
+ });
+ return builder.create(graph);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
index c535d30..ac3de30 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
@@ -25,7 +25,6 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.VertexComputing;
-import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
@@ -33,12 +32,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedException;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.util.function.ConstantSupplier;
import java.util.NoSuchElementException;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -66,14 +64,15 @@ public abstract class VertexProgramStep extends AbstractStep<ComputerResult, Com
if (this.first && this.getPreviousStep() instanceof EmptyStep) {
this.first = false;
final Graph graph = this.getTraversal().getGraph().get();
- future = this.generateComputer(graph).program(this.generateProgram(graph)).submit();
+ future = this.generateComputer(graph).program(this.generateProgram(graph, Optional.empty())).submit();
final ComputerResult result = future.get();
this.processMemorySideEffects(result.memory());
return this.getTraversal().getTraverserGenerator().generate(result, this, 1l);
} else {
final Traverser.Admin<ComputerResult> traverser = this.starts.next();
final Graph graph = traverser.get().graph();
- future = this.generateComputer(graph).program(this.generateProgram(graph)).submit();
+ final Memory memory = traverser.get().memory();
+ future = this.generateComputer(graph).program(this.generateProgram(graph, Optional.of(memory))).submit();
final ComputerResult result = future.get();
this.processMemorySideEffects(result.memory());
return traverser.split(result, this);
@@ -122,21 +121,13 @@ public abstract class VertexProgramStep extends AbstractStep<ComputerResult, Com
final TraversalSideEffects sideEffects = this.getTraversal().getSideEffects();
for (final String key : memory.keys()) {
if (sideEffects.exists(key)) {
+ // halted traversers should never be propagated through sideEffects
+ assert !key.equals(TraversalVertexProgram.HALTED_TRAVERSERS);
sideEffects.set(key, memory.get(key));
}
}
- if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS) && !this.isEndStep()) {
- final TraverserSet<Object> haltedTraversers = memory.get(TraversalVertexProgram.HALTED_TRAVERSERS);
- if (!haltedTraversers.isEmpty()) {
- if (sideEffects.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
- sideEffects.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
- else
- sideEffects.register(TraversalVertexProgram.HALTED_TRAVERSERS, new ConstantSupplier<>(haltedTraversers), Operator.addAll);
- }
- }
}
-
protected boolean isEndStep() {
return this.getNextStep() instanceof ComputerResultStep || (this.getNextStep() instanceof ProfileStep && this.getNextStep().getNextStep() instanceof ComputerResultStep);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
index 90819d7..e4cf5ff 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
@@ -41,6 +41,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
/**
@@ -59,7 +60,7 @@ public final class GraphFilterStrategy extends AbstractTraversalStrategy<Travers
return;
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // given that this strategy only works for single OLAP jobs, the graph is the traversal graph
for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) { // will be zero or one step
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, Optional.empty()).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
final Computer computer = step.getComputer();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ImmutablePath.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ImmutablePath.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ImmutablePath.java
index 1a916ca..1e936c8 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ImmutablePath.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ImmutablePath.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.process.traversal.step.util;
import org.apache.tinkerpop.gremlin.process.traversal.Path;
import org.apache.tinkerpop.gremlin.process.traversal.Pop;
-import org.apache.tinkerpop.gremlin.structure.Element;
import java.io.Serializable;
import java.util.ArrayList;
@@ -177,7 +176,7 @@ public class ImmutablePath implements Path, ImmutablePathImpl, Serializable, Clo
}
- private static class TailPath implements Path, ImmutablePathImpl {
+ private static class TailPath implements Path, ImmutablePathImpl, Serializable {
private static final TailPath INSTANCE = new TailPath();
private TailPath() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
index bda6819..e27e8c0 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
@@ -69,5 +69,10 @@ public abstract class GroovyPageRankTest {
public Traversal<Vertex, Map<Object, List<Vertex>>> get_g_V_outXcreatedX_groupXmX_byXlabelX_pageRankX1X_byXpageRankX_byXinEX_timesX1X_inXcreatedX_groupXmX_byXpageRankX_capXmX() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.out('created').group('m').by(label).pageRank(1.0).by('pageRank').by(inE()).times(1).in('created').group('m').by('pageRank').cap('m')")
}
+
+ @Override
+ public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.out('created').pageRank().by(bothE()).by('projectRank').valueMap('name','projectRank')")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
index df4b802..5a3ed9b 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
@@ -31,6 +31,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
import org.junit.runner.RunWith;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -48,6 +49,8 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Vertex> get_g_V_pageRank();
+ public abstract Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX();
+
public abstract Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name();
public abstract Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name_limitX2X();
@@ -78,6 +81,20 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
+ public void g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
+ final Traversal<Vertex, Map<String, Object>> traversal = get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX();
+ printTraversalForm(traversal);
+ final Map<String, Double> map = new HashMap<>();
+ traversal.forEachRemaining(m -> map.put(((List<String>) m.get("name")).get(0), ((List<Double>) m.get("projectRank")).get(0)));
+ assertEquals(2, map.size());
+ assertTrue(map.containsKey("lop"));
+ assertTrue(map.containsKey("ripple"));
+ assertTrue(map.get("lop") > map.get("ripple"));
+ assertFalse(traversal.hasNext());
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
public void g_V_pageRank_order_byXpageRank_decrX_name() {
final Traversal<Vertex, String> traversal = get_g_V_pageRank_order_byXpageRank_decrX_name();
printTraversalForm(traversal);
@@ -225,6 +242,11 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
}
@Override
+ public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
+ return g.V().out("created").pageRank().by(__.bothE()).by("projectRank").valueMap("name", "projectRank");
+ }
+
+ @Override
public Traversal<Vertex, Map<String, List<Object>>> get_g_V_pageRank_byXoutEXknowsXX_byXfriendRankX_valueMapXname_friendRankX() {
return g.V().pageRank().by(__.outE("knows")).by("friendRank").valueMap("name", "friendRank");
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
index 2c505e7..8b2a293 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
@@ -59,6 +59,7 @@ import java.util.Set;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
@@ -160,6 +161,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
public static class TestProgram implements VertexProgram {
private PureTraversal<?, ?> traversal = new PureTraversal<>(EmptyTraversal.instance());
+ private TraverserSet<Object> haltedTraversers;
private Step programStep = EmptyStep.instance();
private final Set<MemoryComputeKey> memoryComputeKeys = new HashSet<>();
@@ -168,6 +170,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
public void loadState(final Graph graph, final Configuration configuration) {
VertexProgram.super.loadState(graph, configuration);
this.traversal = PureTraversal.loadState(configuration, VertexProgramStep.ROOT_TRAVERSAL, graph);
+ this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
this.programStep = new TraversalMatrix<>(this.traversal.get()).getStepById(configuration.getString(ProgramVertexProgramStep.STEP_ID));
this.memoryComputeKeys.addAll(MemoryTraversalSideEffects.getMemoryComputeKeys(this.traversal.get()));
this.memoryComputeKeys.add(MemoryComputeKey.of(TraversalVertexProgram.HALTED_TRAVERSERS, Operator.addAll, false, false));
@@ -178,24 +181,24 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
public void storeState(final Configuration configuration) {
VertexProgram.super.storeState(configuration);
this.traversal.storeState(configuration, VertexProgramStep.ROOT_TRAVERSAL);
+ TraversalVertexProgram.storeHaltedTraversers(configuration, this.haltedTraversers);
configuration.setProperty(ProgramVertexProgramStep.STEP_ID, this.programStep.getId());
}
@Override
public void setup(final Memory memory) {
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.SETUP);
- final TraversalSideEffects sideEffects = this.traversal.get().getSideEffects();
- final TraverserSet<Object> haltedTraversers = sideEffects.get(TraversalVertexProgram.HALTED_TRAVERSERS);
- sideEffects.remove(TraversalVertexProgram.HALTED_TRAVERSERS);
- this.checkSideEffects(MemoryTraversalSideEffects.State.SETUP);
- final Map<Vertex, Long> map = (Map<Vertex, Long>) haltedTraversers.iterator().next().get();
+ final Map<Vertex, Long> map = (Map<Vertex, Long>) this.haltedTraversers.iterator().next().get();
assertEquals(2, map.size());
assertTrue(map.values().contains(3l));
assertTrue(map.values().contains(1l));
final TraverserSet<Object> activeTraversers = new TraverserSet<>();
- map.keySet().forEach(vertex -> activeTraversers.add(haltedTraversers.peek().split(vertex, EmptyStep.instance())));
+ map.keySet().forEach(vertex -> activeTraversers.add(this.haltedTraversers.peek().split(vertex, EmptyStep.instance())));
+ this.haltedTraversers.clear();
+ this.checkSideEffects();
memory.set(TraversalVertexProgram.ACTIVE_TRAVERSERS, activeTraversers);
+
}
@Override
@@ -203,7 +206,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
final TraverserGenerator generator = this.traversal.get().getTraverserGenerator();
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.EXECUTE);
- this.checkSideEffects(MemoryTraversalSideEffects.State.EXECUTE);
+ this.checkSideEffects();
final TraverserSet<Vertex> activeTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
if (vertex.label().equals("software")) {
assertEquals(1, activeTraversers.stream().filter(v -> v.get().equals(vertex)).count());
@@ -231,7 +234,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
public boolean terminate(final Memory memory) {
final TraverserGenerator generator = this.traversal.get().getTraverserGenerator();
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.TERMINATE);
- checkSideEffects(MemoryTraversalSideEffects.State.TERMINATE);
+ checkSideEffects();
if (memory.isInitialIteration()) {
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
return false;
@@ -248,16 +251,18 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
@Override
public void workerIterationStart(final Memory memory) {
+ assertNotNull(this.haltedTraversers);
+ this.haltedTraversers.clear();
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.WORKER_ITERATION_START);
- checkSideEffects(MemoryTraversalSideEffects.State.WORKER_ITERATION_START);
+ checkSideEffects();
}
@Override
public void workerIterationEnd(final Memory memory) {
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.WORKER_ITERATION_END);
- checkSideEffects(MemoryTraversalSideEffects.State.WORKER_ITERATION_END);
+ checkSideEffects();
}
@Override
@@ -299,16 +304,13 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
////////
- private void checkSideEffects(final MemoryTraversalSideEffects.State state) {
+ private void checkSideEffects() {
+ assertEquals(0, this.haltedTraversers.size());
+ assertTrue(this.haltedTraversers.isEmpty());
final TraversalSideEffects sideEffects = this.traversal.get().getSideEffects();
assertTrue(sideEffects instanceof MemoryTraversalSideEffects);
- if (state.masterState()) {
- assertEquals(1, sideEffects.keys().size());
- assertFalse(sideEffects.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
- } else {
- assertEquals(2, sideEffects.keys().size());
- assertTrue(sideEffects.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
- }
+ assertEquals(1, sideEffects.keys().size());
+ assertFalse(sideEffects.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
assertTrue(sideEffects.exists("x"));
final BulkSet<String> bulkSet = sideEffects.get("x");
assertEquals(4, bulkSet.size());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 9a9e934..9e05e53 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -264,14 +264,13 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
throw new IllegalStateException(e.getMessage());
}
} else { // standard GraphComputer semantics
+ // get a configuration that will be propagated to all workers
+ final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
+ this.vertexProgram.storeState(vertexProgramConfiguration);
// set up the vertex program and wire up configurations
this.vertexProgram.setup(memory);
JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
memory.broadcastMemory(sparkContext);
- final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
- this.vertexProgram.storeState(vertexProgramConfiguration);
- ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
// execute the vertex program
while (true) {
if (Thread.interrupted()) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index 19d21b3..a2cc518 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -20,6 +20,7 @@
package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
@@ -29,6 +30,8 @@ import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.op
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
+import java.util.Optional;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -43,7 +46,7 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
public void apply(final Traversal.Admin<?, ?> traversal) {
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined
for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, Optional.empty()).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
if (SparkStarBarrierInterceptor.isLegal(computerTraversal)) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4ee30289/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
index 153282e..280c7ba 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
@@ -20,6 +20,7 @@
package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
@@ -36,6 +37,7 @@ import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Optional;
import java.util.Set;
/**
@@ -59,7 +61,7 @@ public final class SparkSingleIterationStrategy extends AbstractTraversalStrateg
public void apply(final Traversal.Admin<?, ?> traversal) {
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined
for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph).getTraversal().get().clone();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, Optional.empty()).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
boolean doesMessagePass = TraversalHelper.hasStepOfAssignableClassRecursively(Scope.global, MULTI_ITERATION_CLASSES, computerTraversal);
[18/27] incubator-tinkerpop git commit: This closes #233
Posted by sp...@apache.org.
This closes #233
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/366edb3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/366edb3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/366edb3b
Branch: refs/heads/TINKERPOP-1308
Commit: 366edb3bb395df0545ce0dda1fefd02d714a2865
Parents: 192d5ec
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 24 16:36:35 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 24 16:36:35 2016 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[11/27] incubator-tinkerpop git commit: Named the thread pool for
gremlin server sessions. CTR
Posted by sp...@apache.org.
Named the thread pool for gremlin server sessions. CTR
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/aa447eb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/aa447eb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/aa447eb0
Branch: refs/heads/TINKERPOP-1308
Commit: aa447eb0390625d561e018315d8719c666ad2d31
Parents: 3feb752
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon May 23 08:14:24 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon May 23 08:14:24 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../apache/tinkerpop/gremlin/server/op/session/Session.java | 6 +++++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/aa447eb0/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 1e33187..5bf3ada 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.3 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Named the thread pool used by Gremlin Server sessions: "gremlin-server-session-$n".
* Fixed a bug in `BulkSet.equals()` which made itself apparent when using `store()` and `aggregate()` with labeled `cap()`.
* Ensured that all asserts of vertex and edge counts were being applied properly in the test suite.
* Fixed bug in `gremlin-driver` where certain channel-level errors would not allow the driver to reconnect.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/aa447eb0/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
index 01e7bbd..0ed2041 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.server.Context;
import org.apache.tinkerpop.gremlin.server.GraphManager;
import org.apache.tinkerpop.gremlin.server.Settings;
import org.apache.tinkerpop.gremlin.server.util.LifeCycleHook;
+import org.apache.tinkerpop.gremlin.server.util.ThreadFactoryUtil;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,6 +36,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -59,11 +61,13 @@ public class Session {
*/
private final GremlinExecutor gremlinExecutor;
+ private final ThreadFactory threadFactoryWorker = ThreadFactoryUtil.create("session-%d");
+
/**
* By binding the session to run ScriptEngine evaluations in a specific thread, each request will respect
* the ThreadLocal nature of Graph implementations.
*/
- private final ExecutorService executor = Executors.newSingleThreadExecutor();
+ private final ExecutorService executor = Executors.newSingleThreadExecutor(threadFactoryWorker);
private final ConcurrentHashMap<String, Session> sessions;
[04/27] incubator-tinkerpop git commit: minor tweaks to
TraverserVertexProgram to make it more efficient.
Posted by sp...@apache.org.
minor tweaks to TraverserVertexProgram to make it more efficient.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a6aacdc5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a6aacdc5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a6aacdc5
Branch: refs/heads/TINKERPOP-1308
Commit: a6aacdc51eba895e04f9d75d64343470fbaca5c5
Parents: 07e1299
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 18 17:11:26 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 18 17:11:26 2016 -0600
----------------------------------------------------------------------
.../process/computer/traversal/TraversalVertexProgram.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a6aacdc5/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index 211a5e5..aba44d3 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -196,19 +196,18 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
public void setup(final Memory memory) {
// memory is local
MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.SETUP);
- final MemoryTraversalSideEffects sideEffects = ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects());
- sideEffects.storeSideEffectsInMemory();
+ ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).storeSideEffectsInMemory();
memory.set(VOTE_TO_HALT, true);
memory.set(MUTATED_MEMORY_KEYS, new HashSet<>());
memory.set(COMPLETED_BARRIERS, new HashSet<>());
// if halted traversers are being sent from a previous VertexProgram in an OLAP chain (non-distributed traversers), get them into the stream
- if (null != this.haltedTraversers) {
+ if (null != this.haltedTraversers && !this.haltedTraversers.isEmpty()) {
final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
IteratorUtils.removeOnNext(this.haltedTraversers.iterator()).forEachRemaining(traverser -> {
traverser.setStepId(this.traversal.get().getStartStep().getId());
toProcessTraversers.add(traverser);
});
- assert haltedTraversers.isEmpty(); // TODO: this should be empty
+ assert haltedTraversers.isEmpty();
final TraverserSet<Object> remoteActiveTraversers = new TraverserSet<>();
MasterExecutor.processTraversers(this.traversal, this.traversalMatrix, toProcessTraversers, remoteActiveTraversers, this.haltedTraversers);
memory.set(HALTED_TRAVERSERS, this.haltedTraversers);
@@ -217,6 +216,8 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
memory.set(HALTED_TRAVERSERS, new TraverserSet<>());
memory.set(ACTIVE_TRAVERSERS, new TraverserSet<>());
}
+ // local variable will no longer be used so null it for GC
+ this.haltedTraversers = null;
}
@Override
[22/27] incubator-tinkerpop git commit: Allow calls to addCustom on
GryoMapper to override.
Posted by sp...@apache.org.
Allow calls to addCustom on GryoMapper to override.
By allowing override, users get complete control over the gryo serialization process and open up more options for Gremlin Server to offer other optional serialization views over gryo.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2e030502
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2e030502
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2e030502
Branch: refs/heads/TINKERPOP-1308
Commit: 2e030502804116edc4fb636acf4f5fbe3ae3e2ce
Parents: 6113c92
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri May 20 08:33:59 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu May 26 13:05:16 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../upgrade/release-3.2.x-incubating.asciidoc | 7 +++
.../gremlin/structure/io/gryo/GryoMapper.java | 44 +++++++++++---
.../structure/io/gryo/GryoMapperTest.java | 63 +++++++++++++++-----
4 files changed, 92 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2e030502/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 171b772..d2a8e3e 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.2.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* `GryoMapper` allows overrides of existing serializers on calls to `addCustom` on the builder.
* Fixed a `NullPointerException` bug around nested `group()`-steps in OLAP.
* Fixed a severe bug around halted traversers in a multi-job OLAP traversal chain.
* Ensure a separation of `GraphComputer` and `VertexProgram` configurations in `SparkGraphComputer` and `GiraphGraphComputer`.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2e030502/docs/src/upgrade/release-3.2.x-incubating.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.2.x-incubating.asciidoc b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
index 66e5f8c..dec022f 100644
--- a/docs/src/upgrade/release-3.2.x-incubating.asciidoc
+++ b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
@@ -40,6 +40,13 @@ for Gremlin Console.
See: link:https://issues.apache.org/jira/browse/TINKERPOP-1297[TINKERPOP-1297]
+GryoMapper Construction
+^^^^^^^^^^^^^^^^^^^^^^^
+
+It is now possible to override existing serializers with calls to `addCustom` on the `GryoMapper` builder. This option
+allows complete control over the serializers used by Gryo. Of course, this also makes it possible to produce completely
+non-compliant Gryo files. This feature should be used with caution.
+
TraversalVertexProgram
^^^^^^^^^^^^^^^^^^^^^^
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2e030502/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index d510706..9cae845 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -68,6 +68,7 @@ import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexProperty;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphGryoSerializer;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.apache.tinkerpop.shaded.kryo.ClassResolver;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.KryoSerializable;
@@ -104,12 +105,14 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeMap;
@@ -396,29 +399,43 @@ public final class GryoMapper implements Mapper<Kryo> {
}
/**
- * Register custom classes to serializes with gryo using default serialization.
+ * Register custom classes to serializes with gryo using default serialization. Note that calling this method
+ * for a class that is already registered will override that registration.
*/
public Builder addCustom(final Class... custom) {
- if (custom != null && custom.length > 0)
- serializationList.addAll(Arrays.asList(custom).stream()
- .map(c -> Triplet.<Class, Function<Kryo, Serializer>, Integer>with(c, null, currentSerializationId.getAndIncrement()))
- .collect(Collectors.<Triplet<Class, Function<Kryo, Serializer>, Integer>>toList()));
+ if (custom != null && custom.length > 0) {
+ for (Class clazz : custom) {
+ addCustom(clazz, (Function<Kryo, Serializer>) null);
+ }
+ }
return this;
}
/**
- * Register custom class to serialize with a custom serialization class.
+ * Register custom class to serialize with a custom serialization class. Note that calling this method for
+ * a class that is already registered will override that registration.
*/
public Builder addCustom(final Class clazz, final Serializer serializer) {
- serializationList.add(Triplet.with(clazz, kryo -> serializer, currentSerializationId.getAndIncrement()));
+ if (null == serializer)
+ addCustom(clazz);
+ else
+ addCustom(clazz, kryo -> serializer);
return this;
}
/**
- * Register a custom class to serialize with a custom serializer as returned from a {@link Function}.
+ * Register a custom class to serialize with a custom serializer as returned from a {@link Function}. Note
+ * that calling this method for a class that is already registered will override that registration.
*/
public Builder addCustom(final Class clazz, final Function<Kryo, Serializer> serializer) {
- serializationList.add(Triplet.with(clazz, serializer, currentSerializationId.getAndIncrement()));
+ final Optional<Triplet<Class, Function<Kryo, Serializer>, Integer>> found = findSerializer(clazz);
+ if (found.isPresent()) {
+ final Triplet<Class, Function<Kryo, Serializer>, Integer> t = found.get();
+ serializationList.remove(t);
+ serializationList.add(t.setAt1(serializer));
+ } else
+ serializationList.add(Triplet.with(clazz, serializer, currentSerializationId.getAndIncrement()));
+
return this;
}
@@ -473,5 +490,14 @@ public final class GryoMapper implements Mapper<Kryo> {
return new GryoMapper(this);
}
+
+ private Optional<Triplet<Class, Function<Kryo, Serializer>, Integer>> findSerializer(final Class clazz) {
+ final Iterator<Triplet<Class, Function<Kryo, Serializer>, Integer>> itty = IteratorUtils.filter(
+ serializationList, t -> t.getValue0().equals(clazz)).iterator();
+ if (itty.hasNext())
+ return Optional.of(itty.next());
+ else
+ return Optional.empty();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2e030502/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapperTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapperTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapperTest.java
index 861c6f6..29ddc57 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapperTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapperTest.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.shaded.kryo.ClassResolver;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Registration;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.junit.Test;
@@ -57,9 +58,12 @@ import java.util.Map;
import java.util.function.Supplier;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.__;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.fail;
/**
* @author Stephen Mallette (http://stephen.genoprime.com)
@@ -220,85 +224,100 @@ public class GryoMapperTest {
}
@Test
- public void shouldHandleDuration()throws Exception {
+ public void shouldOverrideExistingSerializer() throws Exception {
+ final GryoMapper mapper = GryoMapper.build()
+ .addCustom(Duration.class, new OverrideDurationSerializer()).create();
+
+ try (final OutputStream stream = new ByteArrayOutputStream()) {
+ final Output out = new Output(stream);
+ mapper.createMapper().writeObject(out, Duration.ZERO);
+ fail("The OverrideDurationSerializer throws exceptions so this should not have worked");
+ } catch (Exception ex) {
+ assertThat(ex, instanceOf(UnsupportedOperationException.class));
+ assertEquals("I don't do anything", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldHandleDuration() throws Exception {
final Duration o = Duration.ZERO;
assertEquals(o, serializeDeserialize(o, Duration.class));
}
@Test
- public void shouldHandleInstant()throws Exception {
+ public void shouldHandleInstant() throws Exception {
final Instant o = Instant.ofEpochMilli(System.currentTimeMillis());
assertEquals(o, serializeDeserialize(o, Instant.class));
}
@Test
- public void shouldHandleLocalDate()throws Exception {
+ public void shouldHandleLocalDate() throws Exception {
final LocalDate o = LocalDate.now();
assertEquals(o, serializeDeserialize(o, LocalDate.class));
}
@Test
- public void shouldHandleLocalDateTime()throws Exception {
+ public void shouldHandleLocalDateTime() throws Exception {
final LocalDateTime o = LocalDateTime.now();
assertEquals(o, serializeDeserialize(o, LocalDateTime.class));
}
@Test
- public void shouldHandleLocalTime()throws Exception {
+ public void shouldHandleLocalTime() throws Exception {
final LocalTime o = LocalTime.now();
assertEquals(o, serializeDeserialize(o, LocalTime.class));
}
@Test
- public void shouldHandleMonthDay()throws Exception {
+ public void shouldHandleMonthDay() throws Exception {
final MonthDay o = MonthDay.now();
assertEquals(o, serializeDeserialize(o, MonthDay.class));
}
@Test
- public void shouldHandleOffsetDateTime()throws Exception {
+ public void shouldHandleOffsetDateTime() throws Exception {
final OffsetDateTime o = OffsetDateTime.now();
assertEquals(o, serializeDeserialize(o, OffsetDateTime.class));
}
@Test
- public void shouldHandleOffsetTime()throws Exception {
+ public void shouldHandleOffsetTime() throws Exception {
final OffsetTime o = OffsetTime.now();
assertEquals(o, serializeDeserialize(o, OffsetTime.class));
}
@Test
- public void shouldHandlePeriod()throws Exception {
+ public void shouldHandlePeriod() throws Exception {
final Period o = Period.ofDays(3);
assertEquals(o, serializeDeserialize(o, Period.class));
}
@Test
- public void shouldHandleYear()throws Exception {
+ public void shouldHandleYear() throws Exception {
final Year o = Year.now();
assertEquals(o, serializeDeserialize(o, Year.class));
}
@Test
- public void shouldHandleYearMonth()throws Exception {
+ public void shouldHandleYearMonth() throws Exception {
final YearMonth o = YearMonth.now();
assertEquals(o, serializeDeserialize(o, YearMonth.class));
}
@Test
- public void shouldHandleZonedDateTime()throws Exception {
+ public void shouldHandleZonedDateTime() throws Exception {
final ZonedDateTime o = ZonedDateTime.now();
assertEquals(o, serializeDeserialize(o, ZonedDateTime.class));
}
@Test
- public void shouldHandleZonedOffset()throws Exception {
+ public void shouldHandleZonedOffset() throws Exception {
final ZoneOffset o = ZonedDateTime.now().getOffset();
assertEquals(o, serializeDeserialize(o, ZoneOffset.class));
}
@Test
- public void shouldHandleTraversalExplanation()throws Exception {
+ public void shouldHandleTraversalExplanation() throws Exception {
final TraversalExplanation te = __().out().outV().outE().explain();
assertEquals(te.toString(), serializeDeserialize(te, TraversalExplanation.class).toString());
}
@@ -349,4 +368,20 @@ public class GryoMapperTest {
}
}
}
+
+ final static class OverrideDurationSerializer extends Serializer<Duration>
+ {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Duration duration)
+ {
+ throw new UnsupportedOperationException("I don't do anything");
+ }
+
+ @Override
+ public Duration read(final Kryo kryo, final Input input, final Class<Duration> durationClass)
+ {
+ throw new UnsupportedOperationException("I don't do anything");
+ }
+ }
+
}
[05/27] incubator-tinkerpop git commit: minor tweaks to
TraverserVertexProgram to make it more efficient.
Posted by sp...@apache.org.
minor tweaks to TraverserVertexProgram to make it more efficient.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/7d29ef32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/7d29ef32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/7d29ef32
Branch: refs/heads/TINKERPOP-1308
Commit: 7d29ef32a42b143f9654457a46ab64a57446ef14
Parents: a6aacdc
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 18 17:12:40 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 18 17:12:40 2016 -0600
----------------------------------------------------------------------
.../process/computer/traversal/TraversalVertexProgram.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d29ef32/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index aba44d3..c7e7ef9 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -304,8 +304,9 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
return false;
} else {
// finalize locally with any last traversers dangling in the local traversal
- while (this.traversal.get().getEndStep().hasNext()) {
- final Traverser.Admin traverser = this.traversal.get().getEndStep().next();
+ final Step<?, ?> endStep = this.traversal.get().getEndStep();
+ while (endStep.hasNext()) {
+ final Traverser.Admin traverser = endStep.next();
traverser.detach();
haltedTraversers.add(traverser);
}
[07/27] incubator-tinkerpop git commit: added EmptyMemory and its
used as the memory for the first generateProgram() of an OLAP chain. Much
better than Optional(Memory).
Posted by sp...@apache.org.
added EmptyMemory and its used as the memory for the first generateProgram() of an OLAP chain. Much better than Optional(Memory).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b100f033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b100f033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b100f033
Branch: refs/heads/TINKERPOP-1308
Commit: b100f033e02ee5647dd31b2d556a1dcadac29818
Parents: 6107ff9
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 18 19:36:18 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 18 19:36:18 2016 -0600
----------------------------------------------------------------------
.../traversal/step/VertexComputing.java | 8 ++--
.../step/map/PageRankVertexProgramStep.java | 4 +-
.../step/map/PeerPressureVertexProgramStep.java | 9 +----
.../step/map/ProgramVertexProgramStep.java | 10 ++---
.../step/map/TraversalVertexProgramStep.java | 9 ++---
.../traversal/step/map/VertexProgramStep.java | 6 +--
.../optimization/GraphFilterStrategy.java | 4 +-
.../process/computer/util/EmptyMemory.java | 39 ++++++++++++++++++++
.../optimization/SparkInterceptorStrategy.java | 6 +--
.../SparkSingleIterationStrategy.java | 5 +--
10 files changed, 59 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
index b599c42..33f185a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/VertexComputing.java
@@ -25,8 +25,6 @@ import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import java.util.Optional;
-
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -51,11 +49,11 @@ public interface VertexComputing {
/**
* Generate the {@link VertexProgram}.
*
- * @param graph the {@link Graph} that the program will be executed over.
- * @param memoryOptional a reference to a {@link Memory} from the previous OLAP job if it exists.
+ * @param graph the {@link Graph} that the program will be executed over.
+ * @param memory the {@link Memory} from the previous OLAP job if it exists, else its an empty memory structure.
* @return the generated vertex program instance.
*/
- public VertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional);
+ public VertexProgram generateProgram(final Graph graph, final Memory memory);
/**
* @deprecated As of release 3.2.1. Please use {@link VertexComputing#getComputer()}.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
index 660f5b8..fcb2eec 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PageRankVertexProgramStep.java
@@ -21,7 +21,6 @@ package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.lambda.HaltedTraversersCountTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
@@ -38,7 +37,6 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
/**
@@ -84,7 +82,7 @@ public final class PageRankVertexProgramStep extends VertexProgramStep implement
}
@Override
- public PageRankVertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
+ public PageRankVertexProgram generateProgram(final Graph graph, final Memory memory) {
final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
final PageRankVertexProgram.Builder builder = PageRankVertexProgram.build()
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
index 2ae36a5..7bd726e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
@@ -36,7 +36,6 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
/**
@@ -85,15 +84,9 @@ public final class PeerPressureVertexProgramStep extends VertexProgramStep imple
}
@Override
- public PeerPressureVertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
+ public PeerPressureVertexProgram generateProgram(final Graph graph, final Memory memory) {
final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
- /*
- memoryOptional.ifPresent(memory -> {
- if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
- builder.configure(TraversalVertexProgram.HALTED_TRAVERSERS, memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
- });
- */
return PeerPressureVertexProgram.build()
.property(this.clusterProperty)
.maxIterations(this.times)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
index d060c99..82e70dd 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ProgramVertexProgramStep.java
@@ -31,7 +31,6 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -51,16 +50,13 @@ public final class ProgramVertexProgramStep extends VertexProgramStep {
}
@Override
- public VertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
+ public VertexProgram generateProgram(final Graph graph, final Memory memory) {
final MapConfiguration base = new MapConfiguration(this.configuration);
base.setDelimiterParsingDisabled(true);
PureTraversal.storeState(base, ROOT_TRAVERSAL, TraversalHelper.getRootTraversal(this.getTraversal()).clone());
base.setProperty(STEP_ID, this.getId());
- memoryOptional.ifPresent(memory -> {
- if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS)) {
- TraversalVertexProgram.storeHaltedTraversers(base, memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
- }
- });
+ if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
+ TraversalVertexProgram.storeHaltedTraversers(base, memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
return VertexProgram.createVertexProgram(graph, base);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
index 5851f35..58e44a2 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/TraversalVertexProgramStep.java
@@ -33,7 +33,6 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
/**
@@ -64,17 +63,15 @@ public final class TraversalVertexProgramStep extends VertexProgramStep implemen
}
@Override
- public TraversalVertexProgram generateProgram(final Graph graph, final Optional<Memory> memoryOptional) {
+ public TraversalVertexProgram generateProgram(final Graph graph, final Memory memory) {
final Traversal.Admin<?, ?> computerSpecificTraversal = this.computerTraversal.getPure();
computerSpecificTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()).clone());
this.getTraversal().getStrategies().toList().forEach(computerSpecificTraversal.getStrategies()::addStrategies);
computerSpecificTraversal.setSideEffects(new MemoryTraversalSideEffects(this.getTraversal().getSideEffects()));
computerSpecificTraversal.setParent(this);
final TraversalVertexProgram.Builder builder = TraversalVertexProgram.build().traversal(computerSpecificTraversal);
- memoryOptional.ifPresent(memory -> {
- if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
- builder.haltedTraversers(memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
- });
+ if (memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS))
+ builder.haltedTraversers(memory.get(TraversalVertexProgram.HALTED_TRAVERSERS));
return builder.create(graph);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
index ac3de30..a5d7105 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStep.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.VertexComputing;
+import org.apache.tinkerpop.gremlin.process.computer.util.EmptyMemory;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
@@ -36,7 +37,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedE
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.util.NoSuchElementException;
-import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -64,7 +64,7 @@ public abstract class VertexProgramStep extends AbstractStep<ComputerResult, Com
if (this.first && this.getPreviousStep() instanceof EmptyStep) {
this.first = false;
final Graph graph = this.getTraversal().getGraph().get();
- future = this.generateComputer(graph).program(this.generateProgram(graph, Optional.empty())).submit();
+ future = this.generateComputer(graph).program(this.generateProgram(graph, EmptyMemory.instance())).submit();
final ComputerResult result = future.get();
this.processMemorySideEffects(result.memory());
return this.getTraversal().getTraverserGenerator().generate(result, this, 1l);
@@ -72,7 +72,7 @@ public abstract class VertexProgramStep extends AbstractStep<ComputerResult, Com
final Traverser.Admin<ComputerResult> traverser = this.starts.next();
final Graph graph = traverser.get().graph();
final Memory memory = traverser.get().memory();
- future = this.generateComputer(graph).program(this.generateProgram(graph, Optional.of(memory))).submit();
+ future = this.generateComputer(graph).program(this.generateProgram(graph, memory)).submit();
final ComputerResult result = future.get();
this.processMemorySideEffects(result.memory());
return traverser.split(result, this);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
index e4cf5ff..c32777b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/optimization/GraphFilterStrategy.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.computer.Computer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.util.EmptyMemory;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
@@ -41,7 +42,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
/**
@@ -60,7 +60,7 @@ public final class GraphFilterStrategy extends AbstractTraversalStrategy<Travers
return;
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // given that this strategy only works for single OLAP jobs, the graph is the traversal graph
for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) { // will be zero or one step
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, Optional.empty()).getTraversal().get().clone();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, EmptyMemory.instance()).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
final Computer computer = step.getComputer();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
new file mode 100644
index 0000000..72b1bbf
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.computer.util;
+
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class EmptyMemory {
+
+ private static final Memory INSTANCE = new ImmutableMemory(new MapMemory());
+
+ private EmptyMemory() {
+
+ }
+
+ public static Memory instance() {
+ return INSTANCE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
index a2cc518..bb8476a 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategy.java
@@ -20,8 +20,8 @@
package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.util.EmptyMemory;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy;
@@ -30,8 +30,6 @@ import org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.op
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
-import java.util.Optional;
-
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -46,7 +44,7 @@ public final class SparkInterceptorStrategy extends AbstractTraversalStrategy<Tr
public void apply(final Traversal.Admin<?, ?> traversal) {
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined
for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, Optional.empty()).getTraversal().get().clone();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, EmptyMemory.instance()).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
if (SparkStarBarrierInterceptor.isLegal(computerTraversal)) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b100f033/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
index 280c7ba..a4acf4c 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategy.java
@@ -20,8 +20,8 @@
package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.optimization;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.computer.util.EmptyMemory;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
@@ -37,7 +37,6 @@ import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
import java.util.Arrays;
import java.util.HashSet;
-import java.util.Optional;
import java.util.Set;
/**
@@ -61,7 +60,7 @@ public final class SparkSingleIterationStrategy extends AbstractTraversalStrateg
public void apply(final Traversal.Admin<?, ?> traversal) {
final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined
for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {
- final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, Optional.empty()).getTraversal().get().clone();
+ final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, EmptyMemory.instance()).getTraversal().get().clone();
if (!computerTraversal.isLocked())
computerTraversal.applyStrategies();
boolean doesMessagePass = TraversalHelper.hasStepOfAssignableClassRecursively(Scope.global, MULTI_ITERATION_CLASSES, computerTraversal);
[13/27] incubator-tinkerpop git commit: fixed the NPE that occurs in
OLAP when you have a local/OLTP group() nested within an OLAP group(). The
solution is elegant, but the problem,
its not as efficient as the code when we had the NPE.... dar. Going to f
Posted by sp...@apache.org.
fixed the NPE that occurs in OLAP when you have a local/OLTP group() nested within an OLAP group(). The solution is elegant, but the problem, its not as efficient as the code when we had the NPE.... dar. Going to fiddle some more tomorrow to see if I can get it faster --- 600ms vs 400ms differences.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c890ceba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c890ceba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c890ceba
Branch: refs/heads/TINKERPOP-1308
Commit: c890cebad5231cdab8744878bebb4ea363689a3a
Parents: 44d40f6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon May 23 16:24:58 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon May 23 16:24:58 2016 -0600
----------------------------------------------------------------------
.../process/traversal/step/map/GroupStep.java | 210 ++++++++++---------
.../step/sideEffect/GroupSideEffectStep.java | 59 ++----
.../step/sideEffect/GroovyGroupTest.groovy | 5 +
.../traversal/step/sideEffect/GroupTest.java | 42 ++++
.../SparkStarBarrierInterceptor.java | 1 -
.../structure/TinkerGraphPlayTest.java | 4 +-
6 files changed, 176 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c890ceba/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
index 4f5df35..b430d8f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
@@ -29,7 +29,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.lambda.IdentityTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.lambda.TokenTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.ByModulating;
-import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
@@ -39,6 +38,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.apache.tinkerpop.gremlin.util.function.HashMapSupplier;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.javatuples.Pair;
import java.io.Serializable;
import java.util.ArrayList;
@@ -46,25 +46,24 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.function.BinaryOperator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>> implements ByModulating, TraversalParent, GraphComputing {
+public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>> implements ByModulating, TraversalParent {
private char state = 'k';
private Traversal.Admin<S, K> keyTraversal = null;
- private Traversal.Admin<S, V> valueTraversal = this.integrateChild(__.fold().asAdmin());
- private Traversal.Admin<S, ?> preTraversal = null; // used in OLAP
- private ReducingBarrierStep reducingBarrierStep = null; // used in OLAP
- private boolean onGraphComputer = false;
+ private Traversal.Admin<S, ?> preTraversal;
+ private Traversal.Admin<S, V> valueTraversal;
public GroupStep(final Traversal.Admin traversal) {
super(traversal);
- this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal, this.onGraphComputer));
+ this.valueTraversal = this.integrateChild(__.fold().asAdmin());
+ this.preTraversal = this.integrateChild(splitOnBarrierStep(this.valueTraversal).get(0));
+ this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal));
this.setSeedSupplier(HashMapSupplier.instance());
}
@@ -75,7 +74,8 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
this.state = 'v';
} else if ('v' == this.state) {
this.valueTraversal = this.integrateChild(convertValueTraversal(kvTraversal));
- this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal, this.onGraphComputer));
+ this.preTraversal = this.integrateChild(splitOnBarrierStep(this.valueTraversal).get(0));
+ this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal));
this.state = 'x';
} else {
throw new IllegalStateException("The key and value traversals for group()-step have already been set: " + this);
@@ -85,21 +85,11 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
@Override
public Map<K, V> projectTraverser(final Traverser.Admin<S> traverser) {
final Map<K, V> map = new HashMap<>(1);
- final K key = TraversalUtil.applyNullable(traverser, this.keyTraversal);
- if (this.onGraphComputer) {
- if (null == this.reducingBarrierStep) {
- final TraverserSet traverserSet = new TraverserSet();
- this.preTraversal.reset();
- this.preTraversal.addStart(traverser.split());
- this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
- map.put(key, (V) traverserSet);
- } else {
- this.valueTraversal.reset();
- this.valueTraversal.addStart(traverser.split());
- map.put(key, (V) this.reducingBarrierStep.nextBarrier());
- }
- } else
- map.put(key, (V) traverser);
+ final TraverserSet traverserSet = new TraverserSet<>();
+ this.preTraversal.reset();
+ this.preTraversal.addStart(traverser.split());
+ this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverserSet);
return map;
}
@@ -128,11 +118,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
if (null != this.keyTraversal)
clone.keyTraversal = this.keyTraversal.clone();
clone.valueTraversal = this.valueTraversal.clone();
- if (null != this.preTraversal)
- clone.preTraversal = this.preTraversal.clone();
- final Optional<Barrier> optional = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, clone.valueTraversal);
- if (optional.isPresent() && optional.get() instanceof ReducingBarrierStep)
- clone.reducingBarrierStep = (ReducingBarrierStep) optional.get();
+ clone.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(clone.valueTraversal).get(0));
return clone;
}
@@ -154,37 +140,19 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
@Override
public Map<K, V> generateFinalResult(final Map<K, V> object) {
- return GroupStep.doFinalReduction((Map<K, Object>) object, this.valueTraversal, this.onGraphComputer);
- }
-
- @Override
- public void onGraphComputer() {
- this.preTraversal = this.integrateChild(splitOnBarrierStep(this.valueTraversal).get(0));
- final Optional<Barrier> optional = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, this.valueTraversal);
- if (optional.isPresent() && optional.get() instanceof ReducingBarrierStep)
- this.reducingBarrierStep = (ReducingBarrierStep) optional.get();
- this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal, this.onGraphComputer = true));
+ return GroupStep.doFinalReduction((Map<K, Object>) object, this.valueTraversal);
}
///////////////////////
public static final class GroupBiOperator<K, V> implements BinaryOperator<Map<K, V>>, Serializable {
- private boolean onGraphComputer;
- private BinaryOperator reducingBinaryOperator; // OLAP (w/ reducer)
- private transient Traversal.Admin<?, V> valueTraversal; // OLTP
- private transient Map<K, Integer> counters; // OLTP
+ private Traversal.Admin<?, V> valueTraversal;
+ private ReducingBarrierStep reducingBarrierStep = null;
- public GroupBiOperator(final Traversal.Admin<?, V> valueTraversal, final boolean onGraphComputer) {
- this.onGraphComputer = onGraphComputer;
- if (this.onGraphComputer) {
- final Optional<Barrier> optional = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversal);
- if (optional.isPresent() && optional.get() instanceof ReducingBarrierStep)
- this.reducingBinaryOperator = ((ReducingBarrierStep) optional.get()).getBiOperator();
- } else {
- this.valueTraversal = valueTraversal;
- this.counters = new HashMap<>();
- }
+ public GroupBiOperator(final Traversal.Admin<?, V> valueTraversal) {
+ this.valueTraversal = valueTraversal.clone();
+ this.reducingBarrierStep = TraversalHelper.getFirstStepOfAssignableClass(ReducingBarrierStep.class, this.valueTraversal).orElse(null);
}
public GroupBiOperator() {
@@ -194,43 +162,84 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
@Override
public Map<K, V> apply(final Map<K, V> mapA, final Map<K, V> mapB) {
for (final K key : mapB.keySet()) {
- if (this.onGraphComputer) {
- final Object objectB = mapB.get(key);
- if (null != this.reducingBinaryOperator) {
- // OLAP -- if there is a mid-traversal, apply the binary reducer and propagate the mutating barrier
- final Object objectA = mapA.get(key);
- mapA.put(key, (V) (null == objectA ? objectB : this.reducingBinaryOperator.apply(objectA, objectB)));
- } else {
- // OLAP -- if there is no mid-traversal reducer, aggregate pre-barrier traversers into a traverser set (expensive, but that's that)
- final Object objectA = mapA.get(key);
- final TraverserSet traverserSet;
- if (null == objectA) {
- traverserSet = new TraverserSet();
- mapA.put(key, (V) traverserSet);
- } else
- traverserSet = (TraverserSet) objectA;
- traverserSet.addAll((TraverserSet) objectB);
- }
+ Object objectA = mapA.get(key);
+ final Object objectB = mapB.get(key);
+ assert null != objectB;
+ if (null == objectA) {
+ objectA = objectB;
} else {
- // OLTP -- do mid-barrier reductions if they exist, else don't. Bulking is also available here because of addStart() prior to barrier.
- final Traverser.Admin traverser = (Traverser.Admin) mapB.get(key);
- Traversal.Admin valueTraversalClone = (Traversal.Admin) mapA.get(key);
- if (null == valueTraversalClone) {
- this.counters.put(key, 0);
- valueTraversalClone = this.valueTraversal.clone();
- mapA.put(key, (V) valueTraversalClone);
- }
- valueTraversalClone.addStart(traverser);
- if (this.counters.compute(key, (k, i) -> ++i) > 1000) {
- this.counters.put(key, 0);
- TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversalClone).ifPresent(Barrier::processAllStarts);
+ if (objectA instanceof TraverserSet) {
+ if (objectB instanceof TraverserSet) {
+ final TraverserSet set = (TraverserSet) objectA;
+ set.addAll((TraverserSet) objectB);
+ if (null != this.reducingBarrierStep && set.size() > 1000) {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addStarts(set.iterator());
+ objectA = this.reducingBarrierStep.nextBarrier();
+ }
+ } else if (objectB instanceof Pair) {
+ final TraverserSet set = (TraverserSet) objectA;
+ set.addAll((TraverserSet) ((Pair) objectB).getValue0());
+ if (set.size() > 1000) {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addStarts(set.iterator());
+ this.reducingBarrierStep.addBarrier(((Pair) objectB).getValue1());
+ objectA = this.reducingBarrierStep.nextBarrier();
+ } else {
+ objectA = Pair.with(set, ((Pair) objectB).getValue1());
+ }
+ } else {
+ objectA = Pair.with(objectA, objectB);
+ }
+ } else if (objectA instanceof Pair) {
+ if (objectB instanceof TraverserSet) {
+ final TraverserSet set = (TraverserSet) ((Pair) objectA).getValue0();
+ set.addAll((TraverserSet) objectB);
+ if (null != this.reducingBarrierStep &&set.size() > 1000) {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addStarts(set.iterator());
+ this.reducingBarrierStep.addBarrier(((Pair) objectA).getValue1());
+ objectA = this.reducingBarrierStep.nextBarrier();
+ }
+ } else if (objectB instanceof Pair) {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addBarrier(((Pair) objectA).getValue1());
+ this.reducingBarrierStep.addBarrier(((Pair) objectB).getValue1());
+ this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
+ this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectB).getValue0()).iterator());
+ objectA = this.reducingBarrierStep.nextBarrier();
+ } else {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addBarrier(((Pair) objectA).getValue1());
+ this.reducingBarrierStep.addBarrier(objectB);
+ this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
+ objectA = this.reducingBarrierStep.nextBarrier();
+ }
+ } else {
+ if (objectB instanceof TraverserSet) {
+ objectA = Pair.with(objectB, objectA);
+ } else if (objectB instanceof Pair) {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addBarrier(objectA);
+ this.reducingBarrierStep.addBarrier(((Pair) objectB).getValue1());
+ this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectB).getValue0()).iterator());
+ objectA = this.reducingBarrierStep.nextBarrier();
+ } else {
+ this.valueTraversal.reset();
+ this.reducingBarrierStep.addBarrier(objectA);
+ this.reducingBarrierStep.addBarrier(objectB);
+ objectA = this.reducingBarrierStep.nextBarrier();
+ }
}
}
+ mapA.put(key, (V) objectA);
}
return mapA;
+
}
}
+
///////////////////////
public static <S, E> Traversal.Admin<S, E> convertValueTraversal(final Traversal.Admin<S, E> valueTraversal) {
@@ -263,26 +272,25 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
}
}
- public static <K, V> Map<K, V> doFinalReduction(final Map<K, Object> map, final Traversal.Admin<?, V> valueTraversal, final boolean onGraphComputer) {
+ public static <K, V> Map<K, V> doFinalReduction(final Map<K, Object> map, final Traversal.Admin<?, V> valueTraversal) {
final Map<K, V> reducedMap = new HashMap<>(map.size());
- // if not on OLAP, who cares --- don't waste time computing barriers
- final boolean hasReducingBarrier = onGraphComputer &&
- TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversal).isPresent() &&
- TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversal).get() instanceof ReducingBarrierStep;
- final Traversal.Admin<?, ?> postTraversal = (onGraphComputer & !hasReducingBarrier) ? splitOnBarrierStep(valueTraversal.clone()).get(1) : null;
+ final ReducingBarrierStep reducingBarrierStep = TraversalHelper.getFirstStepOfAssignableClass(ReducingBarrierStep.class, valueTraversal).orElse(null);
IteratorUtils.removeOnNext(map.entrySet().iterator()).forEachRemaining(entry -> {
- if (onGraphComputer) {
- if (hasReducingBarrier) { // OLAP with reduction (barrier)
- valueTraversal.reset();
- TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversal).get().addBarrier(entry.getValue());
- reducedMap.put(entry.getKey(), valueTraversal.next());
- } else { // OLAP without reduction (traverser set)
- postTraversal.reset();
- postTraversal.addStarts(((TraverserSet) entry.getValue()).iterator());
- reducedMap.put(entry.getKey(), (V) postTraversal.next());
- }
- } else // OLTP is just a traversal
- reducedMap.put(entry.getKey(), ((Traversal.Admin<?, V>) entry.getValue()).next());
+ valueTraversal.reset();
+ if (null == reducingBarrierStep) {
+ reducedMap.put(entry.getKey(), entry.getValue() instanceof TraverserSet ?
+ ((TraverserSet<V>) entry.getValue()).iterator().next().get() :
+ (V) entry.getValue());
+ } else {
+ if (entry.getValue() instanceof TraverserSet)
+ reducingBarrierStep.addStarts(((TraverserSet) entry.getValue()).iterator());
+ else if (entry.getValue() instanceof Pair) {
+ reducingBarrierStep.addStarts(((TraverserSet) (((Pair) entry.getValue()).getValue0())).iterator());
+ reducingBarrierStep.addBarrier((((Pair) entry.getValue()).getValue1()));
+ } else
+ reducingBarrierStep.addBarrier(entry.getValue());
+ reducedMap.put(entry.getKey(), valueTraversal.next());
+ }
});
assert map.isEmpty();
map.clear();
@@ -290,3 +298,5 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
return (Map<K, V>) map;
}
}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c890ceba/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
index a906312..4fc4ffa 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
@@ -21,16 +21,12 @@ package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.ByModulating;
-import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
import org.apache.tinkerpop.gremlin.process.traversal.step.SideEffectCapable;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.apache.tinkerpop.gremlin.util.function.HashMapSupplier;
@@ -39,27 +35,26 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implements SideEffectCapable<Map<K, ?>, Map<K, V>>, TraversalParent, ByModulating, GraphComputing {
+public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implements SideEffectCapable<Map<K, ?>, Map<K, V>>, TraversalParent, ByModulating {
private char state = 'k';
private Traversal.Admin<S, K> keyTraversal = null;
+ private Traversal.Admin<S, ?> preTraversal = null;
private Traversal.Admin<S, V> valueTraversal = this.integrateChild(__.fold().asAdmin());
- private Traversal.Admin<S, ?> preTraversal = null; // used in OLAP
- private ReducingBarrierStep reducingBarrierStep = null; // used in OLAP
- private boolean onGraphComputer = false;
///
private String sideEffectKey;
public GroupSideEffectStep(final Traversal.Admin traversal, final String sideEffectKey) {
super(traversal);
this.sideEffectKey = sideEffectKey;
- this.getTraversal().getSideEffects().registerIfAbsent(this.sideEffectKey, HashMapSupplier.instance(), new GroupStep.GroupBiOperator<>(this.valueTraversal, this.onGraphComputer));
+ this.valueTraversal = this.integrateChild(__.fold().asAdmin());
+ this.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(this.valueTraversal).get(0));
+ this.getTraversal().getSideEffects().registerIfAbsent(this.sideEffectKey, HashMapSupplier.instance(), new GroupStep.GroupBiOperator<>(this.valueTraversal));
}
@Override
@@ -69,7 +64,8 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
this.state = 'v';
} else if ('v' == this.state) {
this.valueTraversal = this.integrateChild(GroupStep.convertValueTraversal(kvTraversal));
- this.getTraversal().getSideEffects().register(this.sideEffectKey, null, new GroupStep.GroupBiOperator<>(this.valueTraversal, this.onGraphComputer));
+ this.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(this.valueTraversal).get(0));
+ this.getTraversal().getSideEffects().register(this.sideEffectKey, null, new GroupStep.GroupBiOperator<>(this.valueTraversal));
this.state = 'x';
} else {
throw new IllegalStateException("The key and value traversals for group()-step have already been set: " + this);
@@ -78,22 +74,14 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
@Override
protected void sideEffect(final Traverser.Admin<S> traverser) {
- final Map<K, Object> map = new HashMap<>(1);
- final K key = TraversalUtil.applyNullable(traverser, this.keyTraversal);
- if (this.onGraphComputer) {
- if (null == this.reducingBarrierStep) {
- final TraverserSet traverserSet = new TraverserSet<>();
- this.preTraversal.reset();
- this.preTraversal.addStart(traverser.split());
- this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
- map.put(key, traverserSet);
- } else {
- this.valueTraversal.reset();
- this.valueTraversal.addStart(traverser.split());
- map.put(key, (V) this.reducingBarrierStep.nextBarrier());
- }
- } else
- map.put(key, traverser.split());
+ final Map<K, V> map = new HashMap<>(1);
+ final TraverserSet midTraversers = new TraverserSet<>();
+ this.preTraversal.reset();
+ this.preTraversal.addStart(traverser.split());
+ while (this.preTraversal.hasNext()) {
+ midTraversers.add(this.preTraversal.getEndStep().next());
+ }
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) midTraversers);
this.getTraversal().getSideEffects().add(this.sideEffectKey, map);
}
@@ -127,11 +115,7 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
if (null != this.keyTraversal)
clone.keyTraversal = this.keyTraversal.clone();
clone.valueTraversal = this.valueTraversal.clone();
- if (null != this.preTraversal)
- clone.preTraversal = this.preTraversal.clone();
- final Optional<Barrier> optional = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, clone.valueTraversal);
- if (optional.isPresent() && optional.get() instanceof ReducingBarrierStep)
- clone.reducingBarrierStep = (ReducingBarrierStep) optional.get();
+ clone.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(clone.valueTraversal).get(0));
return clone;
}
@@ -153,15 +137,6 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
@Override
public Map<K, V> generateFinalResult(final Map<K, ?> object) {
- return GroupStep.doFinalReduction((Map<K, Object>) object, this.valueTraversal, this.onGraphComputer);
- }
-
- @Override
- public void onGraphComputer() {
- this.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(this.valueTraversal).get(0));
- final Optional<Barrier> optional = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, this.valueTraversal);
- if (optional.isPresent() && optional.get() instanceof ReducingBarrierStep)
- this.reducingBarrierStep = (ReducingBarrierStep) optional.get();
- this.getTraversal().getSideEffects().register(this.sideEffectKey, null, new GroupStep.GroupBiOperator<>(this.valueTraversal, this.onGraphComputer = true));
+ return GroupStep.doFinalReduction((Map<K, Object>) object, this.valueTraversal);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c890ceba/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
index 619815f..ddfb94a 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
@@ -108,5 +108,10 @@ public abstract class GroovyGroupTest {
public Traversal<Vertex, Map<String, Map<Object, Object>>> get_g_V_repeatXunionXoutXknowsX_groupXaX_byXageX__outXcreatedX_groupXbX_byXnameX_byXcountXX_groupXaX_byXnameXX_timesX2X_capXa_bX() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.repeat(union(out('knows').group('a').by('age'), out('created').group('b').by('name').by(count())).group('a').by('name')).times(2).cap('a', 'b')")
}
+
+ @Override
+ public Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX() {
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.group().by(bothE().count).by(group.by(label))")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c890ceba/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
index d844ba7..d4c4d74 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
@@ -84,6 +84,8 @@ public abstract class GroupTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Map<String, Map<Object, Object>>> get_g_V_repeatXunionXoutXknowsX_groupXaX_byXageX__outXcreatedX_groupXbX_byXnameX_byXcountXX_groupXaX_byXnameXX_timesX2X_capXa_bX();
+ public abstract Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX();
+
@Test
@LoadGraphWith(MODERN)
public void g_V_group_byXnameX() {
@@ -356,6 +358,41 @@ public abstract class GroupTest extends AbstractGremlinProcessTest {
checkSideEffects(traversal.asAdmin().getSideEffects(), "a", HashMap.class, "b", HashMap.class);
}
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_group_byXbothE_countX_byXgroup_byXlabelXX() {
+ final Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> traversal = get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX();
+ final Map<Long, Map<String, List<Vertex>>> map = traversal.next();
+ assertFalse(traversal.hasNext());
+ assertEquals(2, map.size());
+ assertTrue(map.containsKey(1l));
+ assertTrue(map.containsKey(3l));
+ //
+ Map<String, List<Vertex>> submap = map.get(1l);
+ assertEquals(2, submap.size());
+ assertTrue(submap.containsKey("software"));
+ assertTrue(submap.containsKey("person"));
+ List<Vertex> list = submap.get("software");
+ assertEquals(1, list.size());
+ assertEquals(convertToVertex(graph, "ripple"), list.get(0));
+ list = submap.get("person");
+ assertEquals(2, list.size());
+ assertTrue(list.contains(convertToVertex(graph, "vadas")));
+ assertTrue(list.contains(convertToVertex(graph, "peter")));
+ //
+ submap = map.get(3l);
+ assertEquals(2, submap.size());
+ assertTrue(submap.containsKey("software"));
+ assertTrue(submap.containsKey("person"));
+ list = submap.get("software");
+ assertEquals(1, list.size());
+ assertEquals(convertToVertex(graph, "lop"), list.get(0));
+ list = submap.get("person");
+ assertEquals(2, list.size());
+ assertTrue(list.contains(convertToVertex(graph, "marko")));
+ assertTrue(list.contains(convertToVertex(graph, "josh")));
+ }
+
public static class Traversals extends GroupTest {
@Override
@@ -437,5 +474,10 @@ public abstract class GroupTest extends AbstractGremlinProcessTest {
public Traversal<Vertex, Map<String, Map<Object, Object>>> get_g_V_repeatXunionXoutXknowsX_groupXaX_byXageX__outXcreatedX_groupXbX_byXnameX_byXcountXX_groupXaX_byXnameXX_timesX2X_capXa_bX() {
return g.V().repeat(__.union(__.out("knows").group("a").by("age"), __.out("created").group("b").by("name").by(count())).group("a").by("name")).times(2).cap("a", "b");
}
+
+ @Override
+ public Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX() {
+ return g.V().<Long, Map<String, List<Vertex>>>group().by(__.bothE().count()).by(__.group().by(T.label));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c890ceba/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
index 768d10a..5c6d729 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
@@ -126,7 +126,6 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
}
}).fold(endStep.getSeedSupplier().get(), biOperator::apply);
} else if (endStep instanceof GroupStep) {
- ((GroupStep) endStep).onGraphComputer();
final GroupStep.GroupBiOperator<Object, Object> biOperator = (GroupStep.GroupBiOperator) endStep.getBiOperator();
result = ((GroupStep) endStep).generateFinalResult(nextRDD.
mapPartitions(partitions -> {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c890ceba/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
index e339519..5557716 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
@@ -86,11 +86,11 @@ public class TinkerGraphPlayTest {
@Ignore
public void benchmarkGroup() throws Exception {
Graph graph = TinkerGraph.open();
- GraphTraversalSource g = graph.traversal().withComputer();
+ GraphTraversalSource g = graph.traversal();
graph.io(GraphMLIo.build()).readGraph("../data/grateful-dead.xml");
/////////
- g.V().group().by(T.label).by(values("name")).forEachRemaining(x -> logger.info(x.toString()));
+ //g.V().group().by(T.label).by(values("name")).forEachRemaining(x -> logger.info(x.toString()));
System.out.println("group: " + g.V().both("followedBy").both("followedBy").group().by("songType").by(count()).next());
System.out.println("groupV3d0: " + g.V().both("followedBy").both("followedBy").groupV3d0().by("songType").by().by(__.count(Scope.local)).next());
[20/27] incubator-tinkerpop git commit: Merge branch 'master' into
TINKERPOP-1307
Posted by sp...@apache.org.
Merge branch 'master' into TINKERPOP-1307
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4318b6c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4318b6c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4318b6c4
Branch: refs/heads/TINKERPOP-1308
Commit: 4318b6c4c8a005117590cfc1e861d1a88ad4c068
Parents: 085be94 afd4048
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 25 08:43:24 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 25 08:43:24 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../traversal/step/sideEffect/SubgraphStep.java | 19 ++++++----
.../step/sideEffect/GroovySubgraphTest.groovy | 5 +++
.../traversal/step/sideEffect/SubgraphTest.java | 40 ++++++++++++++++++--
4 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[03/27] incubator-tinkerpop git commit: TraverserExecutor is smart to
not do another iteration if the traversers will simply be returned to the
master traversals as output. Updated the PageRankTest with a better test.
Posted by sp...@apache.org.
TraverserExecutor is smart to not do another iteration if the traversers will simply be returned to the master traversals as output. Updated the PageRankTest with a better test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/07e12995
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/07e12995
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/07e12995
Branch: refs/heads/TINKERPOP-1308
Commit: 07e12995ae6bed0a3b1da3942bc54371d94a66a9
Parents: 4ee3028
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 18 17:02:22 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 18 17:02:22 2016 -0600
----------------------------------------------------------------------
.../process/computer/traversal/TraverserExecutor.java | 10 ++++++++--
.../process/traversal/step/map/GroovyPageRankTest.groovy | 4 ++--
.../gremlin/process/traversal/step/map/PageRankTest.java | 10 ++++++----
3 files changed, 16 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/07e12995/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
index f5facdb..c5f9bb5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
@@ -159,7 +159,10 @@ public final class TraverserExecutor {
final TraverserSet<Object> barrierSet = barrier.nextBarrier();
IteratorUtils.removeOnNext(barrierSet.iterator()).forEachRemaining(traverser -> {
traverser.addLabels(step.getLabels()); // this might need to be generalized for working with global barriers too
- if (traverser.isHalted() && ((!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) || getHostingVertex(traverser.get()).equals(vertex))) {
+ if (traverser.isHalted() &&
+ (returnHaltedTraversers ||
+ (!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) ||
+ getHostingVertex(traverser.get()).equals(vertex))) {
traverser.detach();
if (returnHaltedTraversers)
memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
@@ -181,7 +184,10 @@ public final class TraverserExecutor {
}
} else { // LOCAL PROCESSING
step.forEachRemaining(traverser -> {
- if (traverser.isHalted() && ((!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) || getHostingVertex(traverser.get()).equals(vertex))) {
+ if (traverser.isHalted() &&
+ (returnHaltedTraversers ||
+ (!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) ||
+ getHostingVertex(traverser.get()).equals(vertex))) {
traverser.detach();
if (returnHaltedTraversers)
memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/07e12995/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
index e27e8c0..8608c4d 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPageRankTest.groovy
@@ -71,8 +71,8 @@ public abstract class GroovyPageRankTest {
}
@Override
- public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
- new ScriptTraversal<>(g, "gremlin-groovy", "g.V.out('created').pageRank().by(bothE()).by('projectRank').valueMap('name','projectRank')")
+ public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX() {
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.out('created').pageRank().by(bothE()).by('projectRank').times(0).valueMap('name','projectRank')")
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/07e12995/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
index 5a3ed9b..1e18b48 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PageRankTest.java
@@ -49,7 +49,7 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Vertex> get_g_V_pageRank();
- public abstract Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX();
+ public abstract Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX();
public abstract Traversal<Vertex, String> get_g_V_pageRank_order_byXpageRank_decrX_name();
@@ -82,7 +82,7 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
public void g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
- final Traversal<Vertex, Map<String, Object>> traversal = get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX();
+ final Traversal<Vertex, Map<String, Object>> traversal = get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX();
printTraversalForm(traversal);
final Map<String, Double> map = new HashMap<>();
traversal.forEachRemaining(m -> map.put(((List<String>) m.get("name")).get(0), ((List<Double>) m.get("projectRank")).get(0)));
@@ -90,6 +90,8 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
assertTrue(map.containsKey("lop"));
assertTrue(map.containsKey("ripple"));
assertTrue(map.get("lop") > map.get("ripple"));
+ assertEquals(3.0d, map.get("lop"), 0.001d);
+ assertEquals(1.0d, map.get("ripple"), 0.001d);
assertFalse(traversal.hasNext());
}
@@ -242,8 +244,8 @@ public abstract class PageRankTest extends AbstractGremlinProcessTest {
}
@Override
- public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_valueMapXname_projectRankX() {
- return g.V().out("created").pageRank().by(__.bothE()).by("projectRank").valueMap("name", "projectRank");
+ public Traversal<Vertex, Map<String, Object>> get_g_V_outXcreatedX_pageRank_byXbothEX_byXprojectRankX_timesX0X_valueMapXname_projectRankX() {
+ return g.V().out("created").pageRank().by(__.bothE()).by("projectRank").times(0).valueMap("name", "projectRank");
}
@Override
[26/27] incubator-tinkerpop git commit: Added gryo "lite" serializer.
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/driver/remote/gremlin-server-integration.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/driver/remote/gremlin-server-integration.yaml b/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/driver/remote/gremlin-server-integration.yaml
index adb0fe3..86245b5 100644
--- a/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/driver/remote/gremlin-server-integration.yaml
+++ b/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/driver/remote/gremlin-server-integration.yaml
@@ -38,6 +38,7 @@ scriptEngines: {
staticImports: [java.lang.Math.PI]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph, custom: [groovy.json.JsonBuilder;org.apache.tinkerpop.gremlin.driver.ser.JsonBuilderGryoSerializer]}}
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph, custom: [groovy.json.JsonBuilder;org.apache.tinkerpop.gremlin.driver.ser.JsonBuilderGryoSerializer]}}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true}}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/server/gremlin-server-integration.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/server/gremlin-server-integration.yaml b/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/server/gremlin-server-integration.yaml
index fe674aa..c1a5214 100644
--- a/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/server/gremlin-server-integration.yaml
+++ b/gremlin-server/src/test/resources/org/apache/tinkerpop/gremlin/server/gremlin-server-integration.yaml
@@ -34,6 +34,7 @@ scriptEngines: {
staticImports: [java.lang.Math.PI]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph, custom: [groovy.json.JsonBuilder;org.apache.tinkerpop.gremlin.driver.ser.JsonBuilderGryoSerializer]}}
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph, custom: [groovy.json.JsonBuilder;org.apache.tinkerpop.gremlin.driver.ser.JsonBuilderGryoSerializer]}}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true}}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }}
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }}
[10/27] incubator-tinkerpop git commit: updated CHANGELOG and upgrade
docs.
Posted by sp...@apache.org.
updated CHANGELOG and upgrade docs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6ffeaf1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6ffeaf1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6ffeaf1b
Branch: refs/heads/TINKERPOP-1308
Commit: 6ffeaf1bf95dbc3403475e3fb9801d2a7870de2b
Parents: 9c53c73
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri May 20 22:11:55 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri May 20 22:11:55 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 6 ++++++
docs/src/upgrade/release-3.2.x-incubating.asciidoc | 9 +++++++++
2 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffeaf1b/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 628b4f4..eb79ae8 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,12 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.2.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Fixed a severe bug around halted traversers in a multi-job OLAP traversal chain.
+* Ensure a separation of `GraphComputer` and `VertexProgram` configurations in `SparkGraphComputer` and `GiraphGraphComputer`.
+* `PeerPressureVertexProgram` now supports dynamic initial vote strength calculations.
+* Added `EmptyMemory` for ease of use when no memory exists.
+* Updated `VertexComputing.generateProgram()` API to include `Memory`. (*breaking*)
+* `ImmutablePath.TailPath` is now serializable like `ImmutablePath`.
* Added a traversal style guide to the recipes cookbook.
* Fixed a bug in master-traversal traverser propagation.
* Added useful methods for custom `VertexPrograms` to be used with `program()`-step.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffeaf1b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.2.x-incubating.asciidoc b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
index 1b5aa51..66e5f8c 100644
--- a/docs/src/upgrade/release-3.2.x-incubating.asciidoc
+++ b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
@@ -112,6 +112,15 @@ Upgrading for Providers
Graph System Providers
^^^^^^^^^^^^^^^^^^^^^^
+VertexComputing API Change
++++++++++++++++++++++++
+
+The `VertexComputing` API is used by steps that wrap a `VertexProgram`. There is a method called
+`VertexComputing.generateProgram()` that has changed which now takes a second argument of `Memory`. To upgrade, simply
+fix the method signature of your `VertexComputing` implementations. The `Memory` argument can be safely ignored to
+effect the exact same semantics as prior. However, now previous OLAP job `Memory` can be leveraged when constructing
+the next `VertexProgram` in an OLAP traversal chain.
+
Interrupting Traversals
+++++++++++++++++++++++
[25/27] incubator-tinkerpop git commit: Clean up scoping of
configuration methods for gryo message serializer.
Posted by sp...@apache.org.
Clean up scoping of configuration methods for gryo message serializer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6aa3e982
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6aa3e982
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6aa3e982
Branch: refs/heads/TINKERPOP-1308
Commit: 6aa3e98264b8633862b7e9dd646278258bc4a42e
Parents: ace64de
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri May 20 13:23:32 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu May 26 13:06:07 2016 -0400
----------------------------------------------------------------------
.../gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java | 4 ++--
.../gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6aa3e982/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
index 9198ae1..498536d 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
@@ -80,8 +80,8 @@ public abstract class AbstractGryoMessageSerializerV1d0 extends AbstractMessageS
* Called from the {@link #configure(Map, Map)} method right before the call to create the builder. Sub-classes
* can choose to alter the builder or completely replace it.
*/
- public GryoMapper.Builder configureBuilder(final GryoMapper.Builder builder, final Map<String, Object> config,
- final Map<String, Graph> graphs) {
+ GryoMapper.Builder configureBuilder(final GryoMapper.Builder builder, final Map<String, Object> config,
+ final Map<String, Graph> graphs) {
return builder;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6aa3e982/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
index dab7c58..14f716d 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
@@ -63,7 +63,7 @@ public class GryoLiteMessageSerializerV1d0 extends AbstractGryoMessageSerializer
}
@Override
- public GryoMapper.Builder configureBuilder(final GryoMapper.Builder builder, final Map<String, Object> config,
+ GryoMapper.Builder configureBuilder(final GryoMapper.Builder builder, final Map<String, Object> config,
final Map<String, Graph> graphs) {
return overrideWithLite(builder);
}
[08/27] incubator-tinkerpop git commit: added traversal-based vote
strength test to PeerPressureTest. Cleaned up internal classes of
TraversalVertexProgram a bit for better organization. Generalized Phase enum
so it can be used by other computer-based cl
Posted by sp...@apache.org.
added traversal-based vote strength test to PeerPressureTest. Cleaned up internal classes of TraversalVertexProgram a bit for better organization. Generalized Phase enum so it can be used by other computer-based classes. added comments to TraveralVertexProgram.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5bcf0b01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5bcf0b01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5bcf0b01
Branch: refs/heads/TINKERPOP-1308
Commit: 5bcf0b01c7bfc77e9eea8fdb7b2c4c54a4fc0000
Parents: b100f03
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu May 19 05:23:36 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu May 19 05:23:36 2016 -0600
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 17 +-
.../gremlin/process/computer/ProgramPhase.java | 40 ++++
.../peerpressure/PeerPressureVertexProgram.java | 21 +-
.../traversal/MemoryTraversalSideEffects.java | 27 +--
.../computer/traversal/SingleMessenger.java | 49 -----
.../traversal/TraversalVertexProgram.java | 18 +-
.../computer/traversal/TraverserExecutor.java | 216 ------------------
.../computer/traversal/WorkerExecutor.java | 220 +++++++++++++++++++
.../step/map/PeerPressureVertexProgramStep.java | 9 +-
.../process/computer/util/EmptyMemory.java | 58 ++++-
.../process/computer/util/SingleMessenger.java | 49 +++++
.../step/map/GroovyPeerPressureTest.groovy | 5 +
.../traversal/step/map/PeerPressureTest.java | 31 +++
.../process/traversal/step/map/ProgramTest.java | 11 +-
.../SparkStarBarrierInterceptor.java | 3 +-
15 files changed, 460 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 3394e90..4c74795 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1434,6 +1434,10 @@ vertex program includes:
The user supplied `VertexProgram` can leverage that information accordingly within their vertex program. Example uses
are provided below.
+WARNING: Developing a `VertexProgram` is for expert users. Moreover, developing one that can be used effectively within
+a traversal requires yet more expertise. This information is recommended to advanced users with a deep understanding of the
+mechanics of Gremlin OLAP (<<graphcomputer,`GraphComputer`>>).
+
[source,java]
----
private TraverserSet<Object> haltedTraversers;
@@ -1447,7 +1451,7 @@ public void loadState(final Graph graph, final Configuration configuration) {
// if master-traversal traversers may be propagated, create a memory compute key
this.memoryComputeKeys.add(MemoryComputeKey.of(TraversalVertexProgram.HALTED_TRAVERSERS, Operator.addAll, false, false));
// returns an empty traverser set if there are no halted traversers
- this.haltedTraversers = TraversalVertexProgram.getHaltedTraversers(configuration);
+ this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
}
public void storeState(final Configuration configuration) {
@@ -1457,22 +1461,21 @@ public void storeState(final Configuration configuration) {
}
public void setup(final Memory memory) {
- if(null != this.haltedTraversers) {
+ if(!this.haltedTraversers.isEmpty()) {
// do what you like with the halted master traversal traversers
}
// once used, no need to keep that information around (master)
- if(null != this.haltedTraversers)
- this.haltedTraversers.clear()
+ this.haltedTraversers = null;
}
public void execute(final Vertex vertex, final Messenger messenger, final Memory memory) {
// once used, no need to keep that information around (workers)
if(null != this.haltedTraversers)
- this.haltedTraversers.clear()
+ this.haltedTraversers = null;
if(vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent()) {
// haltedTraversers in execute() represent worker-traversal traversers
// for example, from a traversal of the form g.V().out().program(...)
- TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS) :
+ TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS);
// create a new halted traverser set that can be used by the next OLAP job in the chain
// these are worker-traversers that are distributed throughout the graph
TraverserSet<Object> newHaltedTraversers = new TraverserSet<>();
@@ -1480,7 +1483,7 @@ public void execute(final Vertex vertex, final Messenger messenger, final Memory
newHaltedTraversers.add(traverser.split(traverser.get().toString(), this.programStep));
});
vertex.property(VertexProperty.Cardinality.single, TraversalVertexProgram.HALTED_TRAVERSERS, newHaltedTraversers);
- // it is possible to create master-traversers that are localized to the master traversal (thread)
+ // it is possible to create master-traversers that are localized to the master traversal (this is how results are ultimately delivered back to the user)
memory.add(TraversalVertexProgram.HALTED_TRAVERSERS,
new TraverserSet<>(this.traversal().get().getTraverserGenerator().generate("an example", this.programStep, 1l)));
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ProgramPhase.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ProgramPhase.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ProgramPhase.java
new file mode 100644
index 0000000..ce39505
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/ProgramPhase.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.computer;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public enum ProgramPhase {
+
+ SETUP,
+ WORKER_ITERATION_START,
+ EXECUTE,
+ WORKER_ITERATION_END,
+ TERMINATE;
+
+ public boolean masterState() {
+ return this == SETUP || this == TERMINATE;
+ }
+
+ public boolean workerState() {
+ return !this.masterState();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
index 8834882..56de255 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/clustering/peerpressure/PeerPressureVertexProgram.java
@@ -34,6 +34,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.MapHelper;
import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.ScriptTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -62,6 +63,7 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
public static final String CLUSTER = "gremlin.peerPressureVertexProgram.cluster";
private static final String VOTE_STRENGTH = "gremlin.peerPressureVertexProgram.voteStrength";
+ private static final String INITIAL_VOTE_STRENGTH_TRAVERSAL = "gremlin.pageRankVertexProgram.initialVoteStrengthTraversal";
private static final String PROPERTY = "gremlin.peerPressureVertexProgram.property";
private static final String MAX_ITERATIONS = "gremlin.peerPressureVertexProgram.maxIterations";
private static final String DISTRIBUTE_VOTE = "gremlin.peerPressureVertexProgram.distributeVote";
@@ -69,6 +71,7 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
private static final String VOTE_TO_HALT = "gremlin.peerPressureVertexProgram.voteToHalt";
private PureTraversal<Vertex, Edge> edgeTraversal = null;
+ private PureTraversal<Vertex, ? extends Number> initialVoteStrengthTraversal = null;
private int maxIterations = 30;
private boolean distributeVote = false;
private String property = CLUSTER;
@@ -81,6 +84,8 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
@Override
public void loadState(final Graph graph, final Configuration configuration) {
+ if (configuration.containsKey(INITIAL_VOTE_STRENGTH_TRAVERSAL))
+ this.initialVoteStrengthTraversal = PureTraversal.loadState(configuration, INITIAL_VOTE_STRENGTH_TRAVERSAL, graph);
if (configuration.containsKey(EDGE_TRAVERSAL)) {
this.edgeTraversal = PureTraversal.loadState(configuration, EDGE_TRAVERSAL, graph);
this.voteScope = MessageScope.Local.of(() -> this.edgeTraversal.get().clone());
@@ -99,6 +104,8 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
configuration.setProperty(DISTRIBUTE_VOTE, this.distributeVote);
if (null != this.edgeTraversal)
this.edgeTraversal.storeState(configuration, EDGE_TRAVERSAL);
+ if (null != this.initialVoteStrengthTraversal)
+ this.initialVoteStrengthTraversal.storeState(configuration, INITIAL_VOTE_STRENGTH_TRAVERSAL);
}
@Override
@@ -137,14 +144,19 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
if (this.distributeVote) {
messenger.sendMessage(this.countScope, Pair.with('c', 1.0d));
} else {
- double voteStrength = 1.0d;
+ double voteStrength = (null == this.initialVoteStrengthTraversal ?
+ 1.0d :
+ TraversalUtil.apply(vertex, this.initialVoteStrengthTraversal.get()).doubleValue());
vertex.property(VertexProperty.Cardinality.single, this.property, vertex.id());
vertex.property(VertexProperty.Cardinality.single, VOTE_STRENGTH, voteStrength);
messenger.sendMessage(this.voteScope, new Pair<>((Serializable) vertex.id(), voteStrength));
memory.add(VOTE_TO_HALT, false);
}
} else if (1 == memory.getIteration() && this.distributeVote) {
- double voteStrength = 1.0d / IteratorUtils.reduce(IteratorUtils.map(messenger.receiveMessages(), Pair::getValue1), 0.0d, (a, b) -> a + b);
+ double voteStrength = (null == this.initialVoteStrengthTraversal ?
+ 1.0d :
+ TraversalUtil.apply(vertex, this.initialVoteStrengthTraversal.get()).doubleValue()) /
+ IteratorUtils.reduce(IteratorUtils.map(messenger.receiveMessages(), Pair::getValue1), 0.0d, (a, b) -> a + b);
vertex.property(VertexProperty.Cardinality.single, this.property, vertex.id());
vertex.property(VertexProperty.Cardinality.single, VOTE_STRENGTH, voteStrength);
messenger.sendMessage(this.voteScope, new Pair<>((Serializable) vertex.id(), voteStrength));
@@ -227,6 +239,11 @@ public class PeerPressureVertexProgram extends StaticVertexProgram<Pair<Serializ
return this;
}
+ public Builder initialVoteStrength(final Traversal.Admin<Vertex, ? extends Number> initialVoteStrengthTraversal) {
+ PureTraversal.storeState(this.configuration, INITIAL_VOTE_STRENGTH_TRAVERSAL, initialVoteStrengthTraversal);
+ return this;
+ }
+
/**
* @deprecated As of release 3.2.0, replaced by {@link PeerPressureVertexProgram.Builder#edges(Traversal.Admin)}
*/
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
index 23d33f1..bf9f8c0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MemoryTraversalSideEffects.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.process.computer.traversal;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
+import org.apache.tinkerpop.gremlin.process.computer.ProgramPhase;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
@@ -38,23 +39,7 @@ public final class MemoryTraversalSideEffects implements TraversalSideEffects {
private TraversalSideEffects sideEffects;
private Memory memory;
- private State state;
-
- public enum State {
- SETUP,
- WORKER_ITERATION_START,
- EXECUTE,
- WORKER_ITERATION_END,
- TERMINATE;
-
- public boolean masterState() {
- return this == SETUP || this == TERMINATE;
- }
-
- public boolean workerState() {
- return !this.masterState();
- }
- }
+ private ProgramPhase phase;
private MemoryTraversalSideEffects() {
// for serialization
@@ -93,7 +78,7 @@ public final class MemoryTraversalSideEffects implements TraversalSideEffects {
@Override
public void add(final String key, final Object value) {
- if (this.state.workerState())
+ if (this.phase.workerState())
this.memory.add(key, value);
else
this.memory.set(key, this.sideEffects.getReducer(key).apply(this.memory.get(key), value));
@@ -168,20 +153,20 @@ public final class MemoryTraversalSideEffects implements TraversalSideEffects {
}
public void storeSideEffectsInMemory() {
- if (this.state.workerState())
+ if (this.phase.workerState())
this.sideEffects.forEach(this.memory::add);
else
this.sideEffects.forEach(this.memory::set);
}
- public static void setMemorySideEffects(final Traversal.Admin<?, ?> traversal, final Memory memory, final State state) {
+ public static void setMemorySideEffects(final Traversal.Admin<?, ?> traversal, final Memory memory, final ProgramPhase phase) {
final TraversalSideEffects sideEffects = traversal.getSideEffects();
if (!(sideEffects instanceof MemoryTraversalSideEffects)) {
traversal.setSideEffects(new MemoryTraversalSideEffects(sideEffects));
}
final MemoryTraversalSideEffects memoryTraversalSideEffects = ((MemoryTraversalSideEffects) traversal.getSideEffects());
memoryTraversalSideEffects.memory = memory;
- memoryTraversalSideEffects.state = state;
+ memoryTraversalSideEffects.phase = phase;
}
public static Set<MemoryComputeKey> getMemoryComputeKeys(final Traversal.Admin<?, ?> traversal) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/SingleMessenger.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/SingleMessenger.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/SingleMessenger.java
deleted file mode 100644
index 26ed8a4..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/SingleMessenger.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.process.computer.traversal;
-
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SingleMessenger<M> implements Messenger<M> {
-
- private final Messenger<M> baseMessenger;
- private final M message;
-
- public SingleMessenger(final Messenger<M> baseMessenger, final M message) {
- this.baseMessenger = baseMessenger;
- this.message = message;
- }
-
- @Override
- public Iterator<M> receiveMessages() {
- return IteratorUtils.of(this.message);
- }
-
- @Override
- public void sendMessage(final MessageScope messageScope, final M message) {
- this.baseMessenger.sendMessage(messageScope, message);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index c7e7ef9..266426f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -27,12 +27,14 @@ import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.computer.ProgramPhase;
import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ComputerResultStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
+import org.apache.tinkerpop.gremlin.process.computer.util.SingleMessenger;
import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramHelper;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
@@ -195,13 +197,13 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
@Override
public void setup(final Memory memory) {
// memory is local
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.SETUP);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.SETUP);
((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).storeSideEffectsInMemory();
memory.set(VOTE_TO_HALT, true);
memory.set(MUTATED_MEMORY_KEYS, new HashSet<>());
memory.set(COMPLETED_BARRIERS, new HashSet<>());
- // if halted traversers are being sent from a previous VertexProgram in an OLAP chain (non-distributed traversers), get them into the stream
- if (null != this.haltedTraversers && !this.haltedTraversers.isEmpty()) {
+ // if halted traversers are being sent from a previous VertexProgram in an OLAP chain (non-distributed traversers), get them into the flow
+ if (!this.haltedTraversers.isEmpty()) {
final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
IteratorUtils.removeOnNext(this.haltedTraversers.iterator()).forEachRemaining(traverser -> {
traverser.setStepId(this.traversal.get().getStartStep().getId());
@@ -231,7 +233,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
if (null != this.haltedTraversers)
this.haltedTraversers = null;
// memory is distributed
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.EXECUTE);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.EXECUTE);
// if a barrier was completed in another worker, it is also completed here (ensure distributed barriers are synchronized)
final Set<String> completedBarriers = memory.get(COMPLETED_BARRIERS);
for (final String stepId : completedBarriers) {
@@ -245,11 +247,13 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
//////////////////
if (memory.isInitialIteration()) { // ITERATION 1
final TraverserSet<Object> activeTraversers = new TraverserSet<>();
+ // if halted traversers are being sent from a previous VertexProgram in an OLAP chain (distributed traversers), get them into the flow
IteratorUtils.removeOnNext(haltedTraversers.iterator()).forEachRemaining(traverser -> {
traverser.setStepId(this.traversal.get().getStartStep().getId());
activeTraversers.add(traverser);
});
assert haltedTraversers.isEmpty();
+ // for g.V()/E()
if (this.traversal.get().getStartStep() instanceof GraphStep) {
final GraphStep<Element, Element> graphStep = (GraphStep<Element, Element>) this.traversal.get().getStartStep();
graphStep.reset();
@@ -270,9 +274,9 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
activeTraversers.add((Traverser.Admin) traverser);
});
}
- memory.add(VOTE_TO_HALT, activeTraversers.isEmpty() || TraverserExecutor.execute(vertex, new SingleMessenger<>(messenger, activeTraversers), this.traversalMatrix, memory, this.returnHaltedTraversers));
+ memory.add(VOTE_TO_HALT, activeTraversers.isEmpty() || WorkerExecutor.execute(vertex, new SingleMessenger<>(messenger, activeTraversers), this.traversalMatrix, memory, this.returnHaltedTraversers));
} else { // ITERATION 1+
- memory.add(VOTE_TO_HALT, TraverserExecutor.execute(vertex, messenger, this.traversalMatrix, memory, this.returnHaltedTraversers));
+ memory.add(VOTE_TO_HALT, WorkerExecutor.execute(vertex, messenger, this.traversalMatrix, memory, this.returnHaltedTraversers));
}
if (this.returnHaltedTraversers || haltedTraversers.isEmpty())
vertex.<TraverserSet>property(HALTED_TRAVERSERS).remove();
@@ -281,7 +285,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
@Override
public boolean terminate(final Memory memory) {
// memory is local
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.TERMINATE);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.TERMINATE);
final boolean voteToHalt = memory.<Boolean>get(VOTE_TO_HALT);
memory.set(VOTE_TO_HALT, true);
memory.set(ACTIVE_TRAVERSERS, new TraverserSet<>());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
deleted file mode 100644
index c5f9bb5..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.process.computer.traversal;
-
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
-import org.apache.tinkerpop.gremlin.process.traversal.step.Bypassing;
-import org.apache.tinkerpop.gremlin.process.traversal.step.LocalBarrier;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMatrix;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Property;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.Attachable;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class TraverserExecutor {
-
- public static boolean execute(final Vertex vertex, final Messenger<TraverserSet<Object>> messenger, final TraversalMatrix<?, ?> traversalMatrix, final Memory memory, final boolean returnHaltedTraversers) {
-
- final TraversalSideEffects traversalSideEffects = traversalMatrix.getTraversal().getSideEffects();
- final AtomicBoolean voteToHalt = new AtomicBoolean(true);
- final TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS);
- final TraverserSet<Object> activeTraversers = new TraverserSet<>();
- final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
-
- ////////////////////////////////
- // GENERATE LOCAL TRAVERSERS //
- ///////////////////////////////
-
- // these are traversers that are going from OLTP to OLAP
- final TraverserSet<Object> maybeActiveTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
- final Iterator<Traverser.Admin<Object>> iterator = maybeActiveTraversers.iterator();
- while (iterator.hasNext()) {
- final Traverser.Admin<Object> traverser = iterator.next();
- if (vertex.equals(TraverserExecutor.getHostingVertex(traverser.get()))) {
- // iterator.remove(); ConcurrentModificationException
- traverser.attach(Attachable.Method.get(vertex));
- traverser.setSideEffects(traversalSideEffects);
- toProcessTraversers.add(traverser);
- }
- }
- // these are traversers that exist from from a local barrier
- vertex.<TraverserSet<Object>>property(TraversalVertexProgram.ACTIVE_TRAVERSERS).ifPresent(previousActiveTraversers -> {
- IteratorUtils.removeOnNext(previousActiveTraversers.iterator()).forEachRemaining(traverser -> {
- traverser.attach(Attachable.Method.get(vertex));
- traverser.setSideEffects(traversalSideEffects);
- toProcessTraversers.add(traverser);
- });
- vertex.property(TraversalVertexProgram.ACTIVE_TRAVERSERS).remove();
- });
- // these are traversers that have been messaged to the vertex
- final Iterator<TraverserSet<Object>> messages = messenger.receiveMessages();
- while (messages.hasNext()) {
- final Iterator<Traverser.Admin<Object>> traversers = messages.next().iterator();
- while (traversers.hasNext()) {
- final Traverser.Admin<Object> traverser = traversers.next();
- traversers.remove();
- if (traverser.isHalted()) {
- if (returnHaltedTraversers)
- memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
- else
- haltedTraversers.add(traverser);
- } else {
- traverser.attach(Attachable.Method.get(vertex));
- traverser.setSideEffects(traversalSideEffects);
- toProcessTraversers.add(traverser);
- }
- }
- }
-
- ///////////////////////////////
- // PROCESS LOCAL TRAVERSERS //
- //////////////////////////////
-
- // while there are still local traversers, process them until they leave the vertex or halt (i.e. isHalted()).
- while (!toProcessTraversers.isEmpty()) {
- // process local traversers and if alive, repeat, else halt.
- Step<Object, Object> previousStep = EmptyStep.instance();
- Iterator<Traverser.Admin<Object>> traversers = toProcessTraversers.iterator();
- while (traversers.hasNext()) {
- final Traverser.Admin<Object> traverser = traversers.next();
- traversers.remove();
- final Step<Object, Object> currentStep = traversalMatrix.getStepById(traverser.getStepId());
- if (!currentStep.getId().equals(previousStep.getId()) && !(previousStep instanceof EmptyStep))
- TraverserExecutor.drainStep(vertex, previousStep, activeTraversers, haltedTraversers, memory, returnHaltedTraversers);
- currentStep.addStart(traverser);
- previousStep = currentStep;
- }
- TraverserExecutor.drainStep(vertex, previousStep, activeTraversers, haltedTraversers, memory, returnHaltedTraversers);
- assert toProcessTraversers.isEmpty();
- // process all the local objects and send messages or store locally again
- if (!activeTraversers.isEmpty()) {
- traversers = activeTraversers.iterator();
- while (traversers.hasNext()) {
- final Traverser.Admin<Object> traverser = traversers.next();
- traversers.remove();
- if (traverser.get() instanceof Element || traverser.get() instanceof Property) { // GRAPH OBJECT
- // if the element is remote, then message, else store it locally for re-processing
- final Vertex hostingVertex = TraverserExecutor.getHostingVertex(traverser.get());
- if (!vertex.equals(hostingVertex)) { // necessary for path access
- voteToHalt.set(false);
- traverser.detach();
- messenger.sendMessage(MessageScope.Global.of(hostingVertex), new TraverserSet<>(traverser));
- } else {
- if (traverser.get() instanceof Attachable) // necessary for path access to local object
- traverser.attach(Attachable.Method.get(vertex));
- toProcessTraversers.add(traverser);
- }
- } else // STANDARD OBJECT
- toProcessTraversers.add(traverser);
- }
- assert activeTraversers.isEmpty();
- }
- }
- return voteToHalt.get();
- }
-
- private static void drainStep(final Vertex vertex, final Step<Object, Object> step, final TraverserSet<Object> activeTraversers, final TraverserSet<Object> haltedTraversers, final Memory memory, final boolean returnHaltedTraversers) {
- if (step instanceof Barrier) {
- if (step instanceof Bypassing)
- ((Bypassing) step).setBypass(true);
- if (step instanceof LocalBarrier) {
- final LocalBarrier<Object> barrier = (LocalBarrier<Object>) step;
- final TraverserSet<Object> traverserSet = vertex.<TraverserSet<Object>>property(TraversalVertexProgram.ACTIVE_TRAVERSERS).orElse(new TraverserSet<>());
- vertex.property(TraversalVertexProgram.ACTIVE_TRAVERSERS, traverserSet);
- while (barrier.hasNextBarrier()) {
- final TraverserSet<Object> barrierSet = barrier.nextBarrier();
- IteratorUtils.removeOnNext(barrierSet.iterator()).forEachRemaining(traverser -> {
- traverser.addLabels(step.getLabels()); // this might need to be generalized for working with global barriers too
- if (traverser.isHalted() &&
- (returnHaltedTraversers ||
- (!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) ||
- getHostingVertex(traverser.get()).equals(vertex))) {
- traverser.detach();
- if (returnHaltedTraversers)
- memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
- else
- haltedTraversers.add(traverser);
- } else {
- traverser.detach();
- traverserSet.add(traverser);
- }
- });
- }
- memory.add(TraversalVertexProgram.MUTATED_MEMORY_KEYS, new HashSet<>(Collections.singleton(step.getId())));
- } else {
- final Barrier barrier = (Barrier) step;
- while (barrier.hasNextBarrier()) {
- memory.add(step.getId(), barrier.nextBarrier());
- }
- memory.add(TraversalVertexProgram.MUTATED_MEMORY_KEYS, new HashSet<>(Collections.singleton(step.getId())));
- }
- } else { // LOCAL PROCESSING
- step.forEachRemaining(traverser -> {
- if (traverser.isHalted() &&
- (returnHaltedTraversers ||
- (!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) ||
- getHostingVertex(traverser.get()).equals(vertex))) {
- traverser.detach();
- if (returnHaltedTraversers)
- memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
- else
- haltedTraversers.add(traverser);
- } else {
- activeTraversers.add(traverser);
- }
- });
- }
- }
-
- private static Vertex getHostingVertex(final Object object) {
- Object obj = object;
- while (true) {
- if (obj instanceof Vertex)
- return (Vertex) obj;
- else if (obj instanceof Edge)
- return ((Edge) obj).outVertex();
- else if (obj instanceof Property)
- obj = ((Property) obj).element();
- else
- throw new IllegalStateException("The host of the object is unknown: " + obj.toString() + ':' + obj.getClass().getCanonicalName());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
new file mode 100644
index 0000000..c4bd659
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.traversal;
+
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Bypassing;
+import org.apache.tinkerpop.gremlin.process.traversal.step.LocalBarrier;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMatrix;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Element;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.Attachable;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+final class WorkerExecutor {
+
+ private WorkerExecutor() {
+
+ }
+
+ protected static boolean execute(final Vertex vertex, final Messenger<TraverserSet<Object>> messenger, final TraversalMatrix<?, ?> traversalMatrix, final Memory memory, final boolean returnHaltedTraversers) {
+
+ final TraversalSideEffects traversalSideEffects = traversalMatrix.getTraversal().getSideEffects();
+ final AtomicBoolean voteToHalt = new AtomicBoolean(true);
+ final TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS);
+ final TraverserSet<Object> activeTraversers = new TraverserSet<>();
+ final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
+
+ ////////////////////////////////
+ // GENERATE LOCAL TRAVERSERS //
+ ///////////////////////////////
+
+ // these are traversers that are going from OLTP to OLAP
+ final TraverserSet<Object> maybeActiveTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
+ final Iterator<Traverser.Admin<Object>> iterator = maybeActiveTraversers.iterator();
+ while (iterator.hasNext()) {
+ final Traverser.Admin<Object> traverser = iterator.next();
+ if (vertex.equals(WorkerExecutor.getHostingVertex(traverser.get()))) {
+ // iterator.remove(); ConcurrentModificationException
+ traverser.attach(Attachable.Method.get(vertex));
+ traverser.setSideEffects(traversalSideEffects);
+ toProcessTraversers.add(traverser);
+ }
+ }
+ // these are traversers that exist from from a local barrier
+ vertex.<TraverserSet<Object>>property(TraversalVertexProgram.ACTIVE_TRAVERSERS).ifPresent(previousActiveTraversers -> {
+ IteratorUtils.removeOnNext(previousActiveTraversers.iterator()).forEachRemaining(traverser -> {
+ traverser.attach(Attachable.Method.get(vertex));
+ traverser.setSideEffects(traversalSideEffects);
+ toProcessTraversers.add(traverser);
+ });
+ vertex.property(TraversalVertexProgram.ACTIVE_TRAVERSERS).remove();
+ });
+ // these are traversers that have been messaged to the vertex
+ final Iterator<TraverserSet<Object>> messages = messenger.receiveMessages();
+ while (messages.hasNext()) {
+ final Iterator<Traverser.Admin<Object>> traversers = messages.next().iterator();
+ while (traversers.hasNext()) {
+ final Traverser.Admin<Object> traverser = traversers.next();
+ traversers.remove();
+ if (traverser.isHalted()) {
+ if (returnHaltedTraversers)
+ memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
+ else
+ haltedTraversers.add(traverser);
+ } else {
+ traverser.attach(Attachable.Method.get(vertex));
+ traverser.setSideEffects(traversalSideEffects);
+ toProcessTraversers.add(traverser);
+ }
+ }
+ }
+
+ ///////////////////////////////
+ // PROCESS LOCAL TRAVERSERS //
+ //////////////////////////////
+
+ // while there are still local traversers, process them until they leave the vertex or halt (i.e. isHalted()).
+ while (!toProcessTraversers.isEmpty()) {
+ // process local traversers and if alive, repeat, else halt.
+ Step<Object, Object> previousStep = EmptyStep.instance();
+ Iterator<Traverser.Admin<Object>> traversers = toProcessTraversers.iterator();
+ while (traversers.hasNext()) {
+ final Traverser.Admin<Object> traverser = traversers.next();
+ traversers.remove();
+ final Step<Object, Object> currentStep = traversalMatrix.getStepById(traverser.getStepId());
+ if (!currentStep.getId().equals(previousStep.getId()) && !(previousStep instanceof EmptyStep))
+ WorkerExecutor.drainStep(vertex, previousStep, activeTraversers, haltedTraversers, memory, returnHaltedTraversers);
+ currentStep.addStart(traverser);
+ previousStep = currentStep;
+ }
+ WorkerExecutor.drainStep(vertex, previousStep, activeTraversers, haltedTraversers, memory, returnHaltedTraversers);
+ assert toProcessTraversers.isEmpty();
+ // process all the local objects and send messages or store locally again
+ if (!activeTraversers.isEmpty()) {
+ traversers = activeTraversers.iterator();
+ while (traversers.hasNext()) {
+ final Traverser.Admin<Object> traverser = traversers.next();
+ traversers.remove();
+ if (traverser.get() instanceof Element || traverser.get() instanceof Property) { // GRAPH OBJECT
+ // if the element is remote, then message, else store it locally for re-processing
+ final Vertex hostingVertex = WorkerExecutor.getHostingVertex(traverser.get());
+ if (!vertex.equals(hostingVertex)) { // necessary for path access
+ voteToHalt.set(false);
+ traverser.detach();
+ messenger.sendMessage(MessageScope.Global.of(hostingVertex), new TraverserSet<>(traverser));
+ } else {
+ if (traverser.get() instanceof Attachable) // necessary for path access to local object
+ traverser.attach(Attachable.Method.get(vertex));
+ toProcessTraversers.add(traverser);
+ }
+ } else // STANDARD OBJECT
+ toProcessTraversers.add(traverser);
+ }
+ assert activeTraversers.isEmpty();
+ }
+ }
+ return voteToHalt.get();
+ }
+
+ private static void drainStep(final Vertex vertex, final Step<Object, Object> step, final TraverserSet<Object> activeTraversers, final TraverserSet<Object> haltedTraversers, final Memory memory, final boolean returnHaltedTraversers) {
+ if (step instanceof Barrier) {
+ if (step instanceof Bypassing)
+ ((Bypassing) step).setBypass(true);
+ if (step instanceof LocalBarrier) {
+ final LocalBarrier<Object> barrier = (LocalBarrier<Object>) step;
+ final TraverserSet<Object> traverserSet = vertex.<TraverserSet<Object>>property(TraversalVertexProgram.ACTIVE_TRAVERSERS).orElse(new TraverserSet<>());
+ vertex.property(TraversalVertexProgram.ACTIVE_TRAVERSERS, traverserSet);
+ while (barrier.hasNextBarrier()) {
+ final TraverserSet<Object> barrierSet = barrier.nextBarrier();
+ IteratorUtils.removeOnNext(barrierSet.iterator()).forEachRemaining(traverser -> {
+ traverser.addLabels(step.getLabels()); // this might need to be generalized for working with global barriers too
+ if (traverser.isHalted() &&
+ (returnHaltedTraversers ||
+ (!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) ||
+ getHostingVertex(traverser.get()).equals(vertex))) {
+ traverser.detach();
+ if (returnHaltedTraversers)
+ memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
+ else
+ haltedTraversers.add(traverser);
+ } else {
+ traverser.detach();
+ traverserSet.add(traverser);
+ }
+ });
+ }
+ memory.add(TraversalVertexProgram.MUTATED_MEMORY_KEYS, new HashSet<>(Collections.singleton(step.getId())));
+ } else {
+ final Barrier barrier = (Barrier) step;
+ while (barrier.hasNextBarrier()) {
+ memory.add(step.getId(), barrier.nextBarrier());
+ }
+ memory.add(TraversalVertexProgram.MUTATED_MEMORY_KEYS, new HashSet<>(Collections.singleton(step.getId())));
+ }
+ } else { // LOCAL PROCESSING
+ step.forEachRemaining(traverser -> {
+ if (traverser.isHalted() &&
+ (returnHaltedTraversers ||
+ (!(traverser.get() instanceof Element) && !(traverser.get() instanceof Property)) ||
+ getHostingVertex(traverser.get()).equals(vertex))) {
+ traverser.detach();
+ if (returnHaltedTraversers)
+ memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(traverser));
+ else
+ haltedTraversers.add(traverser);
+ } else {
+ activeTraversers.add(traverser);
+ }
+ });
+ }
+ }
+
+ private static Vertex getHostingVertex(final Object object) {
+ Object obj = object;
+ while (true) {
+ if (obj instanceof Vertex)
+ return (Vertex) obj;
+ else if (obj instanceof Edge)
+ return ((Edge) obj).outVertex();
+ else if (obj instanceof Property)
+ obj = ((Property) obj).element();
+ else
+ throw new IllegalStateException("The host of the object is unknown: " + obj.toString() + ':' + obj.getClass().getCanonicalName());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
index 7bd726e..0ea5112 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/PeerPressureVertexProgramStep.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.process.computer.traversal.step.map;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.lambda.HaltedTraversersCountTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
@@ -87,11 +88,13 @@ public final class PeerPressureVertexProgramStep extends VertexProgramStep imple
public PeerPressureVertexProgram generateProgram(final Graph graph, final Memory memory) {
final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
- return PeerPressureVertexProgram.build()
+ final PeerPressureVertexProgram.Builder builder = PeerPressureVertexProgram.build()
.property(this.clusterProperty)
.maxIterations(this.times)
- .edges(detachedTraversal)
- .create(graph);
+ .edges(detachedTraversal);
+ if (this.previousTraversalVertexProgram())
+ builder.initialVoteStrength(new HaltedTraversersCountTraversal());
+ return builder.create(graph);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
index 72b1bbf..f513ee6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/EmptyMemory.java
@@ -21,19 +21,71 @@ package org.apache.tinkerpop.gremlin.process.computer.util;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import java.util.Collections;
+import java.util.Set;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class EmptyMemory {
+public final class EmptyMemory implements Memory.Admin {
- private static final Memory INSTANCE = new ImmutableMemory(new MapMemory());
+ private static final EmptyMemory INSTANCE = new EmptyMemory();
private EmptyMemory() {
}
- public static Memory instance() {
+ public static EmptyMemory instance() {
return INSTANCE;
}
+ @Override
+ public void setIteration(final int iteration) {
+ throw Memory.Exceptions.memoryIsCurrentlyImmutable();
+ }
+
+ @Override
+ public void setRuntime(final long runtime) {
+ throw Memory.Exceptions.memoryIsCurrentlyImmutable();
+ }
+
+ @Override
+ public Memory asImmutable() {
+ return this;
+ }
+
+ @Override
+ public Set<String> keys() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public <R> R get(final String key) throws IllegalArgumentException {
+ throw Memory.Exceptions.memoryDoesNotExist(key);
+ }
+
+ @Override
+ public void set(final String key, final Object value) throws IllegalArgumentException, IllegalStateException {
+ throw Memory.Exceptions.memoryIsCurrentlyImmutable();
+ }
+
+ @Override
+ public void add(final String key, final Object value) throws IllegalArgumentException, IllegalStateException {
+ throw Memory.Exceptions.memoryIsCurrentlyImmutable();
+ }
+
+ @Override
+ public int getIteration() {
+ return 0;
+ }
+
+ @Override
+ public long getRuntime() {
+ return 0;
+ }
+
+ @Override
+ public boolean exists(final String key) {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/SingleMessenger.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/SingleMessenger.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/SingleMessenger.java
new file mode 100644
index 0000000..63ee282
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/SingleMessenger.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.util;
+
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SingleMessenger<M> implements Messenger<M> {
+
+ private final Messenger<M> baseMessenger;
+ private final M message;
+
+ public SingleMessenger(final Messenger<M> baseMessenger, final M message) {
+ this.baseMessenger = baseMessenger;
+ this.message = message;
+ }
+
+ @Override
+ public Iterator<M> receiveMessages() {
+ return IteratorUtils.of(this.message);
+ }
+
+ @Override
+ public void sendMessage(final MessageScope messageScope, final M message) {
+ this.baseMessenger.sendMessage(messageScope, message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPeerPressureTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPeerPressureTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPeerPressureTest.groovy
index eb1ad67..6ec0750 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPeerPressureTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyPeerPressureTest.groovy
@@ -39,5 +39,10 @@ public abstract class GroovyPeerPressureTest {
public Traversal<Vertex, Map<Object, Number>> get_g_V_peerPressure_byXclusterX_byXoutEXknowsXX_pageRankX1X_byXrankX_byXoutEXknowsXX_timesX2X_group_byXclusterX_byXrank_sumX_limitX100X() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.peerPressure.by('cluster').by(outE('knows')).pageRank(1.0).by('rank').by(outE('knows')).times(1).group.by('cluster').by(values('rank').sum).limit(100)")
}
+
+ @Override
+ public Traversal<Vertex, Map<String, List<Object>>> get_g_V_hasXname_rippleX_inXcreatedX_peerPressure_byXoutEX_byXclusterX_repeatXunionXidentity__bothX_timesX2X_dedup_valueMapXname_clusterX() {
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.has('name', 'ripple').in('created').peerPressure.by(outE()).by('cluster').repeat(union(identity(), both())).times(2).dedup.valueMap('name', 'cluster')")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PeerPressureTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PeerPressureTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PeerPressureTest.java
index 21c4f43..5a2477c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PeerPressureTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/PeerPressureTest.java
@@ -27,7 +27,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
import static org.junit.Assert.assertEquals;
@@ -43,6 +47,7 @@ public abstract class PeerPressureTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Map<Object, Number>> get_g_V_peerPressure_byXclusterX_byXoutEXknowsXX_pageRankX1X_byXrankX_byXoutEXknowsXX_timesX2X_group_byXclusterX_byXrank_sumX_limitX100X();
+ public abstract Traversal<Vertex, Map<String, List<Object>>> get_g_V_hasXname_rippleX_inXcreatedX_peerPressure_byXoutEX_byXclusterX_repeatXunionXidentity__bothX_timesX2X_dedup_valueMapXname_clusterX();
@Test
@LoadGraphWith(MODERN)
@@ -72,6 +77,27 @@ public abstract class PeerPressureTest extends AbstractGremlinProcessTest {
assertEquals(0.0d, (double) map.get(convertToVertexId("peter")), 0.001d);
}
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_hasXname_rippleX_inXcreatedX_peerPressure_byXoutEX_byXclusterX_repeatXunionXidentity__bothX_timesX2X_dedup_valueMapXname_clusterX() {
+ final Traversal<Vertex, Map<String, List<Object>>> traversal = get_g_V_hasXname_rippleX_inXcreatedX_peerPressure_byXoutEX_byXclusterX_repeatXunionXidentity__bothX_timesX2X_dedup_valueMapXname_clusterX();
+ printTraversalForm(traversal);
+ final List<Map<String, List<Object>>> results = traversal.toList();
+ assertEquals(6, results.size());
+ final Map<String, Object> clusters = new HashMap<>();
+ results.forEach(m -> clusters.put((String) m.get("name").get(0), m.get("cluster").get(0)));
+ assertEquals(2, results.get(0).size());
+ assertEquals(6, clusters.size());
+ assertEquals(clusters.get("josh"), clusters.get("ripple"));
+ assertEquals(clusters.get("josh"), clusters.get("lop"));
+ final Set<Object> ids = new HashSet<>(clusters.values());
+ assertEquals(4, ids.size());
+ assertTrue(ids.contains(convertToVertexId("marko")));
+ assertTrue(ids.contains(convertToVertexId("vadas")));
+ assertTrue(ids.contains(convertToVertexId("josh")));
+ assertTrue(ids.contains(convertToVertexId("peter")));
+ }
+
public static class Traversals extends PeerPressureTest {
@@ -84,5 +110,10 @@ public abstract class PeerPressureTest extends AbstractGremlinProcessTest {
public Traversal<Vertex, Map<Object, Number>> get_g_V_peerPressure_byXclusterX_byXoutEXknowsXX_pageRankX1X_byXrankX_byXoutEXknowsXX_timesX2X_group_byXclusterX_byXrank_sumX_limitX100X() {
return g.V().peerPressure().by("cluster").by(__.outE("knows")).pageRank(1.0d).by("rank").by(__.outE("knows")).times(1).<Object, Number>group().by("cluster").by(__.values("rank").sum()).limit(100);
}
+
+ @Override
+ public Traversal<Vertex, Map<String, List<Object>>> get_g_V_hasXname_rippleX_inXcreatedX_peerPressure_byXoutEX_byXclusterX_repeatXunionXidentity__bothX_timesX2X_dedup_valueMapXname_clusterX() {
+ return g.V().has("name", "ripple").in("created").peerPressure().by(__.outE()).by("cluster").repeat(__.union(__.identity(), __.both())).times(2).dedup().valueMap("name", "cluster");
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
index 8b2a293..5f548da 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProgramTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.computer.ProgramPhase;
import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram;
@@ -187,7 +188,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
@Override
public void setup(final Memory memory) {
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.SETUP);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.SETUP);
final Map<Vertex, Long> map = (Map<Vertex, Long>) this.haltedTraversers.iterator().next().get();
assertEquals(2, map.size());
assertTrue(map.values().contains(3l));
@@ -205,7 +206,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
public void execute(final Vertex vertex, final Messenger messenger, final Memory memory) {
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
final TraverserGenerator generator = this.traversal.get().getTraverserGenerator();
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.EXECUTE);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.EXECUTE);
this.checkSideEffects();
final TraverserSet<Vertex> activeTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
if (vertex.label().equals("software")) {
@@ -233,7 +234,7 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
@Override
public boolean terminate(final Memory memory) {
final TraverserGenerator generator = this.traversal.get().getTraverserGenerator();
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.TERMINATE);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.TERMINATE);
checkSideEffects();
if (memory.isInitialIteration()) {
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
@@ -254,14 +255,14 @@ public abstract class ProgramTest extends AbstractGremlinProcessTest {
assertNotNull(this.haltedTraversers);
this.haltedTraversers.clear();
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.WORKER_ITERATION_START);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.WORKER_ITERATION_START);
checkSideEffects();
}
@Override
public void workerIterationEnd(final Memory memory) {
assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
- MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, MemoryTraversalSideEffects.State.WORKER_ITERATION_END);
+ MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.WORKER_ITERATION_END);
checkSideEffects();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5bcf0b01/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
index 6e35cf8..768d10a 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
@@ -22,6 +22,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.ProgramPhase;
import org.apache.tinkerpop.gremlin.process.computer.traversal.MemoryTraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.NumberHelper;
@@ -75,7 +76,7 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
traversal.applyStrategies(); // compile
boolean identityTraversal = traversal.getSteps().isEmpty(); // if the traversal is empty, just return the vertex (fast)
///////////////////////////////
- MemoryTraversalSideEffects.setMemorySideEffects(traversal, memory, MemoryTraversalSideEffects.State.EXECUTE); // any intermediate sideEffect steps are backed by SparkMemory
+ MemoryTraversalSideEffects.setMemorySideEffects(traversal, memory, ProgramPhase.EXECUTE); // any intermediate sideEffect steps are backed by SparkMemory
memory.setInExecute(true);
final JavaRDD<Traverser.Admin<Object>> nextRDD = inputRDD.values()
.filter(vertexWritable -> ElementHelper.idExists(vertexWritable.get().id(), graphStepIds)) // ensure vertex ids are in V(x)
[21/27] incubator-tinkerpop git commit: updated CHANGELOG.
Posted by sp...@apache.org.
updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6113c928
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6113c928
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6113c928
Branch: refs/heads/TINKERPOP-1308
Commit: 6113c9283661b253356fafc753b2d8b30d9555a5
Parents: 4318b6c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed May 25 08:47:42 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed May 25 08:47:42 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6113c928/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 4ba3c80..171b772 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.2.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Fixed a `NullPointerException` bug around nested `group()`-steps in OLAP.
* Fixed a severe bug around halted traversers in a multi-job OLAP traversal chain.
* Ensure a separation of `GraphComputer` and `VertexProgram` configurations in `SparkGraphComputer` and `GiraphGraphComputer`.
* `PeerPressureVertexProgram` now supports dynamic initial vote strength calculations.
[15/27] incubator-tinkerpop git commit: some code clean
up/organization and added a more complex group() nesting that requires an
OLAP walk and an OLTP walk. Also found a bunch of HadoopGraph tests that were
needlessly being Opt_Out'd -- fixed.
Posted by sp...@apache.org.
some code clean up/organization and added a more complex group() nesting that requires an OLAP walk and an OLTP walk. Also found a bunch of HadoopGraph tests that were needlessly being Opt_Out'd -- fixed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/085be946
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/085be946
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/085be946
Branch: refs/heads/TINKERPOP-1308
Commit: 085be94691357db5615ed4281a978b686bce6898
Parents: e087123
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue May 24 01:23:14 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue May 24 01:23:14 2016 -0600
----------------------------------------------------------------------
.../process/traversal/step/map/GroupStep.java | 59 ++++++---
.../step/sideEffect/GroovyGroupTest.groovy | 5 +
.../traversal/step/sideEffect/GroupTest.java | 35 ++++++
.../gremlin/hadoop/structure/HadoopGraph.java | 124 ++++++++++++++-----
4 files changed, 172 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/085be946/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
index 7a93796..77e39bb 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
@@ -40,6 +40,9 @@ import org.apache.tinkerpop.gremlin.util.function.HashMapSupplier;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.javatuples.Pair;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
@@ -85,11 +88,11 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
public Map<K, V> projectTraverser(final Traverser.Admin<S> traverser) {
final Map<K, V> map = new HashMap<>(1);
if (null == this.preTraversal) {
- map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverser.split());
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverser);
} else {
final TraverserSet traverserSet = new TraverserSet<>();
this.preTraversal.reset();
- this.preTraversal.addStart(traverser.split());
+ this.preTraversal.addStart(traverser);
this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverserSet);
}
@@ -150,6 +153,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
public static final class GroupBiOperator<K, V> implements BinaryOperator<Map<K, V>>, Serializable {
+ // size limit before Barrier.processAllStarts() to lazy reduce
private static final int SIZE_LIMIT = 1000;
private Traversal.Admin<?, V> valueTraversal;
@@ -173,6 +177,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
if (null == objectA) {
objectA = objectB;
} else {
+ // TRAVERSER
if (objectA instanceof Traverser.Admin) {
if (objectB instanceof Traverser.Admin) {
final TraverserSet set = new TraverserSet();
@@ -186,29 +191,28 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
this.valueTraversal.reset();
((Step) this.barrierStep).addStarts(set.iterator());
objectA = this.barrierStep.nextBarrier();
- } else {
+ } else
objectA = objectB;
- }
} else if (objectB instanceof Pair) {
final TraverserSet set = (TraverserSet) ((Pair) objectB).getValue0();
set.add((Traverser.Admin) objectA);
- if (set.size() > SIZE_LIMIT) {
+ if (set.size() > SIZE_LIMIT) { // barrier step can never be null -- no need to check
this.valueTraversal.reset();
((Step) this.barrierStep).addStarts(set.iterator());
this.barrierStep.addBarrier(((Pair) objectB).getValue1());
objectA = this.barrierStep.nextBarrier();
- } else {
+ } else
objectA = Pair.with(set, ((Pair) objectB).getValue1());
- }
- } else {
+ } else
objectA = Pair.with(new TraverserSet((Traverser.Admin) objectA), objectB);
- }
+ // TRAVERSER SET
} else if (objectA instanceof TraverserSet) {
if (objectB instanceof Traverser.Admin) {
- ((TraverserSet) objectA).add((Traverser.Admin) objectB);
- if (null != this.barrierStep && ((TraverserSet) objectA).size() > SIZE_LIMIT) {
+ final TraverserSet set = (TraverserSet) objectA;
+ set.add((Traverser.Admin) objectB);
+ if (null != this.barrierStep && set.size() > SIZE_LIMIT) {
this.valueTraversal.reset();
- ((Step) this.barrierStep).addStarts(((TraverserSet) objectA).iterator());
+ ((Step) this.barrierStep).addStarts(set.iterator());
objectA = this.barrierStep.nextBarrier();
}
} else if (objectB instanceof TraverserSet) {
@@ -222,24 +226,30 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
} else if (objectB instanceof Pair) {
final TraverserSet set = (TraverserSet) objectA;
set.addAll((TraverserSet) ((Pair) objectB).getValue0());
- if (set.size() > SIZE_LIMIT) {
+ if (set.size() > SIZE_LIMIT) { // barrier step can never be null -- no need to check
this.valueTraversal.reset();
((Step) this.barrierStep).addStarts(set.iterator());
this.barrierStep.addBarrier(((Pair) objectB).getValue1());
objectA = this.barrierStep.nextBarrier();
- } else {
+ } else
objectA = Pair.with(set, ((Pair) objectB).getValue1());
- }
- } else {
+ } else
objectA = Pair.with(objectA, objectB);
- }
+ // TRAVERSER SET + BARRIER
} else if (objectA instanceof Pair) {
if (objectB instanceof Traverser.Admin) {
- ((TraverserSet) ((Pair) objectA).getValue0()).add((Traverser.Admin) objectB);
+ final TraverserSet set = ((TraverserSet) ((Pair) objectA).getValue0());
+ set.add((Traverser.Admin) objectB);
+ if (set.size() > SIZE_LIMIT) { // barrier step can never be null -- no need to check
+ this.valueTraversal.reset();
+ ((Step) this.barrierStep).addStarts(set.iterator());
+ this.barrierStep.addBarrier(((Pair) objectA).getValue1());
+ objectA = this.barrierStep.nextBarrier();
+ }
} else if (objectB instanceof TraverserSet) {
final TraverserSet set = (TraverserSet) ((Pair) objectA).getValue0();
set.addAll((TraverserSet) objectB);
- if (null != this.barrierStep && set.size() > SIZE_LIMIT) {
+ if (set.size() > SIZE_LIMIT) { // barrier step can never be null -- no need to check
this.valueTraversal.reset();
((Step) this.barrierStep).addStarts(set.iterator());
this.barrierStep.addBarrier(((Pair) objectA).getValue1());
@@ -259,6 +269,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
((Step) this.barrierStep).addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
objectA = this.barrierStep.nextBarrier();
}
+ // BARRIER
} else {
if (objectB instanceof Traverser.Admin) {
objectA = Pair.with(new TraverserSet<>((Traverser.Admin) objectB), objectA);
@@ -282,6 +293,16 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
}
return mapA;
}
+
+ // necessary to control Java Serialization to ensure proper clearing of internal traverser data
+ private void writeObject(final ObjectOutputStream outputStream) throws IOException {
+ outputStream.writeObject(this.valueTraversal.clone());
+ }
+
+ private void readObject(final ObjectInputStream inputStream) throws IOException, ClassNotFoundException {
+ this.valueTraversal = (Traversal.Admin<?, V>) inputStream.readObject();
+ this.barrierStep = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, this.valueTraversal).orElse(null);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/085be946/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
index 156b350..13802b8 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
@@ -113,5 +113,10 @@ public abstract class GroovyGroupTest {
public Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.group().by(bothE().count).by(group().by(label))")
}
+
+ @Override
+ public Traversal<Vertex, Map<String, Map<String, Number>>> get_g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX() {
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.out('followedBy').group.by('songType').by(bothE().group.by(label).by(values('weight').sum))")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/085be946/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
index d4c4d74..356eb58 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupTest.java
@@ -86,6 +86,8 @@ public abstract class GroupTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX();
+ public abstract Traversal<Vertex, Map<String, Map<String, Number>>> get_g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX();
+
@Test
@LoadGraphWith(MODERN)
public void g_V_group_byXnameX() {
@@ -393,6 +395,34 @@ public abstract class GroupTest extends AbstractGremlinProcessTest {
assertTrue(list.contains(convertToVertex(graph, "josh")));
}
+ @Test
+ @LoadGraphWith(GRATEFUL)
+ public void g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX() {
+ final Traversal<Vertex, Map<String, Map<String, Number>>> traversal = get_g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX();
+ final Map<String, Map<String, Number>> map = traversal.next();
+ assertFalse(traversal.hasNext());
+ assertEquals(3, map.size());
+ assertTrue(map.containsKey(""));
+ assertTrue(map.containsKey("original"));
+ assertTrue(map.containsKey("cover"));
+ //
+ Map<String, Number> subMap = map.get("");
+ assertEquals(1, subMap.size());
+ assertEquals(179350, subMap.get("followedBy").intValue());
+ //
+ subMap = map.get("original");
+ assertEquals(3, subMap.size());
+ assertEquals(2185613, subMap.get("followedBy").intValue());
+ assertEquals(0, subMap.get("writtenBy").intValue());
+ assertEquals(0, subMap.get("sungBy").intValue());
+ //
+ subMap = map.get("cover");
+ assertEquals(3, subMap.size());
+ assertEquals(777982, subMap.get("followedBy").intValue());
+ assertEquals(0, subMap.get("writtenBy").intValue());
+ assertEquals(0, subMap.get("sungBy").intValue());
+ }
+
public static class Traversals extends GroupTest {
@Override
@@ -479,5 +509,10 @@ public abstract class GroupTest extends AbstractGremlinProcessTest {
public Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX() {
return g.V().<Long, Map<String, List<Vertex>>>group().by(__.bothE().count()).by(__.group().by(T.label));
}
+
+ @Override
+ public Traversal<Vertex, Map<String, Map<String, Number>>> get_g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX() {
+ return g.V().out("followedBy").<String, Map<String, Number>>group().by("songType").by(__.bothE().group().by(T.label).by(__.values("weight").sum()));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/085be946/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index bf2f08b..d643cd4 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -53,97 +53,113 @@ import java.util.stream.Stream;
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_knows_b__c_knows_bX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-//computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
+ computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-//computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
+ computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-// computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
+ computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_knows_b__c_knows_bX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-//computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
+ computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-//computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
+ computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
-//computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
+ computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals",
method = "g_V_both_both_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals",
method = "g_V_repeatXoutX_timesX3X_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals",
method = "g_V_repeatXoutX_timesX8X_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals",
method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals",
method = "g_V_both_both_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals",
method = "g_V_repeatXoutX_timesX3X_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals",
method = "g_V_repeatXoutX_timesX8X_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals",
method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals",
method = "grateful_V_out_out_profile",
@@ -162,20 +178,64 @@ import java.util.stream.Stream;
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
+ method = "g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
+ method = "g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTestV3d0",
method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTestV3d0",
method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals",
+ method = "g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals",
+ method = "g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals",
+ method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals",
+ method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTestV3d0$Traversals",
+ method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTestV3d0$Traversals",
+ method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX",
+ reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
+ computers = {"ALL"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
method = "shouldStartAndEndWorkersForVertexProgramAndMapReduce",
[14/27] incubator-tinkerpop git commit: came up with a nifty trick
and now OLD and NEW group() are much closer in time. 432ms vs 456ms. Given
that the solution is no longer an NPE in nested groups in OLAP -- this minor
time hit is worth it. :). Running i
Posted by sp...@apache.org.
came up with a nifty trick and now OLD and NEW group() are much closer in time. 432ms vs 456ms. Given that the solution is no longer an NPE in nested groups in OLAP -- this minor time hit is worth it. :). Running integration tests over night.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e087123c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e087123c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e087123c
Branch: refs/heads/TINKERPOP-1308
Commit: e087123ca9bf513a555b3666a201905df43d8f7c
Parents: c890ceb
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon May 23 17:26:35 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon May 23 17:26:35 2016 -0600
----------------------------------------------------------------------
.../process/traversal/step/map/GroupStep.java | 168 ++++++++++++-------
.../step/sideEffect/GroupSideEffectStep.java | 20 ++-
.../step/sideEffect/GroovyGroupTest.groovy | 2 +-
3 files changed, 116 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e087123c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
index b430d8f..7a93796 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
@@ -42,7 +42,6 @@ import org.javatuples.Pair;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -62,7 +61,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
public GroupStep(final Traversal.Admin traversal) {
super(traversal);
this.valueTraversal = this.integrateChild(__.fold().asAdmin());
- this.preTraversal = this.integrateChild(splitOnBarrierStep(this.valueTraversal).get(0));
+ this.preTraversal = this.integrateChild(generatePreTraversal(this.valueTraversal));
this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal));
this.setSeedSupplier(HashMapSupplier.instance());
}
@@ -74,7 +73,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
this.state = 'v';
} else if ('v' == this.state) {
this.valueTraversal = this.integrateChild(convertValueTraversal(kvTraversal));
- this.preTraversal = this.integrateChild(splitOnBarrierStep(this.valueTraversal).get(0));
+ this.preTraversal = this.integrateChild(generatePreTraversal(this.valueTraversal));
this.setReducingBiOperator(new GroupBiOperator<>(this.valueTraversal));
this.state = 'x';
} else {
@@ -85,11 +84,15 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
@Override
public Map<K, V> projectTraverser(final Traverser.Admin<S> traverser) {
final Map<K, V> map = new HashMap<>(1);
- final TraverserSet traverserSet = new TraverserSet<>();
- this.preTraversal.reset();
- this.preTraversal.addStart(traverser.split());
- this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
- map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverserSet);
+ if (null == this.preTraversal) {
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverser.split());
+ } else {
+ final TraverserSet traverserSet = new TraverserSet<>();
+ this.preTraversal.reset();
+ this.preTraversal.addStart(traverser.split());
+ this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverserSet);
+ }
return map;
}
@@ -100,7 +103,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
@Override
public List<Traversal.Admin<?, ?>> getLocalChildren() {
- final List<Traversal.Admin<?, ?>> children = new ArrayList<>(4);
+ final List<Traversal.Admin<?, ?>> children = new ArrayList<>(2);
if (null != this.keyTraversal)
children.add((Traversal.Admin) this.keyTraversal);
children.add(this.valueTraversal);
@@ -118,7 +121,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
if (null != this.keyTraversal)
clone.keyTraversal = this.keyTraversal.clone();
clone.valueTraversal = this.valueTraversal.clone();
- clone.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(clone.valueTraversal).get(0));
+ clone.preTraversal = this.integrateChild(GroupStep.generatePreTraversal(clone.valueTraversal));
return clone;
}
@@ -147,12 +150,14 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
public static final class GroupBiOperator<K, V> implements BinaryOperator<Map<K, V>>, Serializable {
+ private static final int SIZE_LIMIT = 1000;
+
private Traversal.Admin<?, V> valueTraversal;
- private ReducingBarrierStep reducingBarrierStep = null;
+ private Barrier barrierStep;
public GroupBiOperator(final Traversal.Admin<?, V> valueTraversal) {
this.valueTraversal = valueTraversal.clone();
- this.reducingBarrierStep = TraversalHelper.getFirstStepOfAssignableClass(ReducingBarrierStep.class, this.valueTraversal).orElse(null);
+ this.barrierStep = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, this.valueTraversal).orElse(null);
}
public GroupBiOperator() {
@@ -168,23 +173,60 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
if (null == objectA) {
objectA = objectB;
} else {
- if (objectA instanceof TraverserSet) {
- if (objectB instanceof TraverserSet) {
+ if (objectA instanceof Traverser.Admin) {
+ if (objectB instanceof Traverser.Admin) {
+ final TraverserSet set = new TraverserSet();
+ set.add((Traverser.Admin) objectA);
+ set.add((Traverser.Admin) objectB);
+ objectA = set;
+ } else if (objectB instanceof TraverserSet) {
+ final TraverserSet set = (TraverserSet) objectB;
+ set.add((Traverser.Admin) objectA);
+ if (null != this.barrierStep && set.size() > SIZE_LIMIT) {
+ this.valueTraversal.reset();
+ ((Step) this.barrierStep).addStarts(set.iterator());
+ objectA = this.barrierStep.nextBarrier();
+ } else {
+ objectA = objectB;
+ }
+ } else if (objectB instanceof Pair) {
+ final TraverserSet set = (TraverserSet) ((Pair) objectB).getValue0();
+ set.add((Traverser.Admin) objectA);
+ if (set.size() > SIZE_LIMIT) {
+ this.valueTraversal.reset();
+ ((Step) this.barrierStep).addStarts(set.iterator());
+ this.barrierStep.addBarrier(((Pair) objectB).getValue1());
+ objectA = this.barrierStep.nextBarrier();
+ } else {
+ objectA = Pair.with(set, ((Pair) objectB).getValue1());
+ }
+ } else {
+ objectA = Pair.with(new TraverserSet((Traverser.Admin) objectA), objectB);
+ }
+ } else if (objectA instanceof TraverserSet) {
+ if (objectB instanceof Traverser.Admin) {
+ ((TraverserSet) objectA).add((Traverser.Admin) objectB);
+ if (null != this.barrierStep && ((TraverserSet) objectA).size() > SIZE_LIMIT) {
+ this.valueTraversal.reset();
+ ((Step) this.barrierStep).addStarts(((TraverserSet) objectA).iterator());
+ objectA = this.barrierStep.nextBarrier();
+ }
+ } else if (objectB instanceof TraverserSet) {
final TraverserSet set = (TraverserSet) objectA;
set.addAll((TraverserSet) objectB);
- if (null != this.reducingBarrierStep && set.size() > 1000) {
+ if (null != this.barrierStep && set.size() > SIZE_LIMIT) {
this.valueTraversal.reset();
- this.reducingBarrierStep.addStarts(set.iterator());
- objectA = this.reducingBarrierStep.nextBarrier();
+ ((Step) this.barrierStep).addStarts(set.iterator());
+ objectA = this.barrierStep.nextBarrier();
}
} else if (objectB instanceof Pair) {
final TraverserSet set = (TraverserSet) objectA;
set.addAll((TraverserSet) ((Pair) objectB).getValue0());
- if (set.size() > 1000) {
+ if (set.size() > SIZE_LIMIT) {
this.valueTraversal.reset();
- this.reducingBarrierStep.addStarts(set.iterator());
- this.reducingBarrierStep.addBarrier(((Pair) objectB).getValue1());
- objectA = this.reducingBarrierStep.nextBarrier();
+ ((Step) this.barrierStep).addStarts(set.iterator());
+ this.barrierStep.addBarrier(((Pair) objectB).getValue1());
+ objectA = this.barrierStep.nextBarrier();
} else {
objectA = Pair.with(set, ((Pair) objectB).getValue1());
}
@@ -192,50 +234,53 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
objectA = Pair.with(objectA, objectB);
}
} else if (objectA instanceof Pair) {
- if (objectB instanceof TraverserSet) {
+ if (objectB instanceof Traverser.Admin) {
+ ((TraverserSet) ((Pair) objectA).getValue0()).add((Traverser.Admin) objectB);
+ } else if (objectB instanceof TraverserSet) {
final TraverserSet set = (TraverserSet) ((Pair) objectA).getValue0();
set.addAll((TraverserSet) objectB);
- if (null != this.reducingBarrierStep &&set.size() > 1000) {
+ if (null != this.barrierStep && set.size() > SIZE_LIMIT) {
this.valueTraversal.reset();
- this.reducingBarrierStep.addStarts(set.iterator());
- this.reducingBarrierStep.addBarrier(((Pair) objectA).getValue1());
- objectA = this.reducingBarrierStep.nextBarrier();
+ ((Step) this.barrierStep).addStarts(set.iterator());
+ this.barrierStep.addBarrier(((Pair) objectA).getValue1());
+ objectA = this.barrierStep.nextBarrier();
}
} else if (objectB instanceof Pair) {
this.valueTraversal.reset();
- this.reducingBarrierStep.addBarrier(((Pair) objectA).getValue1());
- this.reducingBarrierStep.addBarrier(((Pair) objectB).getValue1());
- this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
- this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectB).getValue0()).iterator());
- objectA = this.reducingBarrierStep.nextBarrier();
+ this.barrierStep.addBarrier(((Pair) objectA).getValue1());
+ this.barrierStep.addBarrier(((Pair) objectB).getValue1());
+ ((Step) this.barrierStep).addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
+ ((Step) this.barrierStep).addStarts(((TraverserSet) ((Pair) objectB).getValue0()).iterator());
+ objectA = this.barrierStep.nextBarrier();
} else {
this.valueTraversal.reset();
- this.reducingBarrierStep.addBarrier(((Pair) objectA).getValue1());
- this.reducingBarrierStep.addBarrier(objectB);
- this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
- objectA = this.reducingBarrierStep.nextBarrier();
+ this.barrierStep.addBarrier(((Pair) objectA).getValue1());
+ this.barrierStep.addBarrier(objectB);
+ ((Step) this.barrierStep).addStarts(((TraverserSet) ((Pair) objectA).getValue0()).iterator());
+ objectA = this.barrierStep.nextBarrier();
}
} else {
- if (objectB instanceof TraverserSet) {
+ if (objectB instanceof Traverser.Admin) {
+ objectA = Pair.with(new TraverserSet<>((Traverser.Admin) objectB), objectA);
+ } else if (objectB instanceof TraverserSet) {
objectA = Pair.with(objectB, objectA);
} else if (objectB instanceof Pair) {
this.valueTraversal.reset();
- this.reducingBarrierStep.addBarrier(objectA);
- this.reducingBarrierStep.addBarrier(((Pair) objectB).getValue1());
- this.reducingBarrierStep.addStarts(((TraverserSet) ((Pair) objectB).getValue0()).iterator());
- objectA = this.reducingBarrierStep.nextBarrier();
+ this.barrierStep.addBarrier(objectA);
+ this.barrierStep.addBarrier(((Pair) objectB).getValue1());
+ ((Step) this.barrierStep).addStarts(((TraverserSet) ((Pair) objectB).getValue0()).iterator());
+ objectA = this.barrierStep.nextBarrier();
} else {
this.valueTraversal.reset();
- this.reducingBarrierStep.addBarrier(objectA);
- this.reducingBarrierStep.addBarrier(objectB);
- objectA = this.reducingBarrierStep.nextBarrier();
+ this.barrierStep.addBarrier(objectA);
+ this.barrierStep.addBarrier(objectB);
+ objectA = this.barrierStep.nextBarrier();
}
}
}
mapA.put(key, (V) objectA);
}
return mapA;
-
}
}
@@ -253,28 +298,21 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
}
}
- public static List<Traversal.Admin<?, ?>> splitOnBarrierStep(final Traversal.Admin<?, ?> valueTraversal) {
- if (TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversal).isPresent()) {
- final Traversal.Admin<?, ?> first = __.identity().asAdmin();
- final Traversal.Admin<?, ?> second = __.identity().asAdmin();
- boolean onSecond = false;
- for (final Step step : valueTraversal.getSteps()) {
- if (step instanceof Barrier)
- onSecond = true;
- if (onSecond)
- second.addStep(step.clone());
- else
- first.addStep(step.clone());
- }
- return Arrays.asList(first, second);
- } else {
- return Arrays.asList(valueTraversal.clone(), __.identity().asAdmin());
+ public static Traversal.Admin<?, ?> generatePreTraversal(final Traversal.Admin<?, ?> valueTraversal) {
+ if (!TraversalHelper.hasStepOfAssignableClass(Barrier.class, valueTraversal))
+ return valueTraversal;
+ final Traversal.Admin<?, ?> first = __.identity().asAdmin();
+ for (final Step step : valueTraversal.getSteps()) {
+ if (step instanceof Barrier)
+ break;
+ first.addStep(step.clone());
}
+ return first.getSteps().size() == 1 ? null : first;
}
public static <K, V> Map<K, V> doFinalReduction(final Map<K, Object> map, final Traversal.Admin<?, V> valueTraversal) {
final Map<K, V> reducedMap = new HashMap<>(map.size());
- final ReducingBarrierStep reducingBarrierStep = TraversalHelper.getFirstStepOfAssignableClass(ReducingBarrierStep.class, valueTraversal).orElse(null);
+ final Barrier reducingBarrierStep = TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, valueTraversal).orElse(null);
IteratorUtils.removeOnNext(map.entrySet().iterator()).forEachRemaining(entry -> {
valueTraversal.reset();
if (null == reducingBarrierStep) {
@@ -282,10 +320,12 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
((TraverserSet<V>) entry.getValue()).iterator().next().get() :
(V) entry.getValue());
} else {
- if (entry.getValue() instanceof TraverserSet)
- reducingBarrierStep.addStarts(((TraverserSet) entry.getValue()).iterator());
+ if (entry.getValue() instanceof Traverser.Admin)
+ ((Step) reducingBarrierStep).addStart((Traverser.Admin) entry.getValue());
+ else if (entry.getValue() instanceof TraverserSet)
+ ((Step) reducingBarrierStep).addStarts(((TraverserSet) entry.getValue()).iterator());
else if (entry.getValue() instanceof Pair) {
- reducingBarrierStep.addStarts(((TraverserSet) (((Pair) entry.getValue()).getValue0())).iterator());
+ ((Step) reducingBarrierStep).addStarts(((TraverserSet) (((Pair) entry.getValue()).getValue0())).iterator());
reducingBarrierStep.addBarrier((((Pair) entry.getValue()).getValue1()));
} else
reducingBarrierStep.addBarrier(entry.getValue());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e087123c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
index 4fc4ffa..b5deb02 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
@@ -53,7 +53,7 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
super(traversal);
this.sideEffectKey = sideEffectKey;
this.valueTraversal = this.integrateChild(__.fold().asAdmin());
- this.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(this.valueTraversal).get(0));
+ this.preTraversal = this.integrateChild(GroupStep.generatePreTraversal(this.valueTraversal));
this.getTraversal().getSideEffects().registerIfAbsent(this.sideEffectKey, HashMapSupplier.instance(), new GroupStep.GroupBiOperator<>(this.valueTraversal));
}
@@ -64,7 +64,7 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
this.state = 'v';
} else if ('v' == this.state) {
this.valueTraversal = this.integrateChild(GroupStep.convertValueTraversal(kvTraversal));
- this.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(this.valueTraversal).get(0));
+ this.preTraversal = this.integrateChild(GroupStep.generatePreTraversal(this.valueTraversal));
this.getTraversal().getSideEffects().register(this.sideEffectKey, null, new GroupStep.GroupBiOperator<>(this.valueTraversal));
this.state = 'x';
} else {
@@ -75,13 +75,15 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
@Override
protected void sideEffect(final Traverser.Admin<S> traverser) {
final Map<K, V> map = new HashMap<>(1);
- final TraverserSet midTraversers = new TraverserSet<>();
- this.preTraversal.reset();
- this.preTraversal.addStart(traverser.split());
- while (this.preTraversal.hasNext()) {
- midTraversers.add(this.preTraversal.getEndStep().next());
+ if (null == this.preTraversal) {
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverser.split());
+ } else {
+ final TraverserSet traverserSet = new TraverserSet<>();
+ this.preTraversal.reset();
+ this.preTraversal.addStart(traverser.split());
+ this.preTraversal.getEndStep().forEachRemaining(traverserSet::add);
+ map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) traverserSet);
}
- map.put(TraversalUtil.applyNullable(traverser, this.keyTraversal), (V) midTraversers);
this.getTraversal().getSideEffects().add(this.sideEffectKey, map);
}
@@ -115,7 +117,7 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
if (null != this.keyTraversal)
clone.keyTraversal = this.keyTraversal.clone();
clone.valueTraversal = this.valueTraversal.clone();
- clone.preTraversal = this.integrateChild(GroupStep.splitOnBarrierStep(clone.valueTraversal).get(0));
+ clone.preTraversal = this.integrateChild(GroupStep.generatePreTraversal(clone.valueTraversal));
return clone;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e087123c/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
index ddfb94a..156b350 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyGroupTest.groovy
@@ -111,7 +111,7 @@ public abstract class GroovyGroupTest {
@Override
public Traversal<Vertex, Map<Long, Map<String, List<Vertex>>>> get_g_V_group_byXbothE_countX_byXgroup_byXlabelXX() {
- new ScriptTraversal<>(g, "gremlin-groovy", "g.V.group().by(bothE().count).by(group.by(label))")
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.group().by(bothE().count).by(group().by(label))")
}
}
}
[17/27] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/tp31'
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/tp31'
Conflicts:
gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/192d5ec7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/192d5ec7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/192d5ec7
Branch: refs/heads/TINKERPOP-1308
Commit: 192d5ec7e9770d1c40d0d54605f8028f44ee5d1b
Parents: 44d40f6 55a509f
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 24 16:20:18 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 24 16:20:18 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../traversal/step/sideEffect/SubgraphStep.java | 19 ++++++----
.../step/sideEffect/GroovySubgraphTest.groovy | 5 +++
.../traversal/step/sideEffect/SubgraphTest.java | 40 ++++++++++++++++++--
4 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/192d5ec7/CHANGELOG.asciidoc
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/192d5ec7/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
----------------------------------------------------------------------
diff --cc gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
index 12adbca,c3ae74a..7e3765a
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovySubgraphTest.groovy
@@@ -39,7 -39,12 +39,12 @@@ public abstract class GroovySubgraphTes
@Override
public Traversal<Vertex, String> get_g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup(
final Graph subgraph) {
- TraversalScriptHelper.compute("g.withSideEffect('sg') { subgraph }.V.repeat(__.bothE('created').subgraph('sg').outV).times(5).name.dedup", g, "subgraph", subgraph)
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.withSideEffect('sg') { subgraph }.V.repeat(__.bothE('created').subgraph('sg').outV).times(5).name.dedup", "subgraph", subgraph)
}
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_withSideEffectXsgX_V_hasXname_danielX_outE_subgraphXsgX_inV(final Graph subgraph) {
- TraversalScriptHelper.compute("g.withSideEffect('sg') { subgraph }.V.has('name','daniel').outE.subgraph('sg').inV", g, "subgraph", subgraph);
++ new ScriptTraversal<>(g, "gremlin-groovy", "g.withSideEffect('sg') { subgraph }.V.has('name','daniel').outE.subgraph('sg').inV", "subgraph", subgraph)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/192d5ec7/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
----------------------------------------------------------------------
diff --cc gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
index b01bcb5,dc55685..9f2a662
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/SubgraphTest.java
@@@ -95,8 -100,30 +100,30 @@@ public abstract class SubgraphTest exte
final Traversal<Vertex, String> traversal = get_g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup(subgraph);
printTraversalForm(traversal);
checkResults(Arrays.asList("marko", "josh", "peter"), traversal);
- final Graph subGraph = traversal.asAdmin().getSideEffects().<Graph>get("sg");
- assertVertexEdgeCounts(subGraph, 5, 4);
- subgraph = traversal.asAdmin().getSideEffects().<Graph>get("sg").get();
++ subgraph = traversal.asAdmin().getSideEffects().<Graph>get("sg");
+ assertVertexEdgeCounts(subgraph, 5, 4);
+
+ graphProvider.clear(subgraph, config);
+ }
+
+ @Test
+ @LoadGraphWith(CREW)
+ @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = FEATURE_ADD_VERTICES)
+ @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = FEATURE_ADD_EDGES)
+ @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = FEATURE_USER_SUPPLIED_IDS)
+ @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = FEATURE_USER_SUPPLIED_IDS)
+ public void g_withSideEffectXsgX_V_hasXname_danielXout_capXsgX() throws Exception {
+ final Configuration config = graphProvider.newGraphConfiguration("subgraph", this.getClass(), name.getMethodName(), CREW);
+ graphProvider.clear(config);
+ final Graph subgraph = graphProvider.openTestGraph(config);
+ /////
+ final Traversal<Vertex, Vertex> traversal = get_g_withSideEffectXsgX_V_hasXname_danielX_outE_subgraphXsgX_inV(subgraph);
+ printTraversalForm(traversal);
+ traversal.iterate();
+ assertVertexEdgeCounts(subgraph, 3, 2);
+
+ final List<String> locations = subgraph.traversal().V().has("name", "daniel").<String>values("location").toList();
+ assertThat(locations, contains("spremberg", "kaiserslautern", "aachen"));
graphProvider.clear(subgraph, config);
}
[12/27] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/tp31'
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/tp31'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/44d40f60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/44d40f60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/44d40f60
Branch: refs/heads/TINKERPOP-1308
Commit: 44d40f60664b8b758f02bc8e04300b931b2c6622
Parents: 6ffeaf1 aa447eb
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon May 23 08:15:09 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon May 23 08:15:09 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../apache/tinkerpop/gremlin/server/op/session/Session.java | 6 +++++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/44d40f60/CHANGELOG.asciidoc
----------------------------------------------------------------------
[27/27] incubator-tinkerpop git commit: Added gryo "lite" serializer.
Posted by sp...@apache.org.
Added gryo "lite" serializer.
Serializes graph elements to "reference" rather than "detached". Altered all the configuration files that required the change. This serializer is just an option - the default configuration from the client side is to use the existing serializer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f811a8ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f811a8ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f811a8ae
Branch: refs/heads/TINKERPOP-1308
Commit: f811a8aecbfa82b1747503a88b13af71b23e3ae9
Parents: 2e03050
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri May 20 12:43:12 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu May 26 13:06:07 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +-
.../structure/io/gryo/GryoSerializers.java | 12 -
.../ser/AbstractGryoMessageSerializerV1d0.java | 311 ++++++++++
.../ser/GryoLiteMessageSerializerV1d0.java | 156 +++++
.../driver/ser/GryoMessageSerializerV1d0.java | 290 +--------
.../tinkerpop/gremlin/driver/ser/SerTokens.java | 1 +
.../gremlin/driver/ser/Serializers.java | 5 +-
.../ser/GryoLiteMessageSerializerV1d0Test.java | 589 +++++++++++++++++++
.../ser/GryoMessageSerializerV1d0Test.java | 2 +-
gremlin-server/conf/gremlin-server-classic.yaml | 1 +
.../conf/gremlin-server-modern-readonly.yaml | 1 +
gremlin-server/conf/gremlin-server-modern.yaml | 1 +
gremlin-server/conf/gremlin-server-neo4j.yaml | 1 +
.../conf/gremlin-server-rest-secure.yaml | 2 -
gremlin-server/conf/gremlin-server-secure.yaml | 1 +
gremlin-server/conf/gremlin-server-spark.yaml | 1 +
gremlin-server/conf/gremlin-server.yaml | 1 +
.../server/GremlinResultSetIntegrateTest.java | 16 +
.../remote/gremlin-server-integration.yaml | 1 +
.../server/gremlin-server-integration.yaml | 1 +
20 files changed, 1100 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index d2a8e3e..6efcdc6 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,7 +26,6 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.2.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-* `GryoMapper` allows overrides of existing serializers on calls to `addCustom` on the builder.
* Fixed a `NullPointerException` bug around nested `group()`-steps in OLAP.
* Fixed a severe bug around halted traversers in a multi-job OLAP traversal chain.
* Ensure a separation of `GraphComputer` and `VertexProgram` configurations in `SparkGraphComputer` and `GiraphGraphComputer`.
@@ -34,6 +33,8 @@ TinkerPop 3.2.1 (NOT OFFICIALLY RELEASED YET)
* Added `EmptyMemory` for ease of use when no memory exists.
* Updated `VertexComputing.generateProgram()` API to include `Memory`. (*breaking*)
* `ImmutablePath.TailPath` is now serializable like `ImmutablePath`.
+* Intoduced the `application/vnd.gremlin-v1.0+gryo-lite` serialization type to Gremlin Server which users "reference" elements rather than "detached".
+* `GryoMapper` allows overrides of existing serializers on calls to `addCustom` on the builder.
* Added a traversal style guide to the recipes cookbook.
* Fixed a bug in master-traversal traverser propagation.
* Added useful methods for custom `VertexPrograms` to be used with `program()`-step.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
index a38de03..ae99ac6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
@@ -64,9 +64,6 @@ final class GryoSerializers {
* Serializes any {@link Vertex} implementation encountered to an {@link DetachedVertex}.
*/
final static class VertexSerializer extends Serializer<Vertex> {
- public VertexSerializer() {
- }
-
@Override
public void write(final Kryo kryo, final Output output, final Vertex vertex) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertex, true));
@@ -82,9 +79,6 @@ final class GryoSerializers {
* Serializes any {@link Property} implementation encountered to an {@link DetachedProperty}.
*/
final static class PropertySerializer extends Serializer<Property> {
- public PropertySerializer() {
- }
-
@Override
public void write(final Kryo kryo, final Output output, final Property property) {
kryo.writeClassAndObject(output, property instanceof VertexProperty ? DetachedFactory.detach((VertexProperty) property, true) : DetachedFactory.detach(property));
@@ -100,9 +94,6 @@ final class GryoSerializers {
* Serializes any {@link VertexProperty} implementation encountered to an {@link DetachedVertexProperty}.
*/
final static class VertexPropertySerializer extends Serializer<VertexProperty> {
- public VertexPropertySerializer() {
- }
-
@Override
public void write(final Kryo kryo, final Output output, final VertexProperty vertexProperty) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertexProperty, true));
@@ -118,9 +109,6 @@ final class GryoSerializers {
* Serializes any {@link Path} implementation encountered to an {@link DetachedPath}.
*/
final static class PathSerializer extends Serializer<Path> {
- public PathSerializer() {
- }
-
@Override
public void write(final Kryo kryo, final Output output, final Path path) {
kryo.writeClassAndObject(output, DetachedFactory.detach(path, false));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
new file mode 100644
index 0000000..4190d96
--- /dev/null
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/AbstractGryoMessageSerializerV1d0.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.driver.ser;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.util.ReferenceCountUtil;
+import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
+import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
+import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
+import org.apache.tinkerpop.shaded.kryo.ClassResolver;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+import java.io.ByteArrayOutputStream;
+import java.lang.reflect.Method;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public abstract class AbstractGryoMessageSerializerV1d0 extends AbstractMessageSerializer {
+ private GryoMapper gryoMapper;
+ private ThreadLocal<Kryo> kryoThreadLocal = new ThreadLocal<Kryo>() {
+ @Override
+ protected Kryo initialValue() {
+ return gryoMapper.createMapper();
+ }
+ };
+
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ public static final String TOKEN_CUSTOM = "custom";
+ public static final String TOKEN_SERIALIZE_RESULT_TO_STRING = "serializeResultToString";
+ public static final String TOKEN_USE_MAPPER_FROM_GRAPH = "useMapperFromGraph";
+ public static final String TOKEN_BUFFER_SIZE = "bufferSize";
+ public static final String TOKEN_CLASS_RESOLVER_SUPPLIER = "classResolverSupplier";
+
+ protected boolean serializeToString = false;
+ private int bufferSize = 4096;
+
+ /**
+ * Creates an instance with a provided mapper configured {@link GryoMapper} instance. Note that this instance
+ * will be overridden by {@link #configure} is called.
+ */
+ public AbstractGryoMessageSerializerV1d0(final GryoMapper kryo) {
+ this.gryoMapper = kryo;
+ }
+
+ /**
+ * Called from the {@link #configure(Map, Map)} method right before the call to create the builder. Sub-classes
+ * can choose to alter the builder or completely replace it.
+ */
+ public GryoMapper.Builder beforeCreateMapper(final GryoMapper.Builder builder, final Map<String, Object> config,
+ final Map<String, Graph> graphs) {
+ return builder;
+ }
+
+ @Override
+ public final void configure(final Map<String, Object> config, final Map<String, Graph> graphs) {
+ final GryoMapper.Builder builder;
+ final Object graphToUseForMapper = config.get(TOKEN_USE_MAPPER_FROM_GRAPH);
+ if (graphToUseForMapper != null) {
+ if (null == graphs) throw new IllegalStateException(String.format(
+ "No graphs have been provided to the serializer and therefore %s is not a valid configuration", TOKEN_USE_MAPPER_FROM_GRAPH));
+
+ final Graph g = graphs.get(graphToUseForMapper.toString());
+ if (null == g) throw new IllegalStateException(String.format(
+ "There is no graph named [%s] configured to be used in the %s setting",
+ graphToUseForMapper, TOKEN_USE_MAPPER_FROM_GRAPH));
+
+ // a graph was found so use the mapper it constructs. this allows gryo to be auto-configured with any
+ // custom classes that the implementation allows for
+ builder = g.io(GryoIo.build()).mapper();
+ } else {
+ // no graph was supplied so just use the default - this will likely be the case when using a graph
+ // with no custom classes or a situation where the user needs complete control like when using two
+ // distinct implementations each with their own custom classes.
+ builder = GryoMapper.build();
+ }
+
+ addIoRegistries(config, builder);
+ addClassResolverSupplier(config, builder);
+ addCustomClasses(config, builder);
+
+ this.serializeToString = Boolean.parseBoolean(config.getOrDefault(TOKEN_SERIALIZE_RESULT_TO_STRING, "false").toString());
+ this.bufferSize = Integer.parseInt(config.getOrDefault(TOKEN_BUFFER_SIZE, "4096").toString());
+
+ this.gryoMapper = beforeCreateMapper(builder, config, graphs).create();
+ }
+
+ private void addClassResolverSupplier(final Map<String, Object> config, final GryoMapper.Builder builder) {
+ final String className = (String) config.getOrDefault(TOKEN_CLASS_RESOLVER_SUPPLIER, null);
+ if (className != null && !className.isEmpty()) {
+ try {
+ final Class<?> clazz = Class.forName(className);
+ try {
+ final Method instanceMethod = clazz.getDeclaredMethod("getInstance");
+ builder.classResolver((Supplier<ClassResolver>) instanceMethod.invoke(null));
+ } catch (Exception methodex) {
+ // tried getInstance() and that failed so try newInstance() no-arg constructor
+ builder.classResolver((Supplier<ClassResolver>) clazz.newInstance());
+ }
+ } catch (Exception ex) {
+ throw new IllegalStateException(ex);
+ }
+ }
+ }
+
+ private void addCustomClasses(final Map<String, Object> config, final GryoMapper.Builder builder) {
+ final List<String> classNameList = getListStringFromConfig(TOKEN_CUSTOM, config);
+
+ classNameList.stream().forEach(serializerDefinition -> {
+ String className;
+ Optional<String> serializerName;
+ if (serializerDefinition.contains(";")) {
+ final String[] split = serializerDefinition.split(";");
+ if (split.length != 2)
+ throw new IllegalStateException(String.format("Invalid format for serializer definition [%s] - expected <class>;<serializer-class>", serializerDefinition));
+
+ className = split[0];
+ serializerName = Optional.of(split[1]);
+ } else {
+ serializerName = Optional.empty();
+ className = serializerDefinition;
+ }
+
+ try {
+ final Class clazz = Class.forName(className);
+ final Serializer serializer;
+ if (serializerName.isPresent()) {
+ final Class serializerClazz = Class.forName(serializerName.get());
+ serializer = (Serializer) serializerClazz.newInstance();
+ builder.addCustom(clazz, kryo -> serializer);
+ } else
+ builder.addCustom(clazz);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Class could not be found", ex);
+ }
+ });
+ }
+
+ @Override
+ public ResponseMessage deserializeResponse(final ByteBuf msg) throws SerializationException {
+ try {
+ final Kryo kryo = kryoThreadLocal.get();
+ final byte[] payload = new byte[msg.capacity()];
+ msg.readBytes(payload);
+ try (final Input input = new Input(payload)) {
+ final UUID requestId = kryo.readObjectOrNull(input, UUID.class);
+ final int status = input.readShort();
+ final String statusMsg = input.readString();
+ final Map<String,Object> statusAttributes = (Map<String,Object>) kryo.readClassAndObject(input);
+ final Object result = kryo.readClassAndObject(input);
+ final Map<String,Object> metaAttributes = (Map<String,Object>) kryo.readClassAndObject(input);
+
+ return ResponseMessage.build(requestId)
+ .code(ResponseStatusCode.getFromValue(status))
+ .statusMessage(statusMsg)
+ .statusAttributes(statusAttributes)
+ .result(result)
+ .responseMetaData(metaAttributes)
+ .create();
+ }
+ } catch (Exception ex) {
+ logger.warn("Response [{}] could not be deserialized by {}.", msg, GryoMessageSerializerV1d0.class.getName());
+ throw new SerializationException(ex);
+ }
+ }
+
+ @Override
+ public ByteBuf serializeResponseAsBinary(final ResponseMessage responseMessage, final ByteBufAllocator allocator) throws SerializationException {
+ ByteBuf encodedMessage = null;
+ try {
+ final Kryo kryo = kryoThreadLocal.get();
+ try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ final Output output = new Output(baos, bufferSize);
+
+ // request id - if present
+ kryo.writeObjectOrNull(output, responseMessage.getRequestId() != null ? responseMessage.getRequestId() : null, UUID.class);
+
+ // status
+ output.writeShort(responseMessage.getStatus().getCode().getValue());
+ output.writeString(responseMessage.getStatus().getMessage());
+ kryo.writeClassAndObject(output, responseMessage.getStatus().getAttributes());
+
+ // result
+ kryo.writeClassAndObject(output, serializeToString ? serializeResultToString(responseMessage) : responseMessage.getResult().getData());
+ kryo.writeClassAndObject(output, responseMessage.getResult().getMeta());
+
+ final long size = output.total();
+ if (size > Integer.MAX_VALUE)
+ throw new SerializationException(String.format("Message size of %s exceeds allocatable space", size));
+
+ output.flush();
+ encodedMessage = allocator.buffer((int) size);
+ encodedMessage.writeBytes(baos.toByteArray());
+ }
+
+ return encodedMessage;
+ } catch (Exception ex) {
+ if (encodedMessage != null) ReferenceCountUtil.release(encodedMessage);
+
+ logger.warn("Response [{}] could not be serialized by {}.", responseMessage.toString(), GryoMessageSerializerV1d0.class.getName());
+ throw new SerializationException(ex);
+ }
+ }
+
+ @Override
+ public RequestMessage deserializeRequest(final ByteBuf msg) throws SerializationException {
+ try {
+ final Kryo kryo = kryoThreadLocal.get();
+ final byte[] payload = new byte[msg.readableBytes()];
+ msg.readBytes(payload);
+ try (final Input input = new Input(payload)) {
+ // by the time the message gets here, the mime length/type have been already read, so this part just
+ // needs to process the payload.
+ final UUID id = kryo.readObject(input, UUID.class);
+ final String processor = input.readString();
+ final String op = input.readString();
+
+ final RequestMessage.Builder builder = RequestMessage.build(op)
+ .overrideRequestId(id)
+ .processor(processor);
+
+ final Map<String, Object> args = kryo.readObject(input, HashMap.class);
+ args.forEach(builder::addArg);
+ return builder.create();
+ }
+ } catch (Exception ex) {
+ logger.warn("Request [{}] could not be deserialized by {}.", msg, GryoMessageSerializerV1d0.class.getName());
+ throw new SerializationException(ex);
+ }
+ }
+
+ @Override
+ public ByteBuf serializeRequestAsBinary(final RequestMessage requestMessage, final ByteBufAllocator allocator) throws SerializationException {
+ ByteBuf encodedMessage = null;
+ try {
+ final Kryo kryo = kryoThreadLocal.get();
+ try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ final Output output = new Output(baos, bufferSize);
+ final String mimeType = mimeTypesSupported()[0];
+ output.writeByte(mimeType.length());
+ output.write(mimeType.getBytes(UTF8));
+
+ kryo.writeObject(output, requestMessage.getRequestId());
+ output.writeString(requestMessage.getProcessor());
+ output.writeString(requestMessage.getOp());
+ kryo.writeObject(output, requestMessage.getArgs());
+
+ final long size = output.total();
+ if (size > Integer.MAX_VALUE)
+ throw new SerializationException(String.format("Message size of %s exceeds allocatable space", size));
+
+ output.flush();
+ encodedMessage = allocator.buffer((int) size);
+ encodedMessage.writeBytes(baos.toByteArray());
+ }
+
+ return encodedMessage;
+ } catch (Exception ex) {
+ if (encodedMessage != null) ReferenceCountUtil.release(encodedMessage);
+
+ logger.warn("Request [{}] could not be serialized by {}.", requestMessage.toString(), GryoMessageSerializerV1d0.class.getName());
+ throw new SerializationException(ex);
+ }
+ }
+
+ private Object serializeResultToString(final ResponseMessage msg) {
+ if (msg.getResult() == null) return "null";
+ if (msg.getResult().getData() == null) return "null";
+
+ // the IteratorHandler should return a collection so keep it as such
+ final Object o = msg.getResult().getData();
+ if (o instanceof Collection) {
+ return ((Collection) o).stream().map(d -> null == d ? "null" : d.toString()).collect(Collectors.toList());
+ } else {
+ return o.toString();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
new file mode 100644
index 0000000..a4ff38b
--- /dev/null
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.driver.ser;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Path;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdge;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceFactory;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferencePath;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceProperty;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexProperty;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+import java.util.Map;
+
+/**
+ * An alternative Gryo serializer that uses "referenced" graph elements during serialization. Referenced elements such
+ * as {@link ReferenceVertex} exclude the label and the properties associated with it and only return the identifier.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class GryoLiteMessageSerializerV1d0 extends AbstractGryoMessageSerializerV1d0 {
+
+ private static final String MIME_TYPE = SerTokens.MIME_GRYO_LITE_V1D0;
+ private static final String MIME_TYPE_STRINGD = SerTokens.MIME_GRYO_LITE_V1D0 + "-stringd";
+
+ /**
+ * Creates an instance with a standard {@link GryoMapper} instance. Note that this instance
+ * will be overridden by {@link #configure} is called.
+ */
+ public GryoLiteMessageSerializerV1d0() {
+ super(overrideWithLite(GryoMapper.build()).create());
+ }
+
+ @Override
+ public String[] mimeTypesSupported() {
+ return new String[]{serializeToString ? MIME_TYPE_STRINGD : MIME_TYPE};
+ }
+
+ @Override
+ public GryoMapper.Builder beforeCreateMapper(final GryoMapper.Builder builder, final Map<String, Object> config,
+ final Map<String, Graph> graphs) {
+ return overrideWithLite(builder);
+ }
+
+ private static GryoMapper.Builder overrideWithLite(final GryoMapper.Builder builder) {
+ // override the core graph Elements so as to serialize with "reference" as opposed to "detached"
+ builder.addCustom(Edge.class, new EdgeLiteSerializer());
+ builder.addCustom(Vertex.class, new VertexLiteSerializer());
+ builder.addCustom(VertexProperty.class, new VertexPropertyLiteSerializer());
+ builder.addCustom(Property.class, new PropertyLiteSerializer());
+ builder.addCustom(Path.class, new PathLiteSerializer());
+ return builder;
+ }
+
+ /**
+ * Serializes any {@link Edge} implementation encountered to a {@link ReferenceEdge}.
+ */
+ final static class EdgeLiteSerializer extends Serializer<Edge> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Edge edge) {
+ kryo.writeClassAndObject(output, ReferenceFactory.detach(edge));
+ }
+
+ @Override
+ public Edge read(final Kryo kryo, final Input input, final Class<Edge> edgeClass) {
+ final Object o = kryo.readClassAndObject(input);
+ return (Edge) o;
+ }
+ }
+
+ /**
+ * Serializes any {@link Vertex} implementation encountered to an {@link ReferenceVertex}.
+ */
+ final static class VertexLiteSerializer extends Serializer<Vertex> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Vertex vertex) {
+ kryo.writeClassAndObject(output, ReferenceFactory.detach(vertex));
+ }
+
+ @Override
+ public Vertex read(final Kryo kryo, final Input input, final Class<Vertex> vertexClass) {
+ return (Vertex) kryo.readClassAndObject(input);
+ }
+ }
+
+ /**
+ * Serializes any {@link Property} implementation encountered to an {@link ReferenceProperty}.
+ */
+ final static class PropertyLiteSerializer extends Serializer<Property> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Property property) {
+ kryo.writeClassAndObject(output, property instanceof VertexProperty ? ReferenceFactory.detach((VertexProperty) property) : ReferenceFactory.detach(property));
+ }
+
+ @Override
+ public Property read(final Kryo kryo, final Input input, final Class<Property> propertyClass) {
+ return (Property) kryo.readClassAndObject(input);
+ }
+ }
+
+ /**
+ * Serializes any {@link VertexProperty} implementation encountered to an {@link ReferenceVertexProperty}.
+ */
+ final static class VertexPropertyLiteSerializer extends Serializer<VertexProperty> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final VertexProperty vertexProperty) {
+ kryo.writeClassAndObject(output, ReferenceFactory.detach(vertexProperty));
+ }
+
+ @Override
+ public VertexProperty read(final Kryo kryo, final Input input, final Class<VertexProperty> vertexPropertyClass) {
+ return (VertexProperty) kryo.readClassAndObject(input);
+ }
+ }
+
+ /**
+ * Serializes any {@link Path} implementation encountered to an {@link ReferencePath}.
+ */
+ final static class PathLiteSerializer extends Serializer<Path> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Path path) {
+ kryo.writeClassAndObject(output, ReferenceFactory.detach(path));
+ }
+
+ @Override
+ public Path read(final Kryo kryo, final Input input, final Class<Path> pathClass) {
+ return (Path) kryo.readClassAndObject(input);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
index ce46dfc..e9b51d1 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
@@ -18,301 +18,33 @@
*/
package org.apache.tinkerpop.gremlin.driver.ser;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.util.ReferenceCountUtil;
-import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
-import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
-import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
-import org.apache.tinkerpop.shaded.kryo.ClassResolver;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
-
-import java.io.ByteArrayOutputStream;
-import java.lang.reflect.Method;
-import java.nio.charset.Charset;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
/**
+ * The standard Gryo serializer that uses "detached" graph elements during serialization. Detached elements such as
+ * {@link DetachedVertex} include the label and the properties associated with it which could be more costly for
+ * network serialization purposes.
+ *
+ * @see GryoLiteMessageSerializerV1d0
+ *
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-public final class GryoMessageSerializerV1d0 extends AbstractMessageSerializer {
- private GryoMapper gryoMapper;
- private ThreadLocal<Kryo> kryoThreadLocal = new ThreadLocal<Kryo>() {
- @Override
- protected Kryo initialValue() {
- return gryoMapper.createMapper();
- }
- };
-
- private static final Charset UTF8 = Charset.forName("UTF-8");
+public final class GryoMessageSerializerV1d0 extends AbstractGryoMessageSerializerV1d0 {
private static final String MIME_TYPE = SerTokens.MIME_GRYO_V1D0;
private static final String MIME_TYPE_STRINGD = SerTokens.MIME_GRYO_V1D0 + "-stringd";
- public static final String TOKEN_CUSTOM = "custom";
- public static final String TOKEN_SERIALIZE_RESULT_TO_STRING = "serializeResultToString";
- public static final String TOKEN_USE_MAPPER_FROM_GRAPH = "useMapperFromGraph";
- public static final String TOKEN_BUFFER_SIZE = "bufferSize";
- public static final String TOKEN_CLASS_RESOLVER_SUPPLIER = "classResolverSupplier";
-
- private boolean serializeToString = false;
- private int bufferSize = 4096;
-
/**
* Creates an instance with a standard {@link GryoMapper} instance. Note that this instance
- * will be overriden by {@link #configure} is called.
- */
- public GryoMessageSerializerV1d0() {
- gryoMapper = GryoMapper.build().create();
- }
-
- /**
- * Creates an instance with a provided mapper configured {@link GryoMapper} instance. Note that this instance
* will be overridden by {@link #configure} is called.
*/
- public GryoMessageSerializerV1d0(final GryoMapper kryo) {
- this.gryoMapper = kryo;
- }
-
- @Override
- public void configure(final Map<String, Object> config, final Map<String, Graph> graphs) {
- final GryoMapper.Builder builder;
- final Object graphToUseForMapper = config.get(TOKEN_USE_MAPPER_FROM_GRAPH);
- if (graphToUseForMapper != null) {
- if (null == graphs) throw new IllegalStateException(String.format(
- "No graphs have been provided to the serializer and therefore %s is not a valid configuration", TOKEN_USE_MAPPER_FROM_GRAPH));
-
- final Graph g = graphs.get(graphToUseForMapper.toString());
- if (null == g) throw new IllegalStateException(String.format(
- "There is no graph named [%s] configured to be used in the %s setting",
- graphToUseForMapper, TOKEN_USE_MAPPER_FROM_GRAPH));
-
- // a graph was found so use the mapper it constructs. this allows gryo to be auto-configured with any
- // custom classes that the implementation allows for
- builder = g.io(GryoIo.build()).mapper();
- } else {
- // no graph was supplied so just use the default - this will likely be the case when using a graph
- // with no custom classes or a situation where the user needs complete control like when using two
- // distinct implementations each with their own custom classes.
- builder = GryoMapper.build();
- }
-
- addIoRegistries(config, builder);
- addClassResolverSupplier(config, builder);
- addCustomClasses(config, builder);
-
- this.serializeToString = Boolean.parseBoolean(config.getOrDefault(TOKEN_SERIALIZE_RESULT_TO_STRING, "false").toString());
- this.bufferSize = Integer.parseInt(config.getOrDefault(TOKEN_BUFFER_SIZE, "4096").toString());
-
- this.gryoMapper = builder.create();
- }
-
- private void addClassResolverSupplier(final Map<String, Object> config, final GryoMapper.Builder builder) {
- final String className = (String) config.getOrDefault(TOKEN_CLASS_RESOLVER_SUPPLIER, null);
- if (className != null && !className.isEmpty()) {
- try {
- final Class<?> clazz = Class.forName(className);
- try {
- final Method instanceMethod = clazz.getDeclaredMethod("getInstance");
- builder.classResolver((Supplier<ClassResolver>) instanceMethod.invoke(null));
- } catch (Exception methodex) {
- // tried getInstance() and that failed so try newInstance() no-arg constructor
- builder.classResolver((Supplier<ClassResolver>) clazz.newInstance());
- }
- } catch (Exception ex) {
- throw new IllegalStateException(ex);
- }
- }
- }
-
- private void addCustomClasses(final Map<String, Object> config, final GryoMapper.Builder builder) {
- final List<String> classNameList = getListStringFromConfig(TOKEN_CUSTOM, config);
-
- classNameList.stream().forEach(serializerDefinition -> {
- String className;
- Optional<String> serializerName;
- if (serializerDefinition.contains(";")) {
- final String[] split = serializerDefinition.split(";");
- if (split.length != 2)
- throw new IllegalStateException(String.format("Invalid format for serializer definition [%s] - expected <class>;<serializer-class>", serializerDefinition));
-
- className = split[0];
- serializerName = Optional.of(split[1]);
- } else {
- serializerName = Optional.empty();
- className = serializerDefinition;
- }
-
- try {
- final Class clazz = Class.forName(className);
- final Serializer serializer;
- if (serializerName.isPresent()) {
- final Class serializerClazz = Class.forName(serializerName.get());
- serializer = (Serializer) serializerClazz.newInstance();
- builder.addCustom(clazz, kryo -> serializer);
- } else
- builder.addCustom(clazz);
- } catch (Exception ex) {
- throw new IllegalStateException("Class could not be found", ex);
- }
- });
+ public GryoMessageSerializerV1d0() {
+ super(GryoMapper.build().create());
}
@Override
public String[] mimeTypesSupported() {
- return new String[]{this.serializeToString ? MIME_TYPE_STRINGD : MIME_TYPE};
- }
-
- @Override
- public ResponseMessage deserializeResponse(final ByteBuf msg) throws SerializationException {
- try {
- final Kryo kryo = kryoThreadLocal.get();
- final byte[] payload = new byte[msg.capacity()];
- msg.readBytes(payload);
- try (final Input input = new Input(payload)) {
- final UUID requestId = kryo.readObjectOrNull(input, UUID.class);
- final int status = input.readShort();
- final String statusMsg = input.readString();
- final Map<String,Object> statusAttributes = (Map<String,Object>) kryo.readClassAndObject(input);
- final Object result = kryo.readClassAndObject(input);
- final Map<String,Object> metaAttributes = (Map<String,Object>) kryo.readClassAndObject(input);
-
- return ResponseMessage.build(requestId)
- .code(ResponseStatusCode.getFromValue(status))
- .statusMessage(statusMsg)
- .statusAttributes(statusAttributes)
- .result(result)
- .responseMetaData(metaAttributes)
- .create();
- }
- } catch (Exception ex) {
- logger.warn("Response [{}] could not be deserialized by {}.", msg, GryoMessageSerializerV1d0.class.getName());
- throw new SerializationException(ex);
- }
- }
-
- @Override
- public ByteBuf serializeResponseAsBinary(final ResponseMessage responseMessage, final ByteBufAllocator allocator) throws SerializationException {
- ByteBuf encodedMessage = null;
- try {
- final Kryo kryo = kryoThreadLocal.get();
- try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
- final Output output = new Output(baos, bufferSize);
-
- // request id - if present
- kryo.writeObjectOrNull(output, responseMessage.getRequestId() != null ? responseMessage.getRequestId() : null, UUID.class);
-
- // status
- output.writeShort(responseMessage.getStatus().getCode().getValue());
- output.writeString(responseMessage.getStatus().getMessage());
- kryo.writeClassAndObject(output, responseMessage.getStatus().getAttributes());
-
- // result
- kryo.writeClassAndObject(output, serializeToString ? serializeResultToString(responseMessage) : responseMessage.getResult().getData());
- kryo.writeClassAndObject(output, responseMessage.getResult().getMeta());
-
- final long size = output.total();
- if (size > Integer.MAX_VALUE)
- throw new SerializationException(String.format("Message size of %s exceeds allocatable space", size));
-
- output.flush();
- encodedMessage = allocator.buffer((int) size);
- encodedMessage.writeBytes(baos.toByteArray());
- }
-
- return encodedMessage;
- } catch (Exception ex) {
- if (encodedMessage != null) ReferenceCountUtil.release(encodedMessage);
-
- logger.warn("Response [{}] could not be serialized by {}.", responseMessage.toString(), GryoMessageSerializerV1d0.class.getName());
- throw new SerializationException(ex);
- }
- }
-
- @Override
- public RequestMessage deserializeRequest(final ByteBuf msg) throws SerializationException {
- try {
- final Kryo kryo = kryoThreadLocal.get();
- final byte[] payload = new byte[msg.readableBytes()];
- msg.readBytes(payload);
- try (final Input input = new Input(payload)) {
- // by the time the message gets here, the mime length/type have been already read, so this part just
- // needs to process the payload.
- final UUID id = kryo.readObject(input, UUID.class);
- final String processor = input.readString();
- final String op = input.readString();
-
- final RequestMessage.Builder builder = RequestMessage.build(op)
- .overrideRequestId(id)
- .processor(processor);
-
- final Map<String, Object> args = kryo.readObject(input, HashMap.class);
- args.forEach(builder::addArg);
- return builder.create();
- }
- } catch (Exception ex) {
- logger.warn("Request [{}] could not be deserialized by {}.", msg, GryoMessageSerializerV1d0.class.getName());
- throw new SerializationException(ex);
- }
- }
-
- @Override
- public ByteBuf serializeRequestAsBinary(final RequestMessage requestMessage, final ByteBufAllocator allocator) throws SerializationException {
- ByteBuf encodedMessage = null;
- try {
- final Kryo kryo = kryoThreadLocal.get();
- try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
- final Output output = new Output(baos, bufferSize);
- final String mimeType = serializeToString ? MIME_TYPE_STRINGD : MIME_TYPE;
- output.writeByte(mimeType.length());
- output.write(mimeType.getBytes(UTF8));
-
- kryo.writeObject(output, requestMessage.getRequestId());
- output.writeString(requestMessage.getProcessor());
- output.writeString(requestMessage.getOp());
- kryo.writeObject(output, requestMessage.getArgs());
-
- final long size = output.total();
- if (size > Integer.MAX_VALUE)
- throw new SerializationException(String.format("Message size of %s exceeds allocatable space", size));
-
- output.flush();
- encodedMessage = allocator.buffer((int) size);
- encodedMessage.writeBytes(baos.toByteArray());
- }
-
- return encodedMessage;
- } catch (Exception ex) {
- if (encodedMessage != null) ReferenceCountUtil.release(encodedMessage);
-
- logger.warn("Request [{}] could not be serialized by {}.", requestMessage.toString(), GryoMessageSerializerV1d0.class.getName());
- throw new SerializationException(ex);
- }
- }
-
- private Object serializeResultToString(final ResponseMessage msg) {
- if (msg.getResult() == null) return "null";
- if (msg.getResult().getData() == null) return "null";
-
- // the IteratorHandler should return a collection so keep it as such
- final Object o = msg.getResult().getData();
- if (o instanceof Collection) {
- return ((Collection) o).stream().map(d -> null == d ? "null" : d.toString()).collect(Collectors.toList());
- } else {
- return o.toString();
- }
+ return new String[]{serializeToString ? MIME_TYPE_STRINGD : MIME_TYPE};
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/SerTokens.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/SerTokens.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/SerTokens.java
index 779355a..c857116 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/SerTokens.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/SerTokens.java
@@ -39,4 +39,5 @@ public final class SerTokens {
public static final String MIME_JSON = "application/json";
public static final String MIME_GRAPHSON_V1D0 = "application/vnd.gremlin-v1.0+json";
public static final String MIME_GRYO_V1D0 = "application/vnd.gremlin-v1.0+gryo";
+ public static final String MIME_GRYO_LITE_V1D0 = "application/vnd.gremlin-v1.0+gryo-lite";
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/Serializers.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/Serializers.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/Serializers.java
index 11a180e..45ae301 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/Serializers.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/Serializers.java
@@ -28,7 +28,8 @@ import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
public enum Serializers {
GRAPHSON(SerTokens.MIME_JSON),
GRAPHSON_V1D0(SerTokens.MIME_GRAPHSON_V1D0),
- GRYO_V1D0(SerTokens.MIME_GRYO_V1D0);
+ GRYO_V1D0(SerTokens.MIME_GRYO_V1D0),
+ GRYO_LITE_V1D0(SerTokens.MIME_GRYO_LITE_V1D0);
private String value;
@@ -62,6 +63,8 @@ public enum Serializers {
return new GraphSONMessageSerializerGremlinV1d0();
case SerTokens.MIME_GRYO_V1D0:
return new GryoMessageSerializerV1d0();
+ case SerTokens.MIME_GRYO_LITE_V1D0:
+ return new GryoLiteMessageSerializerV1d0();
default:
throw new RuntimeException("Could not create a simple MessageSerializer instance of " + value);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
new file mode 100644
index 0000000..0880f5f
--- /dev/null
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
@@ -0,0 +1,589 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.driver.ser;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
+import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
+import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
+import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoClassResolver;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdge;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.apache.tinkerpop.shaded.kryo.ClassResolver;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.KryoException;
+import org.apache.tinkerpop.shaded.kryo.Registration;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.junit.Test;
+
+import java.awt.Color;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Serializer tests that cover non-lossy serialization/deserialization methods.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class GryoLiteMessageSerializerV1d0Test {
+ private static final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(AbstractGryoMessageSerializerV1d0.TOKEN_SERIALIZE_RESULT_TO_STRING, true);
+ }};
+
+ private UUID requestId = UUID.fromString("6457272A-4018-4538-B9AE-08DD5DDC0AA1");
+ private ResponseMessage.Builder responseMessageBuilder = ResponseMessage.build(requestId);
+ private static ByteBufAllocator allocator = UnpooledByteBufAllocator.DEFAULT;
+
+ public MessageSerializer binarySerializer = new GryoLiteMessageSerializerV1d0();
+
+ public MessageSerializer textSerializer = new GryoLiteMessageSerializerV1d0();
+
+ public GryoLiteMessageSerializerV1d0Test() {
+ textSerializer.configure(config, null);
+ }
+
+ @Test
+ public void shouldConfigureIoRegistry() throws Exception {
+ final MessageSerializer serializer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(AbstractGryoMessageSerializerV1d0.TOKEN_IO_REGISTRIES, Arrays.asList(ColorIoRegistry.class.getName()));
+ }};
+
+ serializer.configure(config, null);
+
+ final ResponseMessage toSerialize = ResponseMessage.build(requestId).result(Color.RED).create();
+ final ByteBuf bb = serializer.serializeResponseAsBinary(toSerialize, allocator);
+ final ResponseMessage deserialized = serializer.deserializeResponse(bb);
+
+ assertCommon(deserialized);
+ assertEquals(Color.RED, deserialized.getResult().getData());
+ }
+
+ @Test
+ public void shouldConfigureCustomClassResolver() {
+ final MessageSerializer serializer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(AbstractGryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplier.class.getName());
+ }};
+
+ serializer.configure(config, null);
+
+ try {
+ serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
+ fail("Should fail because the ClassResolver used here always generates an error");
+ } catch (Exception ex) {
+ assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldConfigureCustomClassResolverFromInstance() {
+ final MessageSerializer serializer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(AbstractGryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplierAsInstance.class.getName());
+ }};
+
+ serializer.configure(config, null);
+
+ try {
+ serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
+ fail("Should fail because the ClassResolver used here always generates an error");
+ } catch (Exception ex) {
+ assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldSerializeIterable() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(100);
+
+ final ResponseMessage response = convertBinary(list);
+ assertCommon(response);
+
+ final List<Integer> deserializedFunList = (List<Integer>) response.getResult().getData();
+ assertEquals(2, deserializedFunList.size());
+ assertEquals(new Integer(1), deserializedFunList.get(0));
+ assertEquals(new Integer(100), deserializedFunList.get(1));
+ }
+
+ @Test
+ public void shouldSerializeIterableToString() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(100);
+
+ final ResponseMessage response = convertText(list);
+ assertCommon(response);
+
+ final List deserializedFunList = (List) response.getResult().getData();
+ assertEquals(2, deserializedFunList.size());
+ assertEquals("1", deserializedFunList.get(0));
+ assertEquals("100", deserializedFunList.get(1));
+ }
+
+ @Test
+ public void shouldSerializeIterableToStringWithNull() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(null);
+ list.add(100);
+
+ final ResponseMessage response = convertText(list);
+ assertCommon(response);
+
+ final List deserializedFunList = (List) response.getResult().getData();
+ assertEquals(3, deserializedFunList.size());
+ assertEquals("1", deserializedFunList.get(0).toString());
+ assertEquals("null", deserializedFunList.get(1).toString());
+ assertEquals("100", deserializedFunList.get(2).toString());
+ }
+
+ @Test
+ public void shouldSerializeIterableWithNull() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(null);
+ list.add(100);
+
+ final ResponseMessage response = convertBinary(list);
+ assertCommon(response);
+
+ final List<Integer> deserializedFunList = (List<Integer>) response.getResult().getData();
+ assertEquals(3, deserializedFunList.size());
+ assertEquals(new Integer(1), deserializedFunList.get(0));
+ assertNull(deserializedFunList.get(1));
+ assertEquals(new Integer(100), deserializedFunList.get(2));
+ }
+
+ @Test
+ public void shouldSerializeMap() throws Exception {
+ final Map<String, Object> map = new HashMap<>();
+ final Map<String, String> innerMap = new HashMap<>();
+ innerMap.put("a", "b");
+
+ map.put("x", 1);
+ map.put("y", "some");
+ map.put("z", innerMap);
+
+ final ResponseMessage response = convertBinary(map);
+ assertCommon(response);
+
+ final Map<String, Object> deserializedMap = (Map<String, Object>) response.getResult().getData();
+ assertEquals(3, deserializedMap.size());
+ assertEquals(1, deserializedMap.get("x"));
+ assertEquals("some", deserializedMap.get("y"));
+
+ final Map<String, String> deserializedInnerMap = (Map<String, String>) deserializedMap.get("z");
+ assertEquals(1, deserializedInnerMap.size());
+ assertEquals("b", deserializedInnerMap.get("a"));
+ }
+
+ @Test
+ public void shouldSerializeMapEntry() throws Exception {
+ final Graph graph = TinkerGraph.open();
+ final Vertex v1 = graph.addVertex();
+ final Date d = new Date();
+
+ final Map<Object, Object> map = new HashMap<>();
+ map.put("x", 1);
+ map.put(v1, 100);
+ map.put(d, "test");
+
+ final ResponseMessage response = convertBinary(IteratorUtils.asList(map.entrySet()));
+ assertCommon(response);
+
+ final List<Map.Entry<Object, Object>> deserializedEntries = (List<Map.Entry<Object, Object>>) response.getResult().getData();
+ assertEquals(3, deserializedEntries.size());
+ deserializedEntries.forEach(e -> {
+ if (e.getKey().equals("x"))
+ assertEquals(1, e.getValue());
+ else if (e.getKey().equals(v1))
+ assertEquals(100, e.getValue());
+ else if (e.getKey().equals(d))
+ assertEquals("test", e.getValue());
+ else
+ fail("Map entries contains a key that is not part of what was serialized");
+ });
+ }
+
+ @Test
+ public void shouldSerializeEdge() throws Exception {
+ final Graph g = TinkerGraph.open();
+ final Vertex v1 = g.addVertex();
+ final Vertex v2 = g.addVertex();
+ final Edge e = v1.addEdge("test", v2);
+ e.property("abc", 123);
+
+ final Iterable<Edge> iterable = IteratorUtils.list(g.edges());
+
+ final ResponseMessage response = convertBinary(iterable);
+ assertCommon(response);
+
+ final List<ReferenceEdge> edgeList = (List<ReferenceEdge>) response.getResult().getData();
+ assertEquals(1, edgeList.size());
+
+ final ReferenceEdge deserializedEdge = edgeList.get(0);
+ assertEquals(e.id(), deserializedEdge.id());
+ assertEquals("test", deserializedEdge.label());
+
+ assertEquals(0, IteratorUtils.count(deserializedEdge.properties()));
+ assertEquals(v1.id(), deserializedEdge.outVertex().id());
+ assertEquals("", deserializedEdge.outVertex().label());
+ assertEquals(v2.id(), deserializedEdge.inVertex().id());
+ assertEquals("", deserializedEdge.inVertex().label());
+ }
+
+ @Test
+ public void shouldSerializeTree() throws Exception {
+ final Graph g = TinkerFactory.createModern();
+ final Tree t = g.traversal().V().out().out().tree().by("name").next();
+
+ final ResponseMessage response = convertBinary(t);
+ assertCommon(response);
+
+ final Tree deserialized = (Tree) response.getResult().getData();
+ assertEquals(t, deserialized);
+
+ assertThat(deserialized.containsKey("marko"), is(true));
+ assertEquals(1, deserialized.size());
+
+ final Tree markoChildren = (Tree) deserialized.get("marko");
+ assertThat(markoChildren.containsKey("josh"), is(true));
+ assertEquals(1, markoChildren.size());
+
+ final Tree joshChildren = (Tree) markoChildren.get("josh");
+ assertThat(joshChildren.containsKey("lop"), is(true));
+ assertThat(joshChildren.containsKey("ripple"), is(true));
+ assertEquals(2, joshChildren.size());
+ }
+
+ @Test
+ public void shouldSerializeVertexWithEmbeddedMap() throws Exception {
+ final Graph g = TinkerGraph.open();
+ final Vertex v = g.addVertex();
+ final Map<String, Object> map = new HashMap<>();
+ map.put("x", 500);
+ map.put("y", "some");
+
+ final ArrayList<Object> friends = new ArrayList<>();
+ friends.add("x");
+ friends.add(5);
+ friends.add(map);
+
+ v.property(VertexProperty.Cardinality.single, "friends", friends);
+
+ final List list = IteratorUtils.list(g.vertices());
+
+ final ResponseMessage response = convertBinary(list);
+ assertCommon(response);
+
+ final List<ReferenceVertex> vertexList = (List<ReferenceVertex>) response.getResult().getData();
+ assertEquals(1, vertexList.size());
+
+ final ReferenceVertex deserializedVertex = vertexList.get(0);
+ assertEquals(0L, deserializedVertex.id());
+ assertEquals("", deserializedVertex.label());
+
+ assertEquals(0, IteratorUtils.count(deserializedVertex.properties()));
+ }
+
+ @Test
+ public void shouldSerializeToMapWithElementForKey() throws Exception {
+ final TinkerGraph graph = TinkerFactory.createClassic();
+ final GraphTraversalSource g = graph.traversal();
+ final Map<Vertex, Integer> map = new HashMap<>();
+ map.put(g.V().has("name", "marko").next(), 1000);
+
+ final ResponseMessage response = convertBinary(map);
+ assertCommon(response);
+
+ final Map<Vertex, Integer> deserializedMap = (Map<Vertex, Integer>) response.getResult().getData();
+ assertEquals(1, deserializedMap.size());
+
+ final Vertex deserializedMarko = deserializedMap.keySet().iterator().next();
+ assertEquals(0, IteratorUtils.count(deserializedMarko.properties()));
+ assertEquals(1, deserializedMarko.id());
+ assertEquals("", deserializedMarko.label());
+
+ assertEquals(new Integer(1000), deserializedMap.values().iterator().next());
+ }
+
+ @Test
+ public void shouldSerializeFullResponseMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final ByteBuf bb = binarySerializer.serializeResponseAsBinary(response, allocator);
+ final ResponseMessage deserialized = binarySerializer.deserializeResponse(bb);
+
+ assertEquals(id, deserialized.getRequestId());
+ assertEquals("this", deserialized.getResult().getMeta().get("test"));
+ assertEquals(1, deserialized.getResult().getMeta().get("one"));
+ assertEquals("some-result", deserialized.getResult().getData());
+ assertEquals("that", deserialized.getStatus().getAttributes().get("test"));
+ assertEquals(2, deserialized.getStatus().getAttributes().get("two"));
+ assertEquals(ResponseStatusCode.SUCCESS.getValue(), deserialized.getStatus().getCode().getValue());
+ assertEquals("worked", deserialized.getStatus().getMessage());
+ }
+
+ @Test
+ public void shouldHaveTooSmallBufferToSerializeResponseMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ put("bufferSize", 1);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ try {
+ binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
+ fail("Should have a buffer size that is too small");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(KryoException.class));
+ }
+ }
+
+ @Test
+ public void shouldReturnAllBytesInResponse() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ // set to bufferSize < total message size but still greater than any individual object requires
+ put("bufferSize", 50);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ ByteBuf buf = binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
+ assertTrue(buf.isReadable());
+ assertEquals(82, buf.readableBytes());
+ }
+
+ @Test
+ public void shouldSerializeFullRequestMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+ final ByteBuf bb = binarySerializer.serializeRequestAsBinary(request, allocator);
+ final int mimeLen = bb.readByte();
+ bb.readBytes(new byte[mimeLen]);
+ final RequestMessage deserialized = binarySerializer.deserializeRequest(bb);
+
+ assertEquals(id, deserialized.getRequestId());
+ assertEquals("pro", deserialized.getProcessor());
+ assertEquals("try", deserialized.getOp());
+ assertEquals("this", deserialized.getArgs().get("test"));
+ }
+
+ @Test
+ public void shouldHaveTooSmallBufferToSerializeRequestMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ put("bufferSize", 1);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ try {
+ binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
+ fail("Should have a buffer size that is too small");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(KryoException.class));
+ }
+ }
+
+ @Test
+ public void shouldReturnAllBytesInRequest() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ // set to bufferSize < total message size but still greater than any individual object requires
+ put("bufferSize", 50);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ ByteBuf buf = binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
+ assertTrue(buf.isReadable());
+ assertEquals(76, buf.readableBytes());
+ }
+
+ private void assertCommon(final ResponseMessage response) {
+ assertEquals(requestId, response.getRequestId());
+ assertEquals(ResponseStatusCode.SUCCESS, response.getStatus().getCode());
+ }
+
+ private ResponseMessage convertBinary(final Object toSerialize) throws SerializationException {
+ final ByteBuf bb = binarySerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
+ return binarySerializer.deserializeResponse(bb);
+ }
+
+ private ResponseMessage convertText(final Object toSerialize) throws SerializationException {
+ final ByteBuf bb = textSerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
+ return textSerializer.deserializeResponse(bb);
+ }
+
+ public static class ErrorOnlyClassResolverSupplierAsInstance implements Supplier<ClassResolver> {
+
+ private static final ErrorOnlyClassResolverSupplierAsInstance instance = new ErrorOnlyClassResolverSupplierAsInstance();
+
+ private ErrorOnlyClassResolverSupplierAsInstance() {}
+
+ public static ErrorOnlyClassResolverSupplierAsInstance getInstance() {
+ return instance;
+ }
+
+ @Override
+ public ClassResolver get() {
+ return new ErrorOnlyClassResolver();
+ }
+ }
+
+ public static class ErrorOnlyClassResolverSupplier implements Supplier<ClassResolver> {
+ @Override
+ public ClassResolver get() {
+ return new ErrorOnlyClassResolver();
+ }
+ }
+
+ public static class ErrorOnlyClassResolver extends GryoClassResolver {
+ @Override
+ public Registration getRegistration(Class clazz) {
+ throw new RuntimeException("Registration is not allowed with this ClassResolver - it is not a good implementation");
+ }
+ }
+
+ public static class ColorIoRegistry extends AbstractIoRegistry {
+ public ColorIoRegistry() {
+ register(GryoIo.class, Color.class, new ColorSerializer());
+ }
+ }
+
+ public static class ColorSerializer extends Serializer<Color> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Color color) {
+ output.write(color.equals(Color.RED) ? 1 : 0);
+ }
+
+ @Override
+ public Color read(final Kryo kryo, final Input input, final Class<Color> aClass) {
+ return input.read() == 1 ? Color.RED : Color.BLACK;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
index bb731a9..4064f49 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
@@ -49,7 +49,7 @@ import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.junit.Test;
-import java.awt.*;
+import java.awt.Color;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-classic.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-classic.yaml b/gremlin-server/conf/gremlin-server-classic.yaml
index 0ac4a58..388a27e 100644
--- a/gremlin-server/conf/gremlin-server-classic.yaml
+++ b/gremlin-server/conf/gremlin-server-classic.yaml
@@ -31,6 +31,7 @@ scriptEngines: {
scripts: [scripts/complex-lifecycle.groovy]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
metrics: {
slf4jReporter: {enabled: true, interval: 180000}}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-modern-readonly.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-modern-readonly.yaml b/gremlin-server/conf/gremlin-server-modern-readonly.yaml
index cc0b0ad..5214b50 100644
--- a/gremlin-server/conf/gremlin-server-modern-readonly.yaml
+++ b/gremlin-server/conf/gremlin-server-modern-readonly.yaml
@@ -31,6 +31,7 @@ scriptEngines: {
scripts: [scripts/generate-modern-readonly.groovy]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
metrics: {
slf4jReporter: {enabled: true, interval: 180000}}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-modern.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-modern.yaml b/gremlin-server/conf/gremlin-server-modern.yaml
index c97ae14..cec7f81 100644
--- a/gremlin-server/conf/gremlin-server-modern.yaml
+++ b/gremlin-server/conf/gremlin-server-modern.yaml
@@ -31,6 +31,7 @@ scriptEngines: {
scripts: [scripts/generate-modern.groovy]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
metrics: {
slf4jReporter: {enabled: true, interval: 180000}}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-neo4j.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-neo4j.yaml b/gremlin-server/conf/gremlin-server-neo4j.yaml
index d32ce8c..55aa170 100644
--- a/gremlin-server/conf/gremlin-server-neo4j.yaml
+++ b/gremlin-server/conf/gremlin-server-neo4j.yaml
@@ -45,6 +45,7 @@ scriptEngines: {
staticImports: [java.lang.Math.PI]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+json
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/json
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-rest-secure.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-rest-secure.yaml b/gremlin-server/conf/gremlin-server-rest-secure.yaml
index ff5c550..fd04855 100644
--- a/gremlin-server/conf/gremlin-server-rest-secure.yaml
+++ b/gremlin-server/conf/gremlin-server-rest-secure.yaml
@@ -44,8 +44,6 @@ scriptEngines: {
"org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.TimedInterruptCustomizerProvider":[10000],
"org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.CompileStaticCustomizerProvider":["org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.SimpleSandboxExtension"]}}}}
serializers:
- - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
- - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+json
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/json
processors:
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-secure.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-secure.yaml b/gremlin-server/conf/gremlin-server-secure.yaml
index 8189fef..4597187 100644
--- a/gremlin-server/conf/gremlin-server-secure.yaml
+++ b/gremlin-server/conf/gremlin-server-secure.yaml
@@ -45,6 +45,7 @@ scriptEngines: {
"org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.CompileStaticCustomizerProvider":["org.apache.tinkerpop.gremlin.groovy.jsr223.customizer.SimpleSandboxExtension"]}}}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+json
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/json
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server-spark.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server-spark.yaml b/gremlin-server/conf/gremlin-server-spark.yaml
index a0467fc..9340611 100644
--- a/gremlin-server/conf/gremlin-server-spark.yaml
+++ b/gremlin-server/conf/gremlin-server-spark.yaml
@@ -58,6 +58,7 @@ scriptEngines: {
staticImports: [java.lang.Math.PI]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+json
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/json
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/conf/gremlin-server.yaml
----------------------------------------------------------------------
diff --git a/gremlin-server/conf/gremlin-server.yaml b/gremlin-server/conf/gremlin-server.yaml
index 0723553..c05ac27 100644
--- a/gremlin-server/conf/gremlin-server.yaml
+++ b/gremlin-server/conf/gremlin-server.yaml
@@ -35,6 +35,7 @@ scriptEngines: {
staticImports: [java.lang.Math.PI]}}
serializers:
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo
+ - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo-lite
- { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+json
- { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/json
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f811a8ae/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinResultSetIntegrateTest.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinResultSetIntegrateTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinResultSetIntegrateTest.java
index 129f1a0..3a4ab27 100644
--- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinResultSetIntegrateTest.java
+++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinResultSetIntegrateTest.java
@@ -24,6 +24,7 @@ import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
import org.apache.tinkerpop.gremlin.driver.Result;
import org.apache.tinkerpop.gremlin.driver.ResultSet;
import org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0;
+import org.apache.tinkerpop.gremlin.driver.ser.Serializers;
import org.apache.tinkerpop.gremlin.process.traversal.Path;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
@@ -38,8 +39,10 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPath;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerIoRegistry;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
@@ -164,6 +167,19 @@ public class GremlinResultSetIntegrateTest extends AbstractGremlinServerIntegrat
}
@Test
+ public void shouldHandleVertexResultWithLiteSerialization() throws Exception {
+ final Cluster cluster = Cluster.build().serializer(Serializers.GRYO_LITE_V1D0).create();
+ final Client clientLite = cluster.connect();
+ final ResultSet results = clientLite.submit("g.V(1).next()");
+ final Vertex v = results.all().get().get(0).getVertex();
+ assertThat(v, instanceOf(ReferenceVertex.class));
+
+ assertEquals(1L, v.id());
+ assertEquals("", v.label());
+ assertEquals(0, IteratorUtils.count(v.properties()));
+ }
+
+ @Test
public void shouldHandleVertexPropertyResult() throws Exception {
final ResultSet results = client.submit("g.V().properties('name').next()");
final VertexProperty<String> v = results.all().get().get(0).getVertexProperty();
[24/27] incubator-tinkerpop git commit: Refactored gryo serialization
tests to have better re-use.
Posted by sp...@apache.org.
Refactored gryo serialization tests to have better re-use.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6d3f1827
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6d3f1827
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6d3f1827
Branch: refs/heads/TINKERPOP-1308
Commit: 6d3f18276fbe7803d775779ad6832ef6a6174639
Parents: 6aa3e98
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon May 23 16:37:54 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu May 26 13:06:07 2016 -0400
----------------------------------------------------------------------
.../ser/GryoBaseMessageSerializerV1d0Test.java | 520 +++++++++++++++++++
.../ser/GryoLiteMessageSerializerV1d0Test.java | 439 ----------------
.../ser/GryoMessageSerializerV1d0Test.java | 445 +---------------
3 files changed, 523 insertions(+), 881 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6d3f1827/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoBaseMessageSerializerV1d0Test.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoBaseMessageSerializerV1d0Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoBaseMessageSerializerV1d0Test.java
new file mode 100644
index 0000000..99bd564
--- /dev/null
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoBaseMessageSerializerV1d0Test.java
@@ -0,0 +1,520 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.driver.ser;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
+import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
+import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
+import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoClassResolver;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.apache.tinkerpop.shaded.kryo.ClassResolver;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.KryoException;
+import org.apache.tinkerpop.shaded.kryo.Registration;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.awt.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+@RunWith(Parameterized.class)
+public class GryoBaseMessageSerializerV1d0Test {
+ @Parameterized.Parameters(name = "expect({0})")
+ public static Iterable<Object[]> data() {
+ final GryoMessageSerializerV1d0 v1d0Text = new GryoMessageSerializerV1d0();
+ v1d0Text.configure(config, null);
+
+ final GryoMessageSerializerV1d0 v1d0LiteText = new GryoMessageSerializerV1d0();
+ v1d0LiteText.configure(config, null);
+
+ return Arrays.asList(new Object[][]{
+ {"V1d0", new GryoMessageSerializerV1d0(), v1d0Text},
+ {"V1d0Lite", new GryoLiteMessageSerializerV1d0(), v1d0LiteText }});
+ }
+
+ @Parameterized.Parameter(value = 0)
+ public String name;
+
+ @Parameterized.Parameter(value = 1)
+ public MessageSerializer binarySerializer;
+
+ @Parameterized.Parameter(value = 2)
+ public MessageSerializer textSerializer;
+
+ private static final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(GryoMessageSerializerV1d0.TOKEN_SERIALIZE_RESULT_TO_STRING, true);
+ }};
+
+ private UUID requestId = UUID.fromString("6457272A-4018-4538-B9AE-08DD5DDC0AA1");
+ private ResponseMessage.Builder responseMessageBuilder = ResponseMessage.build(requestId);
+ private static ByteBufAllocator allocator = UnpooledByteBufAllocator.DEFAULT;
+
+ @Test
+ public void shouldConfigureIoRegistry() throws Exception {
+ final MessageSerializer serializer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(GryoMessageSerializerV1d0.TOKEN_IO_REGISTRIES, Arrays.asList(ColorIoRegistry.class.getName()));
+ }};
+
+ serializer.configure(config, null);
+
+ final ResponseMessage toSerialize = ResponseMessage.build(requestId).result(Color.RED).create();
+ final ByteBuf bb = serializer.serializeResponseAsBinary(toSerialize, allocator);
+ final ResponseMessage deserialized = serializer.deserializeResponse(bb);
+
+ assertCommon(deserialized);
+ assertEquals(Color.RED, deserialized.getResult().getData());
+ }
+
+ @Test
+ public void shouldConfigureCustomClassResolver() {
+ final MessageSerializer serializer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(GryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplier.class.getName());
+ }};
+
+ serializer.configure(config, null);
+
+ try {
+ serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
+ fail("Should fail because the ClassResolver used here always generates an error");
+ } catch (Exception ex) {
+ assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldConfigureCustomClassResolverFromInstance() {
+ final MessageSerializer serializer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> config = new HashMap<String, Object>() {{
+ put(GryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplierAsInstance.class.getName());
+ }};
+
+ serializer.configure(config, null);
+
+ try {
+ serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
+ fail("Should fail because the ClassResolver used here always generates an error");
+ } catch (Exception ex) {
+ assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldSerializeIterable() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(100);
+
+ final ResponseMessage response = convertBinary(list);
+ assertCommon(response);
+
+ final java.util.List<Integer> deserializedFunList = (java.util.List<Integer>) response.getResult().getData();
+ assertEquals(2, deserializedFunList.size());
+ assertEquals(new Integer(1), deserializedFunList.get(0));
+ assertEquals(new Integer(100), deserializedFunList.get(1));
+ }
+
+ @Test
+ public void shouldSerializeIterableToString() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(100);
+
+ final ResponseMessage response = convertText(list);
+ assertCommon(response);
+
+ final java.util.List deserializedFunList = (java.util.List) response.getResult().getData();
+ assertEquals(2, deserializedFunList.size());
+ assertEquals("1", deserializedFunList.get(0));
+ assertEquals("100", deserializedFunList.get(1));
+ }
+
+ @Test
+ public void shouldSerializeIterableToStringWithNull() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(null);
+ list.add(100);
+
+ final ResponseMessage response = convertText(list);
+ assertCommon(response);
+
+ final java.util.List deserializedFunList = (java.util.List) response.getResult().getData();
+ assertEquals(3, deserializedFunList.size());
+ assertEquals("1", deserializedFunList.get(0).toString());
+ assertEquals("null", deserializedFunList.get(1).toString());
+ assertEquals("100", deserializedFunList.get(2).toString());
+ }
+
+ @Test
+ public void shouldSerializeIterableWithNull() throws Exception {
+ final ArrayList<Integer> list = new ArrayList<>();
+ list.add(1);
+ list.add(null);
+ list.add(100);
+
+ final ResponseMessage response = convertBinary(list);
+ assertCommon(response);
+
+ final java.util.List<Integer> deserializedFunList = (java.util.List<Integer>) response.getResult().getData();
+ assertEquals(3, deserializedFunList.size());
+ assertEquals(new Integer(1), deserializedFunList.get(0));
+ assertNull(deserializedFunList.get(1));
+ assertEquals(new Integer(100), deserializedFunList.get(2));
+ }
+
+ @Test
+ public void shouldSerializeMap() throws Exception {
+ final Map<String, Object> map = new HashMap<>();
+ final Map<String, String> innerMap = new HashMap<>();
+ innerMap.put("a", "b");
+
+ map.put("x", 1);
+ map.put("y", "some");
+ map.put("z", innerMap);
+
+ final ResponseMessage response = convertBinary(map);
+ assertCommon(response);
+
+ final Map<String, Object> deserializedMap = (Map<String, Object>) response.getResult().getData();
+ assertEquals(3, deserializedMap.size());
+ assertEquals(1, deserializedMap.get("x"));
+ assertEquals("some", deserializedMap.get("y"));
+
+ final Map<String, String> deserializedInnerMap = (Map<String, String>) deserializedMap.get("z");
+ assertEquals(1, deserializedInnerMap.size());
+ assertEquals("b", deserializedInnerMap.get("a"));
+ }
+
+ @Test
+ public void shouldSerializeMapEntry() throws Exception {
+ final Graph graph = TinkerGraph.open();
+ final Vertex v1 = graph.addVertex();
+ final Date d = new Date();
+
+ final Map<Object, Object> map = new HashMap<>();
+ map.put("x", 1);
+ map.put(v1, 100);
+ map.put(d, "test");
+
+ final ResponseMessage response = convertBinary(IteratorUtils.asList(map.entrySet()));
+ assertCommon(response);
+
+ final java.util.List<Map.Entry<Object, Object>> deserializedEntries = (java.util.List<Map.Entry<Object, Object>>) response.getResult().getData();
+ assertEquals(3, deserializedEntries.size());
+ deserializedEntries.forEach(e -> {
+ if (e.getKey().equals("x"))
+ assertEquals(1, e.getValue());
+ else if (e.getKey().equals(v1))
+ assertEquals(100, e.getValue());
+ else if (e.getKey().equals(d))
+ assertEquals("test", e.getValue());
+ else
+ fail("Map entries contains a key that is not part of what was serialized");
+ });
+ }
+
+ @Test
+ public void shouldSerializeTree() throws Exception {
+ final Graph g = TinkerFactory.createModern();
+ final Tree t = g.traversal().V().out().out().tree().by("name").next();
+
+ final ResponseMessage response = convertBinary(t);
+ assertCommon(response);
+
+ final Tree deserialized = (Tree) response.getResult().getData();
+ assertEquals(t, deserialized);
+
+ assertThat(deserialized.containsKey("marko"), is(true));
+ assertEquals(1, deserialized.size());
+
+ final Tree markoChildren = (Tree) deserialized.get("marko");
+ assertThat(markoChildren.containsKey("josh"), is(true));
+ assertEquals(1, markoChildren.size());
+
+ final Tree joshChildren = (Tree) markoChildren.get("josh");
+ assertThat(joshChildren.containsKey("lop"), is(true));
+ assertThat(joshChildren.containsKey("ripple"), is(true));
+ assertEquals(2, joshChildren.size());
+ }
+
+ @Test
+ public void shouldSerializeFullResponseMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final ByteBuf bb = binarySerializer.serializeResponseAsBinary(response, allocator);
+ final ResponseMessage deserialized = binarySerializer.deserializeResponse(bb);
+
+ assertEquals(id, deserialized.getRequestId());
+ assertEquals("this", deserialized.getResult().getMeta().get("test"));
+ assertEquals(1, deserialized.getResult().getMeta().get("one"));
+ assertEquals("some-result", deserialized.getResult().getData());
+ assertEquals("that", deserialized.getStatus().getAttributes().get("test"));
+ assertEquals(2, deserialized.getStatus().getAttributes().get("two"));
+ assertEquals(ResponseStatusCode.SUCCESS.getValue(), deserialized.getStatus().getCode().getValue());
+ assertEquals("worked", deserialized.getStatus().getMessage());
+ }
+
+ @Test
+ public void shouldHaveTooSmallBufferToSerializeResponseMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ put("bufferSize", 1);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ try {
+ binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
+ fail("Should have a buffer size that is too small");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(KryoException.class));
+ }
+ }
+
+ @Test
+ public void shouldReturnAllBytesInResponse() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final Map<String, Object> metaData = new HashMap<>();
+ metaData.put("test", "this");
+ metaData.put("one", 1);
+
+ final Map<String, Object> attributes = new HashMap<>();
+ attributes.put("test", "that");
+ attributes.put("two", 2);
+
+ final ResponseMessage response = ResponseMessage.build(id)
+ .responseMetaData(metaData)
+ .code(ResponseStatusCode.SUCCESS)
+ .result("some-result")
+ .statusAttributes(attributes)
+ .statusMessage("worked")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ // set to bufferSize < total message size but still greater than any individual object requires
+ put("bufferSize", 50);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ ByteBuf buf = binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
+ assertTrue(buf.isReadable());
+ assertEquals(82, buf.readableBytes());
+ }
+
+ @Test
+ public void shouldSerializeFullRequestMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+ final ByteBuf bb = binarySerializer.serializeRequestAsBinary(request, allocator);
+ final int mimeLen = bb.readByte();
+ bb.readBytes(new byte[mimeLen]);
+ final RequestMessage deserialized = binarySerializer.deserializeRequest(bb);
+
+ assertEquals(id, deserialized.getRequestId());
+ assertEquals("pro", deserialized.getProcessor());
+ assertEquals("try", deserialized.getOp());
+ assertEquals("this", deserialized.getArgs().get("test"));
+ }
+
+ @Test
+ public void shouldHaveTooSmallBufferToSerializeRequestMessage() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ put("bufferSize", 1);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ try {
+ binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
+ fail("Should have a buffer size that is too small");
+ } catch (Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertThat(root, instanceOf(KryoException.class));
+ }
+ }
+
+ @Test
+ public void shouldReturnAllBytesInRequest() throws Exception {
+ final UUID id = UUID.randomUUID();
+
+ final RequestMessage request = RequestMessage.build("try")
+ .overrideRequestId(id)
+ .processor("pro")
+ .addArg("test", "this")
+ .create();
+
+ final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
+ final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
+ // set to bufferSize < total message size but still greater than any individual object requires
+ put("bufferSize", 50);
+ }};
+ binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
+
+ ByteBuf buf = binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
+ assertTrue(buf.isReadable());
+ assertEquals(71, buf.readableBytes());
+ }
+
+ private void assertCommon(final ResponseMessage response) {
+ assertEquals(requestId, response.getRequestId());
+ assertEquals(ResponseStatusCode.SUCCESS, response.getStatus().getCode());
+ }
+
+ private ResponseMessage convertBinary(final Object toSerialize) throws SerializationException {
+ final ByteBuf bb = binarySerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
+ return binarySerializer.deserializeResponse(bb);
+ }
+
+ private ResponseMessage convertText(final Object toSerialize) throws SerializationException {
+ final ByteBuf bb = textSerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
+ return textSerializer.deserializeResponse(bb);
+ }
+
+ public static class ErrorOnlyClassResolverSupplierAsInstance implements Supplier<ClassResolver> {
+
+ private static final ErrorOnlyClassResolverSupplierAsInstance instance = new ErrorOnlyClassResolverSupplierAsInstance();
+
+ private ErrorOnlyClassResolverSupplierAsInstance() {}
+
+ public static ErrorOnlyClassResolverSupplierAsInstance getInstance() {
+ return instance;
+ }
+
+ @Override
+ public ClassResolver get() {
+ return new ErrorOnlyClassResolver();
+ }
+ }
+
+ public static class ErrorOnlyClassResolverSupplier implements Supplier<ClassResolver> {
+ @Override
+ public ClassResolver get() {
+ return new ErrorOnlyClassResolver();
+ }
+ }
+
+ public static class ErrorOnlyClassResolver extends GryoClassResolver {
+ @Override
+ public Registration getRegistration(Class clazz) {
+ throw new RuntimeException("Registration is not allowed with this ClassResolver - it is not a good implementation");
+ }
+ }
+
+ public static class ColorIoRegistry extends AbstractIoRegistry {
+ public ColorIoRegistry() {
+ register(GryoIo.class, Color.class, new ColorSerializer());
+ }
+ }
+
+ public static class ColorSerializer extends Serializer<Color> {
+ @Override
+ public void write(final Kryo kryo, final Output output, final Color color) {
+ output.write(color.equals(Color.RED) ? 1 : 0);
+ }
+
+ @Override
+ public Color read(final Kryo kryo, final Input input, final Class<Color> aClass) {
+ return input.read() == 1 ? Color.RED : Color.BLACK;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6d3f1827/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
index 0880f5f..2a23d02 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoLiteMessageSerializerV1d0Test.java
@@ -21,51 +21,28 @@ package org.apache.tinkerpop.gremlin.driver.ser;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
-import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
-import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoClassResolver;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdge;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import org.apache.tinkerpop.shaded.kryo.ClassResolver;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.KryoException;
-import org.apache.tinkerpop.shaded.kryo.Registration;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.junit.Test;
-import java.awt.Color;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.function.Supplier;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
/**
* Serializer tests that cover non-lossy serialization/deserialization methods.
@@ -83,178 +60,6 @@ public class GryoLiteMessageSerializerV1d0Test {
public MessageSerializer binarySerializer = new GryoLiteMessageSerializerV1d0();
- public MessageSerializer textSerializer = new GryoLiteMessageSerializerV1d0();
-
- public GryoLiteMessageSerializerV1d0Test() {
- textSerializer.configure(config, null);
- }
-
- @Test
- public void shouldConfigureIoRegistry() throws Exception {
- final MessageSerializer serializer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> config = new HashMap<String, Object>() {{
- put(AbstractGryoMessageSerializerV1d0.TOKEN_IO_REGISTRIES, Arrays.asList(ColorIoRegistry.class.getName()));
- }};
-
- serializer.configure(config, null);
-
- final ResponseMessage toSerialize = ResponseMessage.build(requestId).result(Color.RED).create();
- final ByteBuf bb = serializer.serializeResponseAsBinary(toSerialize, allocator);
- final ResponseMessage deserialized = serializer.deserializeResponse(bb);
-
- assertCommon(deserialized);
- assertEquals(Color.RED, deserialized.getResult().getData());
- }
-
- @Test
- public void shouldConfigureCustomClassResolver() {
- final MessageSerializer serializer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> config = new HashMap<String, Object>() {{
- put(AbstractGryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplier.class.getName());
- }};
-
- serializer.configure(config, null);
-
- try {
- serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
- fail("Should fail because the ClassResolver used here always generates an error");
- } catch (Exception ex) {
- assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
- }
- }
-
- @Test
- public void shouldConfigureCustomClassResolverFromInstance() {
- final MessageSerializer serializer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> config = new HashMap<String, Object>() {{
- put(AbstractGryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplierAsInstance.class.getName());
- }};
-
- serializer.configure(config, null);
-
- try {
- serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
- fail("Should fail because the ClassResolver used here always generates an error");
- } catch (Exception ex) {
- assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
- }
- }
-
- @Test
- public void shouldSerializeIterable() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(100);
-
- final ResponseMessage response = convertBinary(list);
- assertCommon(response);
-
- final List<Integer> deserializedFunList = (List<Integer>) response.getResult().getData();
- assertEquals(2, deserializedFunList.size());
- assertEquals(new Integer(1), deserializedFunList.get(0));
- assertEquals(new Integer(100), deserializedFunList.get(1));
- }
-
- @Test
- public void shouldSerializeIterableToString() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(100);
-
- final ResponseMessage response = convertText(list);
- assertCommon(response);
-
- final List deserializedFunList = (List) response.getResult().getData();
- assertEquals(2, deserializedFunList.size());
- assertEquals("1", deserializedFunList.get(0));
- assertEquals("100", deserializedFunList.get(1));
- }
-
- @Test
- public void shouldSerializeIterableToStringWithNull() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(null);
- list.add(100);
-
- final ResponseMessage response = convertText(list);
- assertCommon(response);
-
- final List deserializedFunList = (List) response.getResult().getData();
- assertEquals(3, deserializedFunList.size());
- assertEquals("1", deserializedFunList.get(0).toString());
- assertEquals("null", deserializedFunList.get(1).toString());
- assertEquals("100", deserializedFunList.get(2).toString());
- }
-
- @Test
- public void shouldSerializeIterableWithNull() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(null);
- list.add(100);
-
- final ResponseMessage response = convertBinary(list);
- assertCommon(response);
-
- final List<Integer> deserializedFunList = (List<Integer>) response.getResult().getData();
- assertEquals(3, deserializedFunList.size());
- assertEquals(new Integer(1), deserializedFunList.get(0));
- assertNull(deserializedFunList.get(1));
- assertEquals(new Integer(100), deserializedFunList.get(2));
- }
-
- @Test
- public void shouldSerializeMap() throws Exception {
- final Map<String, Object> map = new HashMap<>();
- final Map<String, String> innerMap = new HashMap<>();
- innerMap.put("a", "b");
-
- map.put("x", 1);
- map.put("y", "some");
- map.put("z", innerMap);
-
- final ResponseMessage response = convertBinary(map);
- assertCommon(response);
-
- final Map<String, Object> deserializedMap = (Map<String, Object>) response.getResult().getData();
- assertEquals(3, deserializedMap.size());
- assertEquals(1, deserializedMap.get("x"));
- assertEquals("some", deserializedMap.get("y"));
-
- final Map<String, String> deserializedInnerMap = (Map<String, String>) deserializedMap.get("z");
- assertEquals(1, deserializedInnerMap.size());
- assertEquals("b", deserializedInnerMap.get("a"));
- }
-
- @Test
- public void shouldSerializeMapEntry() throws Exception {
- final Graph graph = TinkerGraph.open();
- final Vertex v1 = graph.addVertex();
- final Date d = new Date();
-
- final Map<Object, Object> map = new HashMap<>();
- map.put("x", 1);
- map.put(v1, 100);
- map.put(d, "test");
-
- final ResponseMessage response = convertBinary(IteratorUtils.asList(map.entrySet()));
- assertCommon(response);
-
- final List<Map.Entry<Object, Object>> deserializedEntries = (List<Map.Entry<Object, Object>>) response.getResult().getData();
- assertEquals(3, deserializedEntries.size());
- deserializedEntries.forEach(e -> {
- if (e.getKey().equals("x"))
- assertEquals(1, e.getValue());
- else if (e.getKey().equals(v1))
- assertEquals(100, e.getValue());
- else if (e.getKey().equals(d))
- assertEquals("test", e.getValue());
- else
- fail("Map entries contains a key that is not part of what was serialized");
- });
- }
-
@Test
public void shouldSerializeEdge() throws Exception {
final Graph g = TinkerGraph.open();
@@ -283,30 +88,6 @@ public class GryoLiteMessageSerializerV1d0Test {
}
@Test
- public void shouldSerializeTree() throws Exception {
- final Graph g = TinkerFactory.createModern();
- final Tree t = g.traversal().V().out().out().tree().by("name").next();
-
- final ResponseMessage response = convertBinary(t);
- assertCommon(response);
-
- final Tree deserialized = (Tree) response.getResult().getData();
- assertEquals(t, deserialized);
-
- assertThat(deserialized.containsKey("marko"), is(true));
- assertEquals(1, deserialized.size());
-
- final Tree markoChildren = (Tree) deserialized.get("marko");
- assertThat(markoChildren.containsKey("josh"), is(true));
- assertEquals(1, markoChildren.size());
-
- final Tree joshChildren = (Tree) markoChildren.get("josh");
- assertThat(joshChildren.containsKey("lop"), is(true));
- assertThat(joshChildren.containsKey("ripple"), is(true));
- assertEquals(2, joshChildren.size());
- }
-
- @Test
public void shouldSerializeVertexWithEmbeddedMap() throws Exception {
final Graph g = TinkerGraph.open();
final Vertex v = g.addVertex();
@@ -357,173 +138,6 @@ public class GryoLiteMessageSerializerV1d0Test {
assertEquals(new Integer(1000), deserializedMap.values().iterator().next());
}
- @Test
- public void shouldSerializeFullResponseMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final Map<String, Object> metaData = new HashMap<>();
- metaData.put("test", "this");
- metaData.put("one", 1);
-
- final Map<String, Object> attributes = new HashMap<>();
- attributes.put("test", "that");
- attributes.put("two", 2);
-
- final ResponseMessage response = ResponseMessage.build(id)
- .responseMetaData(metaData)
- .code(ResponseStatusCode.SUCCESS)
- .result("some-result")
- .statusAttributes(attributes)
- .statusMessage("worked")
- .create();
-
- final ByteBuf bb = binarySerializer.serializeResponseAsBinary(response, allocator);
- final ResponseMessage deserialized = binarySerializer.deserializeResponse(bb);
-
- assertEquals(id, deserialized.getRequestId());
- assertEquals("this", deserialized.getResult().getMeta().get("test"));
- assertEquals(1, deserialized.getResult().getMeta().get("one"));
- assertEquals("some-result", deserialized.getResult().getData());
- assertEquals("that", deserialized.getStatus().getAttributes().get("test"));
- assertEquals(2, deserialized.getStatus().getAttributes().get("two"));
- assertEquals(ResponseStatusCode.SUCCESS.getValue(), deserialized.getStatus().getCode().getValue());
- assertEquals("worked", deserialized.getStatus().getMessage());
- }
-
- @Test
- public void shouldHaveTooSmallBufferToSerializeResponseMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final Map<String, Object> metaData = new HashMap<>();
- metaData.put("test", "this");
- metaData.put("one", 1);
-
- final Map<String, Object> attributes = new HashMap<>();
- attributes.put("test", "that");
- attributes.put("two", 2);
-
- final ResponseMessage response = ResponseMessage.build(id)
- .responseMetaData(metaData)
- .code(ResponseStatusCode.SUCCESS)
- .result("some-result")
- .statusAttributes(attributes)
- .statusMessage("worked")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- put("bufferSize", 1);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- try {
- binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
- fail("Should have a buffer size that is too small");
- } catch (Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertThat(root, instanceOf(KryoException.class));
- }
- }
-
- @Test
- public void shouldReturnAllBytesInResponse() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final Map<String, Object> metaData = new HashMap<>();
- metaData.put("test", "this");
- metaData.put("one", 1);
-
- final Map<String, Object> attributes = new HashMap<>();
- attributes.put("test", "that");
- attributes.put("two", 2);
-
- final ResponseMessage response = ResponseMessage.build(id)
- .responseMetaData(metaData)
- .code(ResponseStatusCode.SUCCESS)
- .result("some-result")
- .statusAttributes(attributes)
- .statusMessage("worked")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- // set to bufferSize < total message size but still greater than any individual object requires
- put("bufferSize", 50);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- ByteBuf buf = binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
- assertTrue(buf.isReadable());
- assertEquals(82, buf.readableBytes());
- }
-
- @Test
- public void shouldSerializeFullRequestMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final RequestMessage request = RequestMessage.build("try")
- .overrideRequestId(id)
- .processor("pro")
- .addArg("test", "this")
- .create();
- final ByteBuf bb = binarySerializer.serializeRequestAsBinary(request, allocator);
- final int mimeLen = bb.readByte();
- bb.readBytes(new byte[mimeLen]);
- final RequestMessage deserialized = binarySerializer.deserializeRequest(bb);
-
- assertEquals(id, deserialized.getRequestId());
- assertEquals("pro", deserialized.getProcessor());
- assertEquals("try", deserialized.getOp());
- assertEquals("this", deserialized.getArgs().get("test"));
- }
-
- @Test
- public void shouldHaveTooSmallBufferToSerializeRequestMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final RequestMessage request = RequestMessage.build("try")
- .overrideRequestId(id)
- .processor("pro")
- .addArg("test", "this")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- put("bufferSize", 1);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- try {
- binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
- fail("Should have a buffer size that is too small");
- } catch (Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertThat(root, instanceOf(KryoException.class));
- }
- }
-
- @Test
- public void shouldReturnAllBytesInRequest() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final RequestMessage request = RequestMessage.build("try")
- .overrideRequestId(id)
- .processor("pro")
- .addArg("test", "this")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoLiteMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- // set to bufferSize < total message size but still greater than any individual object requires
- put("bufferSize", 50);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- ByteBuf buf = binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
- assertTrue(buf.isReadable());
- assertEquals(76, buf.readableBytes());
- }
-
private void assertCommon(final ResponseMessage response) {
assertEquals(requestId, response.getRequestId());
assertEquals(ResponseStatusCode.SUCCESS, response.getStatus().getCode());
@@ -533,57 +147,4 @@ public class GryoLiteMessageSerializerV1d0Test {
final ByteBuf bb = binarySerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
return binarySerializer.deserializeResponse(bb);
}
-
- private ResponseMessage convertText(final Object toSerialize) throws SerializationException {
- final ByteBuf bb = textSerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
- return textSerializer.deserializeResponse(bb);
- }
-
- public static class ErrorOnlyClassResolverSupplierAsInstance implements Supplier<ClassResolver> {
-
- private static final ErrorOnlyClassResolverSupplierAsInstance instance = new ErrorOnlyClassResolverSupplierAsInstance();
-
- private ErrorOnlyClassResolverSupplierAsInstance() {}
-
- public static ErrorOnlyClassResolverSupplierAsInstance getInstance() {
- return instance;
- }
-
- @Override
- public ClassResolver get() {
- return new ErrorOnlyClassResolver();
- }
- }
-
- public static class ErrorOnlyClassResolverSupplier implements Supplier<ClassResolver> {
- @Override
- public ClassResolver get() {
- return new ErrorOnlyClassResolver();
- }
- }
-
- public static class ErrorOnlyClassResolver extends GryoClassResolver {
- @Override
- public Registration getRegistration(Class clazz) {
- throw new RuntimeException("Registration is not allowed with this ClassResolver - it is not a good implementation");
- }
- }
-
- public static class ColorIoRegistry extends AbstractIoRegistry {
- public ColorIoRegistry() {
- register(GryoIo.class, Color.class, new ColorSerializer());
- }
- }
-
- public static class ColorSerializer extends Serializer<Color> {
- @Override
- public void write(final Kryo kryo, final Output output, final Color color) {
- output.write(color.equals(Color.RED) ? 1 : 0);
- }
-
- @Override
- public Color read(final Kryo kryo, final Input input, final Class<Color> aClass) {
- return input.read() == 1 ? Color.RED : Color.BLACK;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6d3f1827/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
index 4064f49..36e17c0 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0Test.java
@@ -18,54 +18,31 @@
*/
package org.apache.tinkerpop.gremlin.driver.ser;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
import org.apache.tinkerpop.gremlin.driver.MessageSerializer;
-import org.apache.tinkerpop.gremlin.driver.message.RequestMessage;
import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage;
import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoClassResolver;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.UnpooledByteBufAllocator;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import org.apache.tinkerpop.shaded.kryo.ClassResolver;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.KryoException;
-import org.apache.tinkerpop.shaded.kryo.Registration;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.junit.Test;
-import java.awt.Color;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.function.Supplier;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
/**
* Serializer tests that cover non-lossy serialization/deserialization methods.
@@ -83,178 +60,6 @@ public class GryoMessageSerializerV1d0Test {
public MessageSerializer binarySerializer = new GryoMessageSerializerV1d0();
- public MessageSerializer textSerializer = new GryoMessageSerializerV1d0();
-
- public GryoMessageSerializerV1d0Test() {
- textSerializer.configure(config, null);
- }
-
- @Test
- public void shouldConfigureIoRegistry() throws Exception {
- final MessageSerializer serializer = new GryoMessageSerializerV1d0();
- final Map<String, Object> config = new HashMap<String, Object>() {{
- put(GryoMessageSerializerV1d0.TOKEN_IO_REGISTRIES, Arrays.asList(ColorIoRegistry.class.getName()));
- }};
-
- serializer.configure(config, null);
-
- final ResponseMessage toSerialize = ResponseMessage.build(requestId).result(Color.RED).create();
- final ByteBuf bb = serializer.serializeResponseAsBinary(toSerialize, allocator);
- final ResponseMessage deserialized = serializer.deserializeResponse(bb);
-
- assertCommon(deserialized);
- assertEquals(Color.RED, deserialized.getResult().getData());
- }
-
- @Test
- public void shouldConfigureCustomClassResolver() {
- final MessageSerializer serializer = new GryoMessageSerializerV1d0();
- final Map<String, Object> config = new HashMap<String, Object>() {{
- put(GryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplier.class.getName());
- }};
-
- serializer.configure(config, null);
-
- try {
- serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
- fail("Should fail because the ClassResolver used here always generates an error");
- } catch (Exception ex) {
- assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
- }
- }
-
- @Test
- public void shouldConfigureCustomClassResolverFromInstance() {
- final MessageSerializer serializer = new GryoMessageSerializerV1d0();
- final Map<String, Object> config = new HashMap<String, Object>() {{
- put(GryoMessageSerializerV1d0.TOKEN_CLASS_RESOLVER_SUPPLIER, ErrorOnlyClassResolverSupplierAsInstance.class.getName());
- }};
-
- serializer.configure(config, null);
-
- try {
- serializer.serializeResponseAsBinary(responseMessageBuilder.create(), allocator);
- fail("Should fail because the ClassResolver used here always generates an error");
- } catch (Exception ex) {
- assertEquals("java.lang.RuntimeException: Registration is not allowed with this ClassResolver - it is not a good implementation", ex.getMessage());
- }
- }
-
- @Test
- public void shouldSerializeIterable() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(100);
-
- final ResponseMessage response = convertBinary(list);
- assertCommon(response);
-
- final List<Integer> deserializedFunList = (List<Integer>) response.getResult().getData();
- assertEquals(2, deserializedFunList.size());
- assertEquals(new Integer(1), deserializedFunList.get(0));
- assertEquals(new Integer(100), deserializedFunList.get(1));
- }
-
- @Test
- public void shouldSerializeIterableToString() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(100);
-
- final ResponseMessage response = convertText(list);
- assertCommon(response);
-
- final List deserializedFunList = (List) response.getResult().getData();
- assertEquals(2, deserializedFunList.size());
- assertEquals("1", deserializedFunList.get(0));
- assertEquals("100", deserializedFunList.get(1));
- }
-
- @Test
- public void shouldSerializeIterableToStringWithNull() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(null);
- list.add(100);
-
- final ResponseMessage response = convertText(list);
- assertCommon(response);
-
- final List deserializedFunList = (List) response.getResult().getData();
- assertEquals(3, deserializedFunList.size());
- assertEquals("1", deserializedFunList.get(0).toString());
- assertEquals("null", deserializedFunList.get(1).toString());
- assertEquals("100", deserializedFunList.get(2).toString());
- }
-
- @Test
- public void shouldSerializeIterableWithNull() throws Exception {
- final ArrayList<Integer> list = new ArrayList<>();
- list.add(1);
- list.add(null);
- list.add(100);
-
- final ResponseMessage response = convertBinary(list);
- assertCommon(response);
-
- final List<Integer> deserializedFunList = (List<Integer>) response.getResult().getData();
- assertEquals(3, deserializedFunList.size());
- assertEquals(new Integer(1), deserializedFunList.get(0));
- assertNull(deserializedFunList.get(1));
- assertEquals(new Integer(100), deserializedFunList.get(2));
- }
-
- @Test
- public void shouldSerializeMap() throws Exception {
- final Map<String, Object> map = new HashMap<>();
- final Map<String, String> innerMap = new HashMap<>();
- innerMap.put("a", "b");
-
- map.put("x", 1);
- map.put("y", "some");
- map.put("z", innerMap);
-
- final ResponseMessage response = convertBinary(map);
- assertCommon(response);
-
- final Map<String, Object> deserializedMap = (Map<String, Object>) response.getResult().getData();
- assertEquals(3, deserializedMap.size());
- assertEquals(1, deserializedMap.get("x"));
- assertEquals("some", deserializedMap.get("y"));
-
- final Map<String, String> deserializedInnerMap = (Map<String, String>) deserializedMap.get("z");
- assertEquals(1, deserializedInnerMap.size());
- assertEquals("b", deserializedInnerMap.get("a"));
- }
-
- @Test
- public void shouldSerializeMapEntry() throws Exception {
- final Graph graph = TinkerGraph.open();
- final Vertex v1 = graph.addVertex();
- final Date d = new Date();
-
- final Map<Object, Object> map = new HashMap<>();
- map.put("x", 1);
- map.put(v1, 100);
- map.put(d, "test");
-
- final ResponseMessage response = convertBinary(IteratorUtils.asList(map.entrySet()));
- assertCommon(response);
-
- final List<Map.Entry<Object, Object>> deserializedEntries = (List<Map.Entry<Object, Object>>) response.getResult().getData();
- assertEquals(3, deserializedEntries.size());
- deserializedEntries.forEach(e -> {
- if (e.getKey().equals("x"))
- assertEquals(1, e.getValue());
- else if (e.getKey().equals(v1))
- assertEquals(100, e.getValue());
- else if (e.getKey().equals(d))
- assertEquals("test", e.getValue());
- else
- fail("Map entries contains a key that is not part of what was serialized");
- });
- }
-
@Test
public void shouldSerializeEdge() throws Exception {
final Graph g = TinkerGraph.open();
@@ -284,30 +89,6 @@ public class GryoMessageSerializerV1d0Test {
}
@Test
- public void shouldSerializeTree() throws Exception {
- final Graph g = TinkerFactory.createModern();
- final Tree t = g.traversal().V().out().out().tree().by("name").next();
-
- final ResponseMessage response = convertBinary(t);
- assertCommon(response);
-
- final Tree deserialized = (Tree) response.getResult().getData();
- assertEquals(t, deserialized);
-
- assertThat(deserialized.containsKey("marko"), is(true));
- assertEquals(1, deserialized.size());
-
- final Tree markoChildren = (Tree) deserialized.get("marko");
- assertThat(markoChildren.containsKey("josh"), is(true));
- assertEquals(1, markoChildren.size());
-
- final Tree joshChildren = (Tree) markoChildren.get("josh");
- assertThat(joshChildren.containsKey("lop"), is(true));
- assertThat(joshChildren.containsKey("ripple"), is(true));
- assertEquals(2, joshChildren.size());
- }
-
- @Test
public void shouldSerializeVertexWithEmbeddedMap() throws Exception {
final Graph g = TinkerGraph.open();
final Vertex v = g.addVertex();
@@ -370,173 +151,6 @@ public class GryoMessageSerializerV1d0Test {
assertEquals(new Integer(1000), deserializedMap.values().iterator().next());
}
- @Test
- public void shouldSerializeFullResponseMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final Map<String, Object> metaData = new HashMap<>();
- metaData.put("test", "this");
- metaData.put("one", 1);
-
- final Map<String, Object> attributes = new HashMap<>();
- attributes.put("test", "that");
- attributes.put("two", 2);
-
- final ResponseMessage response = ResponseMessage.build(id)
- .responseMetaData(metaData)
- .code(ResponseStatusCode.SUCCESS)
- .result("some-result")
- .statusAttributes(attributes)
- .statusMessage("worked")
- .create();
-
- final ByteBuf bb = binarySerializer.serializeResponseAsBinary(response, allocator);
- final ResponseMessage deserialized = binarySerializer.deserializeResponse(bb);
-
- assertEquals(id, deserialized.getRequestId());
- assertEquals("this", deserialized.getResult().getMeta().get("test"));
- assertEquals(1, deserialized.getResult().getMeta().get("one"));
- assertEquals("some-result", deserialized.getResult().getData());
- assertEquals("that", deserialized.getStatus().getAttributes().get("test"));
- assertEquals(2, deserialized.getStatus().getAttributes().get("two"));
- assertEquals(ResponseStatusCode.SUCCESS.getValue(), deserialized.getStatus().getCode().getValue());
- assertEquals("worked", deserialized.getStatus().getMessage());
- }
-
- @Test
- public void shouldHaveTooSmallBufferToSerializeResponseMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final Map<String, Object> metaData = new HashMap<>();
- metaData.put("test", "this");
- metaData.put("one", 1);
-
- final Map<String, Object> attributes = new HashMap<>();
- attributes.put("test", "that");
- attributes.put("two", 2);
-
- final ResponseMessage response = ResponseMessage.build(id)
- .responseMetaData(metaData)
- .code(ResponseStatusCode.SUCCESS)
- .result("some-result")
- .statusAttributes(attributes)
- .statusMessage("worked")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- put("bufferSize", 1);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- try {
- binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
- fail("Should have a buffer size that is too small");
- } catch (Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertThat(root, instanceOf(KryoException.class));
- }
- }
-
- @Test
- public void shouldReturnAllBytesInResponse() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final Map<String, Object> metaData = new HashMap<>();
- metaData.put("test", "this");
- metaData.put("one", 1);
-
- final Map<String, Object> attributes = new HashMap<>();
- attributes.put("test", "that");
- attributes.put("two", 2);
-
- final ResponseMessage response = ResponseMessage.build(id)
- .responseMetaData(metaData)
- .code(ResponseStatusCode.SUCCESS)
- .result("some-result")
- .statusAttributes(attributes)
- .statusMessage("worked")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- // set to bufferSize < total message size but still greater than any individual object requires
- put("bufferSize", 50);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- ByteBuf buf = binarySerializerWithSmallBuffer.serializeResponseAsBinary(response, allocator);
- assertTrue(buf.isReadable());
- assertEquals(82, buf.readableBytes());
- }
-
- @Test
- public void shouldSerializeFullRequestMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final RequestMessage request = RequestMessage.build("try")
- .overrideRequestId(id)
- .processor("pro")
- .addArg("test", "this")
- .create();
- final ByteBuf bb = binarySerializer.serializeRequestAsBinary(request, allocator);
- final int mimeLen = bb.readByte();
- bb.readBytes(new byte[mimeLen]);
- final RequestMessage deserialized = binarySerializer.deserializeRequest(bb);
-
- assertEquals(id, deserialized.getRequestId());
- assertEquals("pro", deserialized.getProcessor());
- assertEquals("try", deserialized.getOp());
- assertEquals("this", deserialized.getArgs().get("test"));
- }
-
- @Test
- public void shouldHaveTooSmallBufferToSerializeRequestMessage() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final RequestMessage request = RequestMessage.build("try")
- .overrideRequestId(id)
- .processor("pro")
- .addArg("test", "this")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- put("bufferSize", 1);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- try {
- binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
- fail("Should have a buffer size that is too small");
- } catch (Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertThat(root, instanceOf(KryoException.class));
- }
- }
-
- @Test
- public void shouldReturnAllBytesInRequest() throws Exception {
- final UUID id = UUID.randomUUID();
-
- final RequestMessage request = RequestMessage.build("try")
- .overrideRequestId(id)
- .processor("pro")
- .addArg("test", "this")
- .create();
-
- final MessageSerializer binarySerializerWithSmallBuffer = new GryoMessageSerializerV1d0();
- final Map<String, Object> configWithSmallBuffer = new HashMap<String, Object>() {{
- // set to bufferSize < total message size but still greater than any individual object requires
- put("bufferSize", 50);
- }};
- binarySerializerWithSmallBuffer.configure(configWithSmallBuffer, null);
-
- ByteBuf buf = binarySerializerWithSmallBuffer.serializeRequestAsBinary(request, allocator);
- assertTrue(buf.isReadable());
- assertEquals(71, buf.readableBytes());
- }
-
private void assertCommon(final ResponseMessage response) {
assertEquals(requestId, response.getRequestId());
assertEquals(ResponseStatusCode.SUCCESS, response.getStatus().getCode());
@@ -546,57 +160,4 @@ public class GryoMessageSerializerV1d0Test {
final ByteBuf bb = binarySerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
return binarySerializer.deserializeResponse(bb);
}
-
- private ResponseMessage convertText(final Object toSerialize) throws SerializationException {
- final ByteBuf bb = textSerializer.serializeResponseAsBinary(responseMessageBuilder.result(toSerialize).create(), allocator);
- return textSerializer.deserializeResponse(bb);
- }
-
- public static class ErrorOnlyClassResolverSupplierAsInstance implements Supplier<ClassResolver> {
-
- private static final ErrorOnlyClassResolverSupplierAsInstance instance = new ErrorOnlyClassResolverSupplierAsInstance();
-
- private ErrorOnlyClassResolverSupplierAsInstance() {}
-
- public static ErrorOnlyClassResolverSupplierAsInstance getInstance() {
- return instance;
- }
-
- @Override
- public ClassResolver get() {
- return new ErrorOnlyClassResolver();
- }
- }
-
- public static class ErrorOnlyClassResolverSupplier implements Supplier<ClassResolver> {
- @Override
- public ClassResolver get() {
- return new ErrorOnlyClassResolver();
- }
- }
-
- public static class ErrorOnlyClassResolver extends GryoClassResolver {
- @Override
- public Registration getRegistration(Class clazz) {
- throw new RuntimeException("Registration is not allowed with this ClassResolver - it is not a good implementation");
- }
- }
-
- public static class ColorIoRegistry extends AbstractIoRegistry {
- public ColorIoRegistry() {
- register(GryoIo.class, Color.class, new ColorSerializer());
- }
- }
-
- public static class ColorSerializer extends Serializer<Color> {
- @Override
- public void write(final Kryo kryo, final Output output, final Color color) {
- output.write(color.equals(Color.RED) ? 1 : 0);
- }
-
- @Override
- public Color read(final Kryo kryo, final Input input, final Class<Color> aClass) {
- return input.read() == 1 ? Color.RED : Color.BLACK;
- }
- }
}
[19/27] incubator-tinkerpop git commit: This closes #195
Posted by sp...@apache.org.
This closes #195
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/afd40488
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/afd40488
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/afd40488
Branch: refs/heads/TINKERPOP-1308
Commit: afd40488fb50d4a9e92419e63bcbd73187eb2912
Parents: 366edb3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 24 17:07:43 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 24 17:07:43 2016 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[16/27] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/TINKERPOP-1088' into tp31
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/TINKERPOP-1088' into tp31
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/55a509f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/55a509f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/55a509f2
Branch: refs/heads/TINKERPOP-1308
Commit: 55a509f2b1fb9d9f1787940197f99c95beb4d0d0
Parents: aa447eb 636b2e5
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 24 14:47:54 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 24 14:47:54 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../traversal/step/sideEffect/SubgraphStep.java | 19 +++++-----
.../step/sideEffect/GroovySubgraphTest.groovy | 5 +++
.../traversal/step/sideEffect/SubgraphTest.java | 38 ++++++++++++++++++--
4 files changed, 52 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/55a509f2/CHANGELOG.asciidoc
----------------------------------------------------------------------
[09/27] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1305'
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-1305'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/9c53c73e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/9c53c73e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/9c53c73e
Branch: refs/heads/TINKERPOP-1308
Commit: 9c53c73e6475d2c701833648a18eaff029626087
Parents: 3c613d9 5bcf0b0
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri May 20 22:08:41 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri May 20 22:08:41 2016 -0600
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 30 ++-
.../process/computer/GiraphGraphComputer.java | 6 +
.../gremlin/process/computer/ProgramPhase.java | 40 ++++
.../peerpressure/PeerPressureVertexProgram.java | 21 +-
.../traversal/MemoryTraversalSideEffects.java | 28 +--
.../computer/traversal/SingleMessenger.java | 49 -----
.../traversal/TraversalVertexProgram.java | 78 +++++--
.../computer/traversal/TraverserExecutor.java | 203 -----------------
.../computer/traversal/WorkerExecutor.java | 220 +++++++++++++++++++
.../traversal/step/VertexComputing.java | 6 +-
.../step/map/PageRankVertexProgramStep.java | 5 +-
.../step/map/PeerPressureVertexProgramStep.java | 13 +-
.../step/map/ProgramVertexProgramStep.java | 6 +-
.../step/map/TraversalVertexProgramStep.java | 10 +-
.../traversal/step/map/VertexProgramStep.java | 21 +-
.../optimization/GraphFilterStrategy.java | 3 +-
.../process/computer/util/EmptyMemory.java | 91 ++++++++
.../process/computer/util/SingleMessenger.java | 49 +++++
.../traversal/step/util/ImmutablePath.java | 3 +-
.../step/map/GroovyPageRankTest.groovy | 5 +
.../step/map/GroovyPeerPressureTest.groovy | 5 +
.../traversal/step/map/PageRankTest.java | 26 +++
.../traversal/step/map/PeerPressureTest.java | 31 +++
.../process/traversal/step/map/ProgramTest.java | 49 +++--
.../process/computer/SparkGraphComputer.java | 7 +-
.../optimization/SparkInterceptorStrategy.java | 3 +-
.../SparkSingleIterationStrategy.java | 3 +-
.../SparkStarBarrierInterceptor.java | 3 +-
28 files changed, 651 insertions(+), 363 deletions(-)
----------------------------------------------------------------------