You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/10/09 17:50:40 UTC
[23/30] incubator-tinkerpop git commit: Minor optimizations to
GroupSideEffectStep. Benchmarks conclude that the new way is better than the
old way. Faster in most cases and no OME issues.
Minor optimizations to GroupSideEffectStep. Benchmarks conclude that the new way is better than the old way. Faster in most cases and no OME issues.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/00439392
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/00439392
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/00439392
Branch: refs/heads/TINKERPOP3-876
Commit: 00439392a96e4339b4b2b5fb328641ce83ccba00
Parents: f8d2c62
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Oct 8 17:37:16 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Oct 8 17:37:16 2015 -0600
----------------------------------------------------------------------
.../process/traversal/step/map/GroupStep.java | 2 +-
.../step/sideEffect/GroupSideEffectStep.java | 20 ++++++++++++++------
.../structure/TinkerGraphPlayTest.java | 4 ++--
3 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/00439392/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
index d050387..32a16a7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
@@ -166,7 +166,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
}
traversal.addStart(traverser);
- int count = this.counters.compute(key, (k, i) -> ++i);
+ final int count = this.counters.compute(key, (k, i) -> ++i);
if (count > 10000) {
this.counters.put(key, 0);
TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, traversal).ifPresent(Barrier::processAllStarts);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/00439392/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
index ab1268f..b2885b9 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroupSideEffectStep.java
@@ -28,11 +28,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.EngineDependent;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.SideEffectCapable;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
-import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.GroupStepHelper;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
@@ -46,6 +46,7 @@ import org.apache.tinkerpop.gremlin.util.function.HashMapSupplier;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -66,6 +67,7 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
private String sideEffectKey;
private boolean onGraphComputer = false;
private GroupStepHelper.GroupMap<S, K, V> groupMap;
+ private final Map<K, Integer> counters = new HashMap<>();
public GroupSideEffectStep(final Traversal.Admin traversal, final String sideEffectKey) {
super(traversal);
@@ -99,7 +101,7 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
values = new BulkSet<>();
map.put(key, values);
}
- this.valueTraversal.addStart(traverser); // the full traverser is provided (not a bulk 1 traverser)
+ this.valueTraversal.addStart(traverser.clone()); // the full traverser is provided (not a bulk 1 traverser) and clone is needed cause sideEffect steps don't split the traverser
this.valueTraversal.fill((Collection) values);
} else { // OLTP
if (null == this.groupMap) {
@@ -112,9 +114,14 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
if (null == traversal) {
traversal = this.valueReduceTraversal.clone();
this.groupMap.put(key, traversal);
+ this.counters.put(key, 0);
+ }
+ traversal.addStart(traverser.clone()); // this is because sideEffect steps don't split the traverser
+ final int count = this.counters.compute(key, (k, i) -> ++i);
+ if (count > 10000) {
+ this.counters.put(key, 0);
+ TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, traversal).ifPresent(Barrier::processAllStarts);
}
- traversal.addStart(traverser);
- TraversalHelper.getStepsOfAssignableClass(Barrier.class, traversal).stream().findFirst().ifPresent(Barrier::processAllStarts);
}
}
@@ -225,9 +232,10 @@ public final class GroupSideEffectStep<S, K, V> extends SideEffectStep<S> implem
@Override
public void reduce(final K key, final Iterator<Collection<?>> values, final ReduceEmitter<K, V> emitter) {
- Traversal.Admin<?, V> reduceTraversalClone = this.reduceTraversal.clone();
+ Traversal.Admin<?,V> reduceTraversalClone = this.reduceTraversal.clone();
while (values.hasNext()) {
- reduceTraversalClone.addStarts(reduceTraversalClone.getTraverserGenerator().generateIterator(values.next().iterator(), (Step) reduceTraversalClone.getStartStep(), 1l));
+ final BulkSet<?> value = (BulkSet<?>) values.next();
+ value.forEach((v,bulk) -> reduceTraversalClone.addStart(reduceTraversalClone.getTraverserGenerator().generate(v, (Step) reduceTraversalClone.getStartStep(), bulk)));
TraversalHelper.getFirstStepOfAssignableClass(Barrier.class, reduceTraversalClone).ifPresent(Barrier::processAllStarts);
}
emitter.emit(key, reduceTraversalClone.next());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/00439392/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
index f86125a..9f815ea 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
@@ -49,12 +49,12 @@ public class TinkerGraphPlayTest {
@Ignore
public void benchmarkGroup() throws Exception {
Graph graph = TinkerGraph.open();
- GraphTraversalSource g = graph.traversal(GraphTraversalSource.computer());
+ GraphTraversalSource g = graph.traversal();
graph.io(GraphMLIo.build()).readGraph("data/grateful-dead.xml");
/////////
- System.out.println("groupV3d0: " + g.V().both("followedBy").both("followedBy").groupV3d0().by("songType").by().by(__.count(Scope.local)).next());
System.out.println("group: " + g.V().both("followedBy").both("followedBy").group().by("songType").by(count()).next());
+ System.out.println("groupV3d0: " + g.V().both("followedBy").both("followedBy").groupV3d0().by("songType").by().by(__.count(Scope.local)).next());
//
System.out.println("\n\nBig Values -- by(songType)");