You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/09 18:14:53 UTC
[1/8] incubator-tinkerpop git commit: Made significant memory
improvements to TraverserExecutor. Realized some massive heaps on some jobs
on Friendster using SparkGraphComputer and tracked it down to how I'm dealing
with traversers in TraverserVertexProg
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master e8a7e3a91 -> 26aedc746
Made significant memory improvements to TraverserExecutor. Realized some massive heaps on some jobs on Friendster using SparkGraphComputer and tracked it down to how I'm dealing with traversers in TraverserVertexProgram. I was not 'draining' sets of traversers and thus, was using an excessive amount of memory. This really shows itself when touching edges where you can easily generate million of traversers and to have multiple copies of that data is bad. To make draining work, I had to update all the Iterators to support .remove() which simply call .remove() of the child iterator. Found a simple optimization for CountGlobalStep that will make OLAP counting much faster.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b54ddf28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b54ddf28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b54ddf28
Branch: refs/heads/master
Commit: b54ddf2830483705c4c4a865b9f2586ed457223a
Parents: 12b9b21
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Feb 5 14:56:10 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Feb 5 14:56:10 2016 -0700
----------------------------------------------------------------------
.../computer/traversal/TraverserExecutor.java | 101 +++++++++++--------
.../traversal/step/map/CountGlobalStep.java | 10 ++
.../gremlin/util/iterator/DoubleIterator.java | 12 ++-
.../gremlin/util/iterator/IteratorUtils.java | 25 +++++
.../gremlin/util/iterator/MultiIterator.java | 7 +-
.../gremlin/util/iterator/SingleIterator.java | 9 +-
6 files changed, 118 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b54ddf28/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
index 038e309..2df682c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.computer.Messenger;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMatrix;
import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -31,6 +32,7 @@ import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.Attachable;
+import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -42,58 +44,75 @@ public final class TraverserExecutor {
final TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS);
final AtomicBoolean voteToHalt = new AtomicBoolean(true);
-
final TraverserSet<Object> aliveTraversers = new TraverserSet<>();
- // gather incoming traversers into a traverser set and gain the 'weighted-set' optimization
+ final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
+
+ final Iterator<TraverserSet<?>> messages = messenger.receiveMessages();
final TraversalSideEffects traversalSideEffects = traversalMatrix.getTraversal().getSideEffects();
- messenger.receiveMessages().forEachRemaining(traverserSet -> {
- traverserSet.forEach(traverser -> {
+ while (messages.hasNext()) {
+ final Iterator<Traverser.Admin<Object>> traversers = (Iterator) messages.next().iterator();
+ messages.remove();
+ while (traversers.hasNext()) {
+ final Traverser.Admin<Object> traverser = traversers.next();
+ traversers.remove();
traverser.setSideEffects(traversalSideEffects);
traverser.attach(Attachable.Method.get(vertex));
- aliveTraversers.add((Traverser.Admin) traverser);
- });
- });
-
+ toProcessTraversers.add((Traverser.Admin) traverser);
+ }
+ }
// while there are still local traversers, process them until they leave the vertex or halt (i.e. isHalted()).
- final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
- while (!aliveTraversers.isEmpty()) {
+ Step<?, ?> previousStep = EmptyStep.instance();
+ while (!toProcessTraversers.isEmpty()) {
+ // process local traversers and if alive, repeat, else halt.
+ Iterator<Traverser.Admin<Object>> traversers = toProcessTraversers.iterator();
+ while (traversers.hasNext()) {
+ final Traverser.Admin<Object> traverser = traversers.next();
+ traversers.remove();
+ final Step<?, ?> currentStep = traversalMatrix.getStepById(traverser.getStepId());
+ if (!currentStep.getId().equals(previousStep.getId()))
+ TraverserExecutor.drainStep(previousStep, aliveTraversers, haltedTraversers);
+ currentStep.addStart((Traverser.Admin) traverser);
+ previousStep = currentStep;
+ }
+ TraverserExecutor.drainStep(previousStep, aliveTraversers, haltedTraversers);
+ assert toProcessTraversers.isEmpty();
// process all the local objects and send messages or store locally again
- aliveTraversers.forEach(traverser -> {
- if (traverser.get() instanceof Element || traverser.get() instanceof Property) { // GRAPH OBJECT
- // if the element is remote, then message, else store it locally for re-processing
- final Vertex hostingVertex = TraverserExecutor.getHostingVertex(traverser.get());
- if (!vertex.equals(hostingVertex)) { // necessary for path access
- voteToHalt.set(false);
- traverser.detach();
- messenger.sendMessage(MessageScope.Global.of(hostingVertex), new TraverserSet<>(traverser));
- } else {
- if (traverser.get() instanceof Attachable) // necessary for path access to local object
- traverser.attach(Attachable.Method.get(vertex));
+ if (!aliveTraversers.isEmpty()) {
+ traversers = aliveTraversers.iterator();
+ while (traversers.hasNext()) {
+ final Traverser.Admin<Object> traverser = traversers.next();
+ traversers.remove();
+ if (traverser.get() instanceof Element || traverser.get() instanceof Property) { // GRAPH OBJECT
+ // if the element is remote, then message, else store it locally for re-processing
+ final Vertex hostingVertex = TraverserExecutor.getHostingVertex(traverser.get());
+ if (!vertex.equals(hostingVertex)) { // necessary for path access
+ voteToHalt.set(false);
+ traverser.detach();
+ messenger.sendMessage(MessageScope.Global.of(hostingVertex), new TraverserSet<>(traverser));
+ } else {
+ if (traverser.get() instanceof Attachable) // necessary for path access to local object
+ traverser.attach(Attachable.Method.get(vertex));
+ toProcessTraversers.add(traverser);
+ }
+ } else // STANDARD OBJECT
toProcessTraversers.add(traverser);
- }
- } else // STANDARD OBJECT
- toProcessTraversers.add(traverser);
- });
-
- // process local traversers and if alive, repeat, else halt.
- aliveTraversers.clear();
- toProcessTraversers.forEach(start -> {
- final Step<?, ?> step = traversalMatrix.getStepById(start.getStepId());
- step.addStart((Traverser.Admin) start);
- step.forEachRemaining(end -> {
- if (end.asAdmin().isHalted()) {
- end.asAdmin().detach();
- haltedTraversers.add((Traverser.Admin) end);
- } else
- aliveTraversers.add((Traverser.Admin) end);
- });
- });
-
- toProcessTraversers.clear();
+ }
+ assert aliveTraversers.isEmpty();
+ }
}
return voteToHalt.get();
}
+ private static void drainStep(final Step<?, ?> step, final TraverserSet<?> aliveTraversers, final TraverserSet<?> haltedTraversers) {
+ step.forEachRemaining(traverser -> {
+ if (traverser.asAdmin().isHalted()) {
+ traverser.asAdmin().detach();
+ haltedTraversers.add((Traverser.Admin) traverser);
+ } else
+ aliveTraversers.add((Traverser.Admin) traverser);
+ });
+ }
+
private static Vertex getHostingVertex(final Object object) {
Object obj = object;
while (true) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b54ddf28/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
index 409c445..d090832 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
@@ -61,6 +61,16 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
return CountGlobalMapReduce.instance();
}
+ @Override
+ public Traverser<Long> processNextStart() {
+ if (this.byPass) {
+ final Traverser.Admin<S> traverser = this.starts.next();
+ return traverser.asAdmin().split(1l, this); // if bypassing, just key all the traversers to 1 long (the count is going to be the bulk of course)
+ } else {
+ return super.processNextStart();
+ }
+ }
+
///////////
private static class CountBiFunction<S> implements BiFunction<Long, Traverser<S>, Long>, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b54ddf28/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/DoubleIterator.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/DoubleIterator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/DoubleIterator.java
index e07bcef..2d7982c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/DoubleIterator.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/DoubleIterator.java
@@ -28,8 +28,8 @@ import java.util.Iterator;
*/
final class DoubleIterator<T> implements Iterator<T>, Serializable {
- private final T a;
- private final T b;
+ private T a;
+ private T b;
private char current = 'a';
protected DoubleIterator(final T a, final T b) {
@@ -43,6 +43,14 @@ final class DoubleIterator<T> implements Iterator<T>, Serializable {
}
@Override
+ public void remove() {
+ if (this.current == 'b')
+ this.a = null;
+ else if (this.current == 'x')
+ this.b = null;
+ }
+
+ @Override
public T next() {
if (this.current == 'x')
throw FastNoSuchElementException.instance();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b54ddf28/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
index dc38a07..808317f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
@@ -105,6 +105,11 @@ public final class IteratorUtils {
}
@Override
+ public void remove() {
+ iterator.remove();
+ }
+
+ @Override
public S next() {
if (this.count++ >= limit)
throw FastNoSuchElementException.instance();
@@ -205,6 +210,11 @@ public final class IteratorUtils {
}
@Override
+ public void remove() {
+ iterator.remove();
+ }
+
+ @Override
public S next() {
final S s = iterator.next();
consumer.accept(s);
@@ -228,6 +238,11 @@ public final class IteratorUtils {
}
@Override
+ public void remove() {
+ iterator.remove();
+ }
+
+ @Override
public E next() {
return function.apply(iterator.next());
}
@@ -257,6 +272,11 @@ public final class IteratorUtils {
}
@Override
+ public void remove() {
+ iterator.remove();
+ }
+
+ @Override
public S next() {
try {
if (null != this.nextResult) {
@@ -312,6 +332,11 @@ public final class IteratorUtils {
}
@Override
+ public void remove() {
+ iterator.remove();
+ }
+
+ @Override
public E next() {
if (this.hasNext())
return this.currentIterator.next();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b54ddf28/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
index 02c15f4..e272a08 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
@@ -42,7 +42,7 @@ public final class MultiIterator<T> implements Iterator<T>, Serializable {
if (this.current >= this.iterators.size())
return false;
- Iterator<T> currentIterator = iterators.get(this.current);
+ Iterator<T> currentIterator = this.iterators.get(this.current);
while (true) {
if (currentIterator.hasNext()) {
@@ -58,6 +58,11 @@ public final class MultiIterator<T> implements Iterator<T>, Serializable {
}
@Override
+ public void remove() {
+ // this.iterators.get(this.current).remove();
+ }
+
+ @Override
public T next() {
if (this.iterators.isEmpty()) throw FastNoSuchElementException.instance();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b54ddf28/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/SingleIterator.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/SingleIterator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/SingleIterator.java
index b680819..bb2da72 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/SingleIterator.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/SingleIterator.java
@@ -26,9 +26,9 @@ import java.util.Iterator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-final class SingleIterator<T> implements Iterator<T>,Serializable {
+final class SingleIterator<T> implements Iterator<T>, Serializable {
- private final T t;
+ private T t;
private boolean alive = true;
protected SingleIterator(final T t) {
@@ -41,6 +41,11 @@ final class SingleIterator<T> implements Iterator<T>,Serializable {
}
@Override
+ public void remove() {
+ this.t = null;
+ }
+
+ @Override
public T next() {
if (!this.alive)
throw FastNoSuchElementException.instance();
[6/8] incubator-tinkerpop git commit: Set up 3.1.2-incubating in
CHANGELOG.
Posted by ok...@apache.org.
Set up 3.1.2-incubating in CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1286b4f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1286b4f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1286b4f1
Branch: refs/heads/master
Commit: 1286b4f1997f9de4624b20f3be8497983ef46492
Parents: a3e2a98
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Feb 9 11:55:31 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Feb 9 11:55:31 2016 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1286b4f1/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 38f6169..6b51ba8 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -22,6 +22,12 @@ TinkerPop 3.1.0 (A 187 On The Undercover Gremlinz)
image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/static/images/gremlin-gangster.png[width=185]
+[[release-3.1.2-incubating]]
+TinkerPop 3.1.2 (NOT OFFICIALLY RELEASED YET)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+
[[release-3.1.1-incubating]]
TinkerPop 3.1.1 (Release Date: February 8, 2016)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[5/8] incubator-tinkerpop git commit: Tweak release docs slightly
given minor lessons learned.
Posted by ok...@apache.org.
Tweak release docs slightly given minor lessons learned.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a3e2a98a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a3e2a98a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a3e2a98a
Branch: refs/heads/master
Commit: a3e2a98ab5004aaec47b4fbb7c9295fce556b4f0
Parents: 7c98a36
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Feb 8 11:29:59 2016 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Feb 8 11:29:59 2016 -0500
----------------------------------------------------------------------
docs/src/dev/developer/release.asciidoc | 67 ++++++++++++++--------------
1 file changed, 34 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a3e2a98a/docs/src/dev/developer/release.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/developer/release.asciidoc b/docs/src/dev/developer/release.asciidoc
index 766a7e7..0069acc 100644
--- a/docs/src/dev/developer/release.asciidoc
+++ b/docs/src/dev/developer/release.asciidoc
@@ -69,13 +69,13 @@ might be high.
.. `mvn verify -DskipIntegrationTests=false -DincludeNeo4j`
.. `mvn verify -DskipPerformanceTests=false`
. `bin/publish-docs.sh <username>` - note that under a release candidate the documentation is published as SNAPSHOT
-. `mvn versions:set -DnewVersion=x.y.z -DgenerateBackupPoms=false` to update the project files to reference a non-SNAPSHOT version
+. `mvn versions:set -DnewVersion=xx.yy.zz -DgenerateBackupPoms=false` to update the project files to reference a non-SNAPSHOT version
. `git diff` and review the updated files (expect all `pom.xml` files and this README)
-. `git commit -a -m "TinkerPop x.y.z release"` and `git push`
-. `git tag -a -m "TinkerPop x.y.z release" x.y.z` and `git push --tags`
-. `mvn clean install -Dmaven.test.skip=true`
-. `mvn versions:set -DnewVersion=x.y.z-SNAPSHOT -DgenerateBackupPoms=false` to go back to SNAPSHOT
-. `git commit -a -m "Returned to x.y.z-SNAPSHOT"` and `git push`
+. `git commit -a -m "TinkerPop xx.yy.zz release"` and `git push`
+. `git tag -a -m "TinkerPop xx.yy.zz release" xx.yy.zz` and `git push --tags`
+. `mvn clean install`
+. `mvn versions:set -DnewVersion=xx.yy.zz-SNAPSHOT -DgenerateBackupPoms=false` to go back to SNAPSHOT
+. `git commit -a -m "Returned to xx.yy.zz-SNAPSHOT"` and `git push`
. Announce the release candidate to `dev` mailing list and await feedback
. Repeat as required or proceed to the next phase
@@ -102,36 +102,37 @@ current release was under development as this new release will have those change
. Update "upgrade documentation":
.. Update the release date.
.. Update the link to CHANGELOG.asciidoc
-. `mvn versions:set -DnewVersion=x.y.z -DgenerateBackupPoms=false` to update project files to reference the non-SNAPSHOT version
+. `mvn versions:set -DnewVersion=xx.yy.zz -DgenerateBackupPoms=false` to update project files to reference the non-SNAPSHOT version
. `git diff` and review the updated files (expect all `pom.xml` files and this README)
-. `git commit -a -m "TinkerPop x.y.z release"` and `git push`
-. `mvn clean install -Dmaven.test.skip=true` - need to build first so that the right version of the console is used with `bin/publish-docs.sh`
-. `bin/process-docs.sh` and validate the generated documentation locally
+. `git commit -a -m "TinkerPop xx.yy.zz release"` and push
+. `mvn clean install` - need to build first so that the right version of the console is used with `bin/publish-docs.sh`
+. `bin/process-docs.sh` and validate the generated documentation locally (don't rely on "SUCCESS" - scroll up through logs to ensure there were no errors and view the HTML directly)
. `bin/publish-docs.sh <username>` - Note that this step requires no additional processing as the previous step.
handled document generation and this step now merely needs to upload what was generated.
-. `mvn deploy -Papache-release -DcreateChecksum=true -Dmaven.test.skip=true` - deploy signed artifacts with checksums to link:https://repository.apache.org/[Apache Nexus]. Review (artifacts versions, file sizes, anything that might be out of place - request another committer to review as well) but do NOT close/release the staging repository at this time.
+. `mvn deploy -Papache-release -DcreateChecksum=true` - deploy signed artifacts with checksums to link:https://repository.apache.org/[Apache Nexus]. Review (artifacts versions, file sizes, anything that might be out of place - request another committer to review as well) but do NOT close/release the staging repository at this time.
. Review generated artifacts to be sure they have both javadocs and asciidocs present then "close" the repo - if the repo is left open it will be automatically dropped after five days and closing the repo will allow it to stay available for a full ninety days which is more than enough time to complete a vote.
. Upload artifacts to `https://dist.apache.org/repos/dist/dev/incubator/tinkerpop` for `[VOTE]` review.
-.. `svn co --depth empty https://dist.apache.org/repos/dist/dev/incubator/tinkerpop/ dev` and `mkdir dev/x.y.z`
-.. `cp ~/.m2/repository/org/apache/tinkerpop/gremlin-console/x.y.z/gremlin-console-x.y.z-distribution.zip* dev/x.y.z`
-.. `cp ~/.m2/repository/org/apache/tinkerpop/gremlin-server/x.y.z/gremlin-server-x.y.z-distribution.zip* dev/x.y.z`
-.. `cp ~/.m2/repository/org/apache/tinkerpop/tinkerpop/x.y.z/tinkerpop-x.y.z-source-release.zip* dev/x.y.z`
-.. `cd dev/x.y.z`
+.. `svn co --depth empty https://dist.apache.org/repos/dist/dev/incubator/tinkerpop/ dev` and `mkdir dev/xx.yy.zz`
+.. `cp ~/.m2/repository/org/apache/tinkerpop/gremlin-console/xx.yy.zz/gremlin-console-xx.yy.zz-distribution.zip* dev/xx.yy.zz`
+.. `cp ~/.m2/repository/org/apache/tinkerpop/gremlin-server/xx.yy.zz/gremlin-server-xx.yy.zz-distribution.zip* dev/xx.yy.zz`
+.. `cp ~/.m2/repository/org/apache/tinkerpop/tinkerpop/xx.yy.zz/tinkerpop-xx.yy.zz-source-release.zip* dev/xx.yy.zz`
+.. `cd dev/xx.yy.zz`
.. pass:[<code>ls * | xargs -n1 -I {} echo "mv apache-{} {}" | sed -e 's/distribution/bin/' -e 's/source-release/src/' -e s'/^\(.*\) \(.*\) \(.*\)$/\1 \3 \2/' | /bin/bash</code>]
-.. `cd ..; svn add x.y.z/; svn ci -m "TinkerPop x.y.z release"`
+.. `cd ..; svn add xx.yy.zz/; svn ci -m "TinkerPop xx.yy.zz release"`
. Execute `bin/validate-distribution.sh` and any other relevant testing.
-. `git tag -a -m "TinkerPop x.y.z release" x.y.z` and `git push --tags`
+. `git tag -a -m "TinkerPop xx.yy.zz release" xx.yy.zz` and `git push --tags`
. Perform JIRA administration tasks:
.. "Release" the current version and set the "release date"
.. If there is to be a follow on release in the current line of code, create that new version specifying the "start date"
. Prepare Git administration tasks (apply the following steps as needed per release branch):
.. Make the appropriate branching changes as required by the release and bump the version to `SNAPSHOT` with
`mvn versions:set -DnewVersion=xx.yy.zz-SNAPSHOT -DgenerateBackupPoms=false`.
-.. `mvn clean install -Dmaven.test.skip=true` - need to build first so that the right version of the console is used with `bin/publish-docs.sh`
-.. `mvn deploy -DskipTests` - deploy the new `SNAPSHOT`
+.. `mvn clean install` - need to build first so that the right version of the console is used with `bin/publish-docs.sh`
+.. `mvn deploy` - deploy the new `SNAPSHOT`
.. `bin/process-docs.sh` and validate the generated `SNAPSHOT` documentation locally
.. `bin/publish-docs.sh <username>` to publish the `SNAPSHOT` docs which enables the README to work properly.
.. Update the links in the `README.asciidoc` to point at the `SNAPSHOT` version.
+.. Commit and push the `SNAPSHOT` changes to git
. Submit for `[VOTE]` at `dev@tinkerpop.apache.org` (see email template below)
. *Wait for vote acceptance* (72 hours)
@@ -141,16 +142,16 @@ Incubator Vote
A positive vote for a particular release from the Apache Incubator is required to move to the following phase.
. Submit for `[VOTE]` at `general@incubator.apache.org` (see email template below)
-.. Include the vote tally: "Apache TinkerPop (http://tinkerpop.apache.org/) would like to release TinkerPop x.y.z. We had a dev@ VOTE which resulted in a tally of +1 (3), 0 (0), and -1 (0). We now present our artifacts for vote by Incubator."
+.. Include the vote tally: "Apache TinkerPop (http://tinkerpop.apache.org/) would like to release TinkerPop xx.yy.zz. We had a dev@ VOTE which resulted in a tally of +1 (3), 0 (0), and -1 (0). We now present our artifacts for vote by Incubator."
. *Wait for vote acceptance* (72 hours)
Release & Promote
-----------------
. Close the staging repository at link:https://repository.apache.org/[Apache Nexus]) and then release.
-. `svn co --depth empty https://dist.apache.org/repos/dist/dev/incubator/tinkerpop dev; svn up dev/x.y.z`
-. `svn co --depth empty https://dist.apache.org/repos/dist/release/incubator/tinkerpop release; mkdir release/x.y.z`
-. `cd release; svn add x.y.z/; svn ci -m "TinkerPop x.y.z release"`
+. `svn co --depth empty https://dist.apache.org/repos/dist/dev/incubator/tinkerpop dev; svn up dev/xx.yy.zz`
+. `svn co --depth empty https://dist.apache.org/repos/dist/release/incubator/tinkerpop release; mkdir release/xx.yy.zz`
+. `cd release; svn add xx.yy.zz/; svn ci -m "TinkerPop xx.yy.zz release"`
. If there is are releases present in SVN that represents lines of code that are no longer under development, then remove those releases. In other words, if `3.1.0-incubating` is present and `3.1.1-incubating` is released then remove `3.1.0-incubating`. However, if `3.0.2-incubating` is present and that line of code is still under potential development, it may stay.
. Update homepage with references to latest distribution and to other internal links elsewhere on the page.
. Wait for Apache Central to sync the jars and src (link:http://repo1.maven.org/maven2/org/apache/tinkerpop/tinkerpop/[http://repo1.maven.org/maven2/org/apache/tinkerpop/tinkerpop/]).
@@ -159,25 +160,25 @@ Release & Promote
Example `[VOTE]` email:
```
-[VOTE] TinkerPop x.y.z Release
+[VOTE] TinkerPop xx.yy.zz Release
Hello,
The release artifacts can be found at this location:
- https://dist.apache.org/repos/dist/dev/incubator/tinkerpop/x.y.z/
+ https://dist.apache.org/repos/dist/dev/incubator/tinkerpop/xx.yy.zz/
The source distribution is provided by:
- apache-tinkerpop-x.y.z-src.zip
+ apache-tinkerpop-xx.yy.zz-src.zip
Two binary distributions are provided for user convenience:
- apache-gremlin-console-x.y.z-bin.zip
- apache-gremlin-server-x.y.z-bin.zip
+ apache-gremlin-console-xx.yy.zz-bin.zip
+ apache-gremlin-server-xx.yy.zz-bin.zip
The online docs can be found here:
- http://tinkerpop.apache.org/docs/x.y.z/ (user docs)
- http://tinkerpop.apache.org/docs/x.y.z/upgrade.html (upgrade docs)
- http://tinkerpop.apache.org/javadocs/x.y.z/core/ (core javadoc)
- http://tinkerpop.apache.org/javadocs/x.y.z/full/ (full javadoc)
+ http://tinkerpop.apache.org/docs/xx.yy.zz/reference/ (user docs)
+ http://tinkerpop.apache.org/docs/xx.yy.zz/upgrade/ (upgrade docs)
+ http://tinkerpop.apache.org/javadocs/xx.yy.zz/core/ (core javadoc)
+ http://tinkerpop.apache.org/javadocs/xx.yy.zz/full/ (full javadoc)
The tag in Apache Git can be found here:
https://git-wip-us.apache.org/repos/asf?p=incubator-tinkerpop.git;...
[7/8] incubator-tinkerpop git commit: merge tweaks. updated CHANGELOG.
Posted by ok...@apache.org.
merge tweaks. updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b07400cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b07400cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b07400cc
Branch: refs/heads/master
Commit: b07400ccb8c1154b75440ae1ad96c12c78e74680
Parents: 1286b4f ba22c7a
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Feb 9 10:09:01 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Feb 9 10:09:01 2016 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +-
.../process/computer/GiraphComputation.java | 3 +-
.../computer/traversal/TraverserExecutor.java | 101 +++++++++++--------
.../traversal/step/map/CountGlobalStep.java | 21 +++-
.../traversal/step/map/GroupCountStep.java | 16 +--
.../process/traversal/step/map/GroupStep.java | 10 +-
.../traversal/step/map/GroupStepV3d0.java | 10 +-
.../traversal/step/map/MaxGlobalStep.java | 23 +++--
.../traversal/step/map/MinGlobalStep.java | 22 ++--
.../traversal/step/map/SumGlobalStep.java | 23 +++--
.../process/traversal/step/map/TreeStep.java | 10 --
.../step/util/ReducingBarrierStep.java | 50 ++++-----
.../gremlin/util/iterator/DoubleIterator.java | 12 ++-
.../gremlin/util/iterator/IteratorUtils.java | 44 ++++++++
.../gremlin/util/iterator/MultiIterator.java | 7 +-
.../gremlin/util/iterator/SingleIterator.java | 9 +-
.../process/computer/TinkerMessenger.java | 13 +--
17 files changed, 244 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b07400cc/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 6b51ba8,4881c52..0cd221f
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -22,16 -22,10 +22,17 @@@ TinkerPop 3.1.0 (A 187 On The Undercove
image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/static/images/gremlin-gangster.png[width=185]
-[[release-3.1.1-incubating]]
-TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
+[[release-3.1.2-incubating]]
+TinkerPop 3.1.2 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
++* Optimized memory-usage in `TraversalVertexProgram`.
++* Greatly reduced the amount of objects required in OLAP for the `ReducingBarrierStep` steps.
+
+[[release-3.1.1-incubating]]
+TinkerPop 3.1.1 (Release Date: February 8, 2016)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
* Made `GryoRecordReader` more robust to 0 byte record splits.
* Fixed a constructor/serialization bug in `LP_O_OB_S_SE_SL_Traverser`.
* Added a lazy iterator, memory safe implementation of MapReduce to `SparkGraphComputer`.
[8/8] incubator-tinkerpop git commit: merged from tp31 and fix
conflicts.
Posted by ok...@apache.org.
merged from tp31 and fix conflicts.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/26aedc74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/26aedc74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/26aedc74
Branch: refs/heads/master
Commit: 26aedc746fd1bed726e73d8ca2b6063307777d5c
Parents: e8a7e3a b07400c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Feb 9 10:14:47 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Feb 9 10:14:47 2016 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 7 ++
docs/src/dev/developer/release.asciidoc | 67 ++++++------
.../process/computer/GiraphComputation.java | 3 +-
.../computer/traversal/TraverserExecutor.java | 101 +++++++++++--------
.../traversal/step/map/CountGlobalStep.java | 21 +++-
.../traversal/step/map/GroupCountStep.java | 16 +--
.../process/traversal/step/map/GroupStep.java | 10 +-
.../traversal/step/map/GroupStepV3d0.java | 10 +-
.../traversal/step/map/MaxGlobalStep.java | 23 +++--
.../traversal/step/map/MinGlobalStep.java | 22 ++--
.../traversal/step/map/SumGlobalStep.java | 23 +++--
.../process/traversal/step/map/TreeStep.java | 10 --
.../step/util/ReducingBarrierStep.java | 50 ++++-----
.../gremlin/util/iterator/DoubleIterator.java | 12 ++-
.../gremlin/util/iterator/IteratorUtils.java | 39 ++++++-
.../gremlin/util/iterator/MultiIterator.java | 2 +-
.../gremlin/util/iterator/SingleIterator.java | 9 +-
pom.xml | 12 ++-
.../process/computer/TinkerMessenger.java | 13 +--
19 files changed, 276 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26aedc74/CHANGELOG.asciidoc
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26aedc74/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 91f515a,0a26050..955cef5
--- a/pom.xml
+++ b/pom.xml
@@@ -742,7 -742,7 +742,8 @@@ limitations under the License
<configuration>
<sourceDirectory>${asciidoc.input.dir}/tutorials/getting-started</sourceDirectory>
<sourceDocumentName>index.asciidoc</sourceDocumentName>
-- <outputDirectory>${htmlsingle.output.dir}/tutorials/getting-started</outputDirectory>
++ <outputDirectory>${htmlsingle.output.dir}/tutorials/getting-started
++ </outputDirectory>
<backend>html5</backend>
<doctype>article</doctype>
<attributes>
@@@ -762,9 -762,9 +763,11 @@@
<goal>process-asciidoc</goal>
</goals>
<configuration>
-- <sourceDirectory>${asciidoc.input.dir}/tutorials/the-gremlin-console</sourceDirectory>
++ <sourceDirectory>${asciidoc.input.dir}/tutorials/the-gremlin-console
++ </sourceDirectory>
<sourceDocumentName>index.asciidoc</sourceDocumentName>
-- <outputDirectory>${htmlsingle.output.dir}/tutorials/the-gremlin-console</outputDirectory>
++ <outputDirectory>${htmlsingle.output.dir}/tutorials/the-gremlin-console
++ </outputDirectory>
<backend>html5</backend>
<doctype>article</doctype>
<attributes>
@@@ -954,7 -954,7 +957,8 @@@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
-- <argLine>-Dlog4j.configuration=${log4j-silent.properties} -Dbuild.dir=${project.build.directory}
++ <argLine>-Dlog4j.configuration=${log4j-silent.properties}
++ -Dbuild.dir=${project.build.directory}
</argLine>
<excludes>
<exclude>**/*IntegrateTest.java</exclude>
[2/8] incubator-tinkerpop git commit: Figured out a neat trick.
IteratorUtils.noRemove(Iterator) so if a providers Messenger doesn't support
Iterator.remove(),
they can simply wrap the iterator they provide in Iterator.noRemove().
Chillin'.
Posted by ok...@apache.org.
Figured out a neat trick. IteratorUtils.noRemove(Iterator) so if a providers Messenger doesn't support Iterator.remove(), they can simply wrap the iterator they provide in Iterator.noRemove(). Chillin'.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/047ee206
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/047ee206
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/047ee206
Branch: refs/heads/master
Commit: 047ee2065023227381e1d2e594354b05331f06fd
Parents: b54ddf2
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Feb 5 15:30:53 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Feb 5 15:30:53 2016 -0700
----------------------------------------------------------------------
.../process/computer/GiraphComputation.java | 3 ++-
.../gremlin/util/iterator/IteratorUtils.java | 19 +++++++++++++++++++
.../gremlin/util/iterator/MultiIterator.java | 2 +-
.../process/computer/TinkerMessenger.java | 13 +++++++------
4 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/047ee206/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
index 1d52566..f094f2d 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
@@ -26,6 +26,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.IOException;
@@ -41,7 +42,7 @@ public final class GiraphComputation extends BasicComputation<ObjectWritable, Ve
public void compute(final Vertex<ObjectWritable, VertexWritable, NullWritable> vertex, final Iterable<ObjectWritable> messages) throws IOException {
final GiraphWorkerContext workerContext = this.getWorkerContext();
final VertexProgram<?> vertexProgram = workerContext.getVertexProgramPool().take();
- vertexProgram.execute(ComputerGraph.vertexProgram(vertex.getValue().get(), vertexProgram), workerContext.getMessenger((GiraphVertex) vertex, this, messages.iterator()), workerContext.getMemory());
+ vertexProgram.execute(ComputerGraph.vertexProgram(vertex.getValue().get(), vertexProgram), workerContext.getMessenger((GiraphVertex) vertex, this, IteratorUtils.noRemove(messages.iterator())), workerContext.getMemory());
workerContext.getVertexProgramPool().offer(vertexProgram);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/047ee206/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
index 808317f..78ffe4a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/IteratorUtils.java
@@ -392,4 +392,23 @@ public final class IteratorUtils {
public static <T> Stream<T> stream(final Iterable<T> iterable) {
return IteratorUtils.stream(iterable.iterator());
}
+
+ public static <T> Iterator<T> noRemove(final Iterator<T> iterator) {
+ return new Iterator<T>() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public void remove() {
+ // do nothing
+ }
+
+ @Override
+ public T next() {
+ return iterator.next();
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/047ee206/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
index e272a08..20c0946 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/iterator/MultiIterator.java
@@ -59,7 +59,7 @@ public final class MultiIterator<T> implements Iterator<T>, Serializable {
@Override
public void remove() {
- // this.iterators.get(this.current).remove();
+ this.iterators.get(this.current).remove();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/047ee206/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerMessenger.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerMessenger.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerMessenger.java
index d0c302a..71d7030 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerMessenger.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerMessenger.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.apache.tinkerpop.gremlin.util.iterator.MultiIterator;
import java.util.Iterator;
@@ -64,19 +65,19 @@ public final class TinkerMessenger<M> implements Messenger<M> {
final Traversal.Admin<Vertex, Edge> incidentTraversal = TinkerMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get().asAdmin(), this.vertex);
final Direction direction = TinkerMessenger.getDirection(incidentTraversal);
final Edge[] edge = new Edge[1]; // simulates storage side-effects available in Gremlin, but not Java8 streams
- multiIterator.addIterator(StreamSupport.stream(Spliterators.spliteratorUnknownSize(VertexProgramHelper.reverse(incidentTraversal.asAdmin()), Spliterator.IMMUTABLE | Spliterator.SIZED), false)
+ multiIterator.addIterator(IteratorUtils.noRemove(StreamSupport.stream(Spliterators.spliteratorUnknownSize(VertexProgramHelper.reverse(incidentTraversal.asAdmin()), Spliterator.IMMUTABLE | Spliterator.SIZED), false)
.map(e -> this.messageBoard.receiveMessages.get((edge[0] = e).vertices(direction).next()))
.filter(q -> null != q)
.flatMap(Queue::stream)
.map(message -> localMessageScope.getEdgeFunction().apply(message, edge[0]))
- .iterator());
+ .iterator()));
} else {
- multiIterator.addIterator(Stream.of(this.vertex)
+ multiIterator.addIterator(IteratorUtils.noRemove(Stream.of(this.vertex)
.map(this.messageBoard.receiveMessages::get)
.filter(q -> null != q)
.flatMap(Queue::stream)
- .iterator());
+ .iterator()));
}
}
return multiIterator;
@@ -93,8 +94,8 @@ public final class TinkerMessenger<M> implements Messenger<M> {
}
private void addMessage(final Vertex vertex, final M message) {
- this.messageBoard.sendMessages.compute(vertex, (v,queue) -> {
- if(null == queue) queue = new ConcurrentLinkedQueue<>();
+ this.messageBoard.sendMessages.compute(vertex, (v, queue) -> {
+ if (null == queue) queue = new ConcurrentLinkedQueue<>();
queue.add(null != this.combiner && !queue.isEmpty() ? this.combiner.combine(queue.remove(), message) : message);
return queue;
});
[3/8] incubator-tinkerpop git commit: Greatly optimized all
ReducingBarrierSteps in OLAP execution. I realized that there are actually
two 'combine' stages. Given that the HALTED_TRAVERSERS are all group'd at a
Vertex prior to MAP, you can COMBINE in Map
Posted by ok...@apache.org.
Greatly optimized all ReducingBarrierSteps in OLAP execution. I realized that there are actually two 'combine' stages. Given that the HALTED_TRAVERSERS are all group'd at a Vertex prior to MAP, you can COMBINE in Map. Thus, you map/combine, combine, and then reduce. For steps like outE().count(), this means that we don't Map traversers for each edge --- instead we create a single count locally before pushing off to combine/reduce. This will make GC and memory usage minimal for these operations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/388d1f25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/388d1f25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/388d1f25
Branch: refs/heads/master
Commit: 388d1f25fec61aebc175508452d74b6375bacabe
Parents: 047ee20
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Sun Feb 7 14:19:28 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Sun Feb 7 14:19:28 2016 -0700
----------------------------------------------------------------------
.../traversal/step/map/CountGlobalStep.java | 31 ++++++------
.../traversal/step/map/GroupCountStep.java | 16 ++-----
.../process/traversal/step/map/GroupStep.java | 10 +++-
.../traversal/step/map/GroupStepV3d0.java | 10 +++-
.../traversal/step/map/MaxGlobalStep.java | 23 +++++----
.../traversal/step/map/MinGlobalStep.java | 22 +++++----
.../traversal/step/map/SumGlobalStep.java | 23 +++++----
.../process/traversal/step/map/TreeStep.java | 10 ----
.../step/util/ReducingBarrierStep.java | 50 ++++++++++----------
9 files changed, 106 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
index d090832..096580d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
@@ -27,11 +27,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.function.ConstantSupplier;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.Serializable;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.Set;
@@ -61,16 +62,6 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
return CountGlobalMapReduce.instance();
}
- @Override
- public Traverser<Long> processNextStart() {
- if (this.byPass) {
- final Traverser.Admin<S> traverser = this.starts.next();
- return traverser.asAdmin().split(1l, this); // if bypassing, just key all the traversers to 1 long (the count is going to be the bulk of course)
- } else {
- return super.processNextStart();
- }
- }
-
///////////
private static class CountBiFunction<S> implements BiFunction<Long, Traverser<S>, Long>, Serializable {
@@ -108,7 +99,11 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
@Override
public void map(final Vertex vertex, final MapEmitter<NullObject, Long> emitter) {
- vertex.<TraverserSet<?>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet -> traverserSet.forEach(traverser -> emitter.emit(traverser.bulk())));
+ final Iterator<Long> values = IteratorUtils.map(vertex.<Set<Traverser.Admin<Long>>>property(TraversalVertexProgram.HALTED_TRAVERSERS).orElse(Collections.emptySet()).iterator(),
+ traverser -> traverser.get() * traverser.bulk());
+ long count = getCount(values);
+ if (count > 0)
+ emitter.emit(count);
}
@Override
@@ -118,11 +113,17 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
@Override
public void reduce(final NullObject key, final Iterator<Long> values, final ReduceEmitter<NullObject, Long> emitter) {
+ long count = getCount(values);
+ if (count > 0)
+ emitter.emit(count);
+ }
+
+ private Long getCount(final Iterator<Long> longs) {
long count = 0l;
- while (values.hasNext()) {
- count = count + values.next();
+ while (longs.hasNext()) {
+ count = count + longs.next();
}
- emitter.emit(count);
+ return count;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupCountStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupCountStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupCountStep.java
index 7f3904e..95b954e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupCountStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupCountStep.java
@@ -97,16 +97,6 @@ public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Lo
}
@Override
- public Traverser<Map<E, Long>> processNextStart() {
- if (this.byPass) {
- final Traverser.Admin<S> traverser = this.starts.next();
- return traverser.asAdmin().split(TraversalUtil.applyNullable(traverser, (Traversal.Admin<S, Map<E, Long>>) this.groupTraversal), this);
- } else {
- return super.processNextStart();
- }
- }
-
- @Override
public String toString() {
return StringFactory.stepString(this, this.groupTraversal);
}
@@ -146,7 +136,11 @@ public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Lo
@Override
public void map(final Vertex vertex, final MapEmitter<E, Long> emitter) {
- vertex.<TraverserSet<E>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet -> traverserSet.forEach(traverser -> emitter.emit(traverser.get(), traverser.bulk())));
+ final Map<E, Long> groupCount = new HashMap<>();
+ vertex.<TraverserSet<Map<E, Long>>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet ->
+ traverserSet.forEach(traverser ->
+ traverser.get().forEach((k, v) -> MapHelper.incr(groupCount, k, (v * traverser.bulk())))));
+ groupCount.forEach(emitter::emit);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
index 32a16a7..069d57e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStep.java
@@ -24,9 +24,11 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
+import org.apache.tinkerpop.gremlin.process.traversal.step.EngineDependent;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
@@ -55,7 +57,7 @@ import java.util.function.Supplier;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>> implements MapReducer, TraversalParent {
+public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>> implements MapReducer, EngineDependent, TraversalParent {
private char state = 'k';
@@ -63,6 +65,7 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
private Traversal.Admin<S, ?> valueTraversal = this.integrateChild(__.identity().asAdmin()); // used in OLAP
private Traversal.Admin<?, V> reduceTraversal = this.integrateChild(__.fold().asAdmin()); // used in OLAP
private Traversal.Admin<S, V> valueReduceTraversal = this.integrateChild(__.fold().asAdmin()); // used in OLTP
+ private boolean byPass = false;
public GroupStep(final Traversal.Admin traversal) {
super(traversal);
@@ -71,6 +74,11 @@ public final class GroupStep<S, K, V> extends ReducingBarrierStep<S, Map<K, V>>
}
@Override
+ public void onEngine(final TraversalEngine traversalEngine) {
+ this.byPass = traversalEngine.isComputer();
+ }
+
+ @Override
public List<Traversal.Admin<?, ?>> getLocalChildren() {
final List<Traversal.Admin<?, ?>> children = new ArrayList<>(4);
if (null != this.keyTraversal)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStepV3d0.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStepV3d0.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStepV3d0.java
index 04acf62..a988d4e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStepV3d0.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroupStepV3d0.java
@@ -25,7 +25,9 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.EngineDependent;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
@@ -55,13 +57,14 @@ import java.util.function.Supplier;
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@Deprecated
-public final class GroupStepV3d0<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R>> implements MapReducer, TraversalParent {
+public final class GroupStepV3d0<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R>> implements MapReducer, EngineDependent, TraversalParent {
private char state = 'k';
private Traversal.Admin<S, K> keyTraversal = null;
private Traversal.Admin<S, V> valueTraversal = null;
private Traversal.Admin<Collection<V>, R> reduceTraversal = null;
+ private boolean byPass = false;
public GroupStepV3d0(final Traversal.Admin traversal) {
super(traversal);
@@ -70,6 +73,11 @@ public final class GroupStepV3d0<S, K, V, R> extends ReducingBarrierStep<S, Map<
}
@Override
+ public void onEngine(final TraversalEngine traversalEngine) {
+ this.byPass = traversalEngine.isComputer();
+ }
+
+ @Override
public <A, B> List<Traversal.Admin<A, B>> getLocalChildren() {
final List<Traversal.Admin<A, B>> children = new ArrayList<>(3);
if (null != this.keyTraversal)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
index d876d2d..c552bab 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
@@ -27,9 +27,9 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.function.ConstantSupplier;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.Serializable;
import java.util.Collections;
@@ -98,7 +98,9 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public void map(final Vertex vertex, final MapEmitter<NullObject, Number> emitter) {
- vertex.<TraverserSet<Number>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet -> traverserSet.forEach(traverser -> emitter.emit(traverser.get())));
+ final Iterator<Number> values = IteratorUtils.map(vertex.<Set<Traverser.Admin<Number>>>property(TraversalVertexProgram.HALTED_TRAVERSERS).orElse(Collections.emptySet()).iterator(), Traverser.Admin::get);
+ if (values.hasNext())
+ emitter.emit(getMax(values));
}
@Override
@@ -108,14 +110,17 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public void reduce(final NullObject key, final Iterator<Number> values, final ReduceEmitter<NullObject, Number> emitter) {
- if (values.hasNext()) {
- Number max = null;
- while (values.hasNext()) {
- final Number value = values.next();
- max = max != null ? max(value, max) : value;
- }
- emitter.emit(max);
+ if (values.hasNext())
+ emitter.emit(getMax(values));
+ }
+
+ private Number getMax(final Iterator<Number> numbers) {
+ Number max = null;
+ while (numbers.hasNext()) {
+ final Number value = numbers.next();
+ max = max != null ? max(value, max) : value;
}
+ return max;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
index 750e36c..88dd46f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
@@ -27,9 +27,9 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.function.ConstantSupplier;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.Serializable;
import java.util.Collections;
@@ -98,7 +98,9 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public void map(final Vertex vertex, final MapEmitter<NullObject, Number> emitter) {
- vertex.<TraverserSet<Number>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet -> traverserSet.forEach(traverser -> emitter.emit(traverser.get())));
+ final Iterator<Number> values = IteratorUtils.map(vertex.<Set<Traverser.Admin<Number>>>property(TraversalVertexProgram.HALTED_TRAVERSERS).orElse(Collections.emptySet()).iterator(), Traverser.Admin::get);
+ if (values.hasNext())
+ emitter.emit(getMin(values));
}
@Override
@@ -108,14 +110,16 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public void reduce(final NullObject key, final Iterator<Number> values, final ReduceEmitter<NullObject, Number> emitter) {
- if (values.hasNext()) {
- Number min = null;
- while (values.hasNext()) {
- final Number value = values.next();
- min = min != null ? min(value, min) : value;
- }
- emitter.emit(min);
+ if (values.hasNext()) emitter.emit(getMin(values));
+ }
+
+ private Number getMin(final Iterator<Number> numbers) {
+ Number min = null;
+ while (numbers.hasNext()) {
+ final Number value = numbers.next();
+ min = min != null ? min(value, min) : value;
}
+ return min;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
index 14573d1..8c13192 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
@@ -27,11 +27,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.util.function.ConstantSupplier;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.Serializable;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.Set;
@@ -104,7 +105,10 @@ public final class SumGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public void map(final Vertex vertex, final MapEmitter<NullObject, Number> emitter) {
- vertex.<TraverserSet<Number>>property(TraversalVertexProgram.HALTED_TRAVERSERS).ifPresent(traverserSet -> traverserSet.forEach(traverser -> emitter.emit(mul(traverser.get(), traverser.bulk()))));
+ final Iterator<Number> values = IteratorUtils.map(vertex.<Set<Traverser.Admin<Number>>>property(TraversalVertexProgram.HALTED_TRAVERSERS).orElse(Collections.emptySet()).iterator(),
+ traverser -> mul(traverser.get(), traverser.bulk()));
+ if (values.hasNext())
+ emitter.emit(getSum(values));
}
@Override
@@ -114,13 +118,16 @@ public final class SumGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public void reduce(final NullObject key, final Iterator<Number> values, final ReduceEmitter<NullObject, Number> emitter) {
- if (values.hasNext()) {
- Number sum = values.next();
- while (values.hasNext()) {
- sum = add(sum, values.next());
- }
- emitter.emit(sum);
+ if (values.hasNext())
+ emitter.emit(getSum(values));
+ }
+
+ private Number getSum(final Iterator<Number> numbers) {
+ Number sum = numbers.next();
+ while (numbers.hasNext()) {
+ sum = add(sum, numbers.next());
}
+ return sum;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/TreeStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/TreeStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/TreeStep.java
index 3de0a9d..92ffb76 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/TreeStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/TreeStep.java
@@ -94,16 +94,6 @@ public final class TreeStep<S> extends ReducingBarrierStep<S, Tree> implements M
}
@Override
- public Traverser<Tree> processNextStart() {
- if (this.byPass) {
- final Traverser.Admin<S> traverser = this.starts.next();
- return traverser.split(this.reducingBiFunction.apply(new Tree(), traverser), this);
- } else {
- return super.processNextStart();
- }
- }
-
- @Override
public String toString() {
return StringFactory.stepString(this, this.traversalRing);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/388d1f25/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ReducingBarrierStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ReducingBarrierStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ReducingBarrierStep.java
index b459071..fb7ecb7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ReducingBarrierStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ReducingBarrierStep.java
@@ -26,10 +26,8 @@ import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramHelper;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
-import org.apache.tinkerpop.gremlin.process.traversal.step.EngineDependent;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException;
@@ -44,14 +42,13 @@ import java.util.function.Supplier;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public abstract class ReducingBarrierStep<S, E> extends AbstractStep<S, E> implements MapReducer, EngineDependent, Barrier {
+public abstract class ReducingBarrierStep<S, E> extends AbstractStep<S, E> implements MapReducer, Barrier {
public static final String REDUCING = Graph.Hidden.hide("reducing");
protected Supplier<E> seedSupplier;
protected BiFunction<E, Traverser<S>, E> reducingBiFunction;
private boolean done = false;
- protected boolean byPass = false;
private E seed = null;
@@ -68,11 +65,6 @@ public abstract class ReducingBarrierStep<S, E> extends AbstractStep<S, E> imple
}
@Override
- public void onEngine(final TraversalEngine traversalEngine) {
- this.byPass = traversalEngine.isComputer();
- }
-
- @Override
public void reset() {
super.reset();
this.done = false;
@@ -80,27 +72,35 @@ public abstract class ReducingBarrierStep<S, E> extends AbstractStep<S, E> imple
}
@Override
- public void processAllStarts() {
- if (!this.byPass) {
- if (this.seed == null) this.seed = this.seedSupplier.get();
- while (this.starts.hasNext())
- this.seed = this.reducingBiFunction.apply(this.seed, this.starts.next());
+ public void addStarts(final Iterator<Traverser<S>> starts) {
+ if (starts.hasNext()) {
+ this.done = false;
+ super.addStarts(starts);
}
}
@Override
+ public void addStart(final Traverser<S> start) {
+ this.done = false;
+ super.addStart(start);
+ }
+
+ @Override
+ public void processAllStarts() {
+ if (this.seed == null) this.seed = this.seedSupplier.get();
+ while (this.starts.hasNext())
+ this.seed = this.reducingBiFunction.apply(this.seed, this.starts.next());
+ }
+
+ @Override
public Traverser<E> processNextStart() {
- if (this.byPass) {
- return (Traverser<E>) this.starts.next();
- } else {
- if (this.done)
- throw FastNoSuchElementException.instance();
- this.processAllStarts();
- this.done = true;
- final Traverser<E> traverser = TraversalHelper.getRootTraversal(this.getTraversal()).getTraverserGenerator().generate(FinalGet.tryFinalGet(this.seed), (Step) this, 1l);
- this.seed = null;
- return traverser;
- }
+ if (this.done)
+ throw FastNoSuchElementException.instance();
+ this.processAllStarts();
+ this.done = true;
+ final Traverser<E> traverser = TraversalHelper.getRootTraversal(this.getTraversal()).getTraverserGenerator().generate(FinalGet.tryFinalGet(this.seed), (Step) this, 1l);
+ this.seed = null;
+ return traverser;
}
@Override
[4/8] incubator-tinkerpop git commit: minor nothing.
Posted by ok...@apache.org.
minor nothing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ba22c7aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ba22c7aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ba22c7aa
Branch: refs/heads/master
Commit: ba22c7aadab066e1f685368cc51faca0b7597831
Parents: 388d1f2
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Sun Feb 7 19:38:13 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Sun Feb 7 19:38:13 2016 -0700
----------------------------------------------------------------------
.../gremlin/process/computer/traversal/TraverserExecutor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ba22c7aa/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
index 2df682c..04f33f2 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
@@ -61,9 +61,9 @@ public final class TraverserExecutor {
}
}
// while there are still local traversers, process them until they leave the vertex or halt (i.e. isHalted()).
- Step<?, ?> previousStep = EmptyStep.instance();
while (!toProcessTraversers.isEmpty()) {
// process local traversers and if alive, repeat, else halt.
+ Step<?, ?> previousStep = EmptyStep.instance();
Iterator<Traverser.Admin<Object>> traversers = toProcessTraversers.iterator();
while (traversers.hasNext()) {
final Traverser.Admin<Object> traverser = traversers.next();