You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/26 18:43:09 UTC
incubator-tinkerpop git commit: Giraph needed RangeBiOperator
registered in GryoMapper. TailBiOperator just uses RangeBiOperator now as
they both have the same logic (for now). Now that
TraversalVertexProgram.terminate() is so complex and ancient hack in
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1166 958791792 -> 86615a216
Giraph needed RangeBiOperator registered in GryoMapper. TailBiOperator just uses RangeBiOperator now as they both have the same logic (for now). Now that TraversalVertexProgram.terminate() is so complex and ancient hack in GiraphMemory needed to be worked around yet again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/86615a21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/86615a21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/86615a21
Branch: refs/heads/TINKERPOP-1166
Commit: 86615a216e9c400f623e87f4d7ee6dbbdc72bcc5
Parents: 9587917
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Feb 26 10:42:55 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Feb 26 10:42:55 2016 -0700
----------------------------------------------------------------------
.../giraph/process/computer/GiraphMemory.java | 11 +++++-----
.../traversal/step/filter/TailGlobalStep.java | 22 +-------------------
.../gremlin/structure/io/gryo/GryoMapper.java | 4 +++-
3 files changed, 9 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/86615a21/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
index 5e87f20..752ba41 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
@@ -91,12 +91,9 @@ public final class GiraphMemory extends MasterCompute implements Memory {
}
this.vertexProgram.setup(this);
} else {
- if (this.vertexProgram.terminate(this)) { // terminate
- // write the memory to HDFS
- final MapMemory memory = new MapMemory(this);
- // a hack to get the last iteration memory values to stick
- this.vertexProgram.terminate(memory);
-
+ // a hack to get the last iteration memory values to stick
+ final MapMemory memory = new MapMemory(this);
+ if (this.vertexProgram.terminate(memory)) { // terminate
final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
if (null != outputLocation) {
try {
@@ -117,6 +114,8 @@ public final class GiraphMemory extends MasterCompute implements Memory {
}
this.haltComputation();
}
+ // to ensure proper terminate that is not the "hack" above
+ this.vertexProgram.terminate(this);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/86615a21/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
index 2d5c798..8d6d8f0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
@@ -28,14 +28,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequire
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-import java.io.Serializable;
import java.util.ArrayDeque;
import java.util.Collections;
import java.util.Deque;
import java.util.Iterator;
import java.util.Optional;
import java.util.Set;
-import java.util.function.BinaryOperator;
/**
* @author Matt Frantz (http://github.com/mhfrantz)
@@ -131,7 +129,7 @@ public final class TailGlobalStep<S> extends AbstractStep<S, S> implements Bypas
@Override
public Optional<MemoryComputeKey> getMemoryComputeKey() {
- return Optional.of(MemoryComputeKey.of(this.getId(), new TailBiOperator(this.limit), false, true));
+ return Optional.of(MemoryComputeKey.of(this.getId(), new RangeGlobalStep.RangeBiOperator(this.limit), false, true));
}
@Override
@@ -141,22 +139,4 @@ public final class TailGlobalStep<S> extends AbstractStep<S, S> implements Bypas
this.forEachRemaining(t -> resultSet.add(t.asAdmin()));
return resultSet;
}
-
- /////////////////////
-
- public static final class TailBiOperator implements BinaryOperator<TraverserSet>, Serializable {
-
- private final long limit;
-
- public TailBiOperator(final long limit) {
- this.limit = limit;
- }
-
- @Override
- public TraverserSet apply(final TraverserSet mutatingSeed, final TraverserSet set) {
- if (mutatingSeed.size() < this.limit)
- mutatingSeed.addAll(set);
- return mutatingSeed;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/86615a21/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index dd79bae..85e31a3 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.process.traversal.Contains;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.Path;
+import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.FoldStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupCountStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupStep;
@@ -326,7 +327,8 @@ public final class GryoMapper implements Mapper<Kryo> {
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MeanGlobalStep.MeanGlobalBiOperator.class, null, 110));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MeanGlobalStep.MeanNumber.class, null, 111));
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TreeStep.TreeBiOperator.class, null, 112));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(GroupStepV3d0.GroupBiOperatorV3d0.class, null, 113)); // ***LAST ID**
+ add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(GroupStepV3d0.GroupBiOperatorV3d0.class, null, 113));
+ add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(RangeGlobalStep.RangeBiOperator.class, null, 114)); // ***LAST ID**
}};