You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/03/22 21:53:32 UTC
[07/12] incubator-tinkerpop git commit: work on profile step on
GraphComputer. Need @rbriody to review and discuss.
work on profile step on GraphComputer. Need @rbriody to review and discuss.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/58615c04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/58615c04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/58615c04
Branch: refs/heads/master
Commit: 58615c04228b3508da5141ab5a6a1bf4f050ef9f
Parents: 73b2d42
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 21 12:36:00 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 21 12:36:00 2016 -0600
----------------------------------------------------------------------
.../process/computer/MemoryComputeKey.java | 2 +-
.../traversal/TraversalVertexProgram.java | 22 ++++++-
.../traversal/step/filter/DedupGlobalStep.java | 7 ++-
.../traversal/step/filter/RangeGlobalStep.java | 6 +-
.../traversal/step/filter/TailGlobalStep.java | 2 +-
.../traversal/step/map/OrderGlobalStep.java | 14 ++---
.../step/sideEffect/ProfileSideEffectStep.java | 16 ++---
.../step/util/CollectingBarrierStep.java | 3 +-
.../traversal/step/util/ProfileStep.java | 41 ++++++++++++-
.../step/util/SupplyingBarrierStep.java | 3 +-
.../ComputerVerificationStrategy.java | 2 +-
.../traversal/util/DefaultTraversalMetrics.java | 61 ++++++++++++++++----
.../process/traversal/step/map/ProfileTest.java | 35 ++++++-----
.../structure/TinkerGraphPlayTest.java | 15 ++---
14 files changed, 171 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MemoryComputeKey.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MemoryComputeKey.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MemoryComputeKey.java
index 61e2fd8..94ca675 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MemoryComputeKey.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/MemoryComputeKey.java
@@ -73,7 +73,7 @@ public final class MemoryComputeKey<A> implements Serializable {
return object instanceof MemoryComputeKey && ((MemoryComputeKey) object).key.equals(this.key);
}
- public static <A> MemoryComputeKey of(final String key, final BinaryOperator<A> reducer, final boolean isBroadcast, final boolean isTransient) {
+ public static <A> MemoryComputeKey<A> of(final String key, final BinaryOperator<A> reducer, final boolean isBroadcast, final boolean isTransient) {
return new MemoryComputeKey<>(key, reducer, isBroadcast, isTransient);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index aaac518..fbe5592 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -54,12 +54,15 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyKeyStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyMapStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyValueStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SackStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.VertexProgramStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.ScriptTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
@@ -141,6 +144,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
// if results will be serialized out, don't save halted traversers across the cluster
this.keepDistributedHaltedTraversers =
!(this.traversal.get().getParent().asStep().getNextStep() instanceof ComputerResultStep || // if its just going to stream it out, don't distribute
+ this.traversal.get().getParent().asStep().getNextStep().getNextStep() instanceof ComputerResultStep ||
this.traversal.get().getParent().asStep().getNextStep() instanceof EmptyStep); // same as above, but if using TraversalVertexProgramStep directly
// register traversal side-effects in memory
final TraversalSideEffects sideEffects = ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).getSideEffects();
@@ -155,6 +159,11 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
for (final MemoryComputing<?> memoryComputing : TraversalHelper.getStepsOfAssignableClassRecursively(MemoryComputing.class, this.traversal.get())) {
this.memoryComputeKeys.add(memoryComputing.getMemoryComputeKey());
}
+ // register profile steps
+ for (final ProfileStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get())) {
+ this.traversal.get().getSideEffects().register(profileStep.getId(), () -> new MutableMetrics(profileStep.getPreviousStep().getId(), profileStep.getPreviousStep().toString()), ProfileStep.ProfileBiOperator.instance());
+
+ }
// register TraversalVertexProgram specific memory compute keys
this.memoryComputeKeys.add(MemoryComputeKey.of(HALTED_TRAVERSERS, Operator.addAll, false, this.keepDistributedHaltedTraversers)); // only keep if it will be preserved
this.memoryComputeKeys.add(MemoryComputeKey.of(ACTIVE_TRAVERSERS, Operator.addAll, true, true));
@@ -170,6 +179,8 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
@Override
public void setup(final Memory memory) {
+ // memory is distributed
+ ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).setMemory(memory, false);
memory.set(VOTE_TO_HALT, true);
memory.set(HALTED_TRAVERSERS, new TraverserSet<>());
memory.set(ACTIVE_TRAVERSERS, new TraverserSet<>());
@@ -177,7 +188,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
memory.set(COMPLETED_BARRIERS, new HashSet<>());
final TraversalSideEffects sideEffects = ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).getSideEffects();
sideEffects.keys().forEach(key -> memory.set(key, sideEffects.get(key)));
- ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).setMemory(memory, false);
+
}
@Override
@@ -187,7 +198,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
@Override
public void execute(final Vertex vertex, final Messenger<TraverserSet<Object>> messenger, final Memory memory) {
- // back the traversal side-effects by the global memory
+ // memory is distributed
((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).setMemory(memory, true);
// if a barrier was completed in another worker, it is also completed here (ensure distributed barries are synchronized)
final Set<String> completedBarriers = memory.get(COMPLETED_BARRIERS);
@@ -235,10 +246,11 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
@Override
public boolean terminate(final Memory memory) {
+ // memory is not distributed
+ ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).setMemory(memory, false);
final boolean voteToHalt = memory.<Boolean>get(VOTE_TO_HALT);
memory.set(VOTE_TO_HALT, true);
memory.set(ACTIVE_TRAVERSERS, new TraverserSet<>());
- // put all side-effect memory into traversal side-effects
if (voteToHalt) {
final Set<String> mutatedMemoryKeys = memory.get(MUTATED_MEMORY_KEYS);
memory.set(MUTATED_MEMORY_KEYS, new HashSet<>());
@@ -272,6 +284,10 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
}
// the result of a TraversalVertexProgram are the halted traversers
memory.set(HALTED_TRAVERSERS, haltedTraversers);
+ // finalize profile side-effect steps.
+ for (final ProfileSideEffectStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileSideEffectStep.class, this.traversal.get())) {
+ this.traversal.get().getSideEffects().set(profileStep.getSideEffectKey(), profileStep.generateFinalResult(this.traversal.get().getSideEffects().get(profileStep.getSideEffectKey())));
+ }
return true;
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
index 649380e..546d24e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
@@ -43,6 +43,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.function.BinaryOperator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -120,7 +121,9 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
@Override
public Set<TraverserRequirement> getRequirements() {
- return this.dedupLabels == null ? this.getSelfAndChildRequirements(TraverserRequirement.BULK) : this.getSelfAndChildRequirements(TraverserRequirement.LABELED_PATH, TraverserRequirement.BULK);
+ return this.dedupLabels == null ?
+ this.getSelfAndChildRequirements(TraverserRequirement.BULK) :
+ this.getSelfAndChildRequirements(TraverserRequirement.LABELED_PATH, TraverserRequirement.BULK);
}
@Override
@@ -167,7 +170,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
@Override
public MemoryComputeKey<TraverserSet<S>> getMemoryComputeKey() {
- return MemoryComputeKey.of(this.getId(), Operator.addAll, false, true);
+ return MemoryComputeKey.of(this.getId(), (BinaryOperator) Operator.addAll, false, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/RangeGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/RangeGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/RangeGlobalStep.java
index 7576cc1..9458ca9 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/RangeGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/RangeGlobalStep.java
@@ -135,7 +135,7 @@ public final class RangeGlobalStep<S> extends FilterStep<S> implements Ranging,
@Override
public MemoryComputeKey<TraverserSet<S>> getMemoryComputeKey() {
- return MemoryComputeKey.of(this.getId(), new RangeBiOperator(this.high), false, true);
+ return MemoryComputeKey.of(this.getId(), new RangeBiOperator<>(this.high), false, true);
}
@Override
@@ -174,7 +174,7 @@ public final class RangeGlobalStep<S> extends FilterStep<S> implements Ranging,
////////////////
- public static final class RangeBiOperator implements BinaryOperator<TraverserSet>, Serializable {
+ public static final class RangeBiOperator<S> implements BinaryOperator<TraverserSet<S>>, Serializable {
private final long highRange;
@@ -187,7 +187,7 @@ public final class RangeGlobalStep<S> extends FilterStep<S> implements Ranging,
}
@Override
- public TraverserSet apply(final TraverserSet mutatingSeed, final TraverserSet set) {
+ public TraverserSet<S> apply(final TraverserSet<S> mutatingSeed, final TraverserSet<S> set) {
if (mutatingSeed.size() < this.highRange)
mutatingSeed.addAll(set);
return mutatingSeed;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
index 9d19cb9..096151d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/TailGlobalStep.java
@@ -126,7 +126,7 @@ public final class TailGlobalStep<S> extends AbstractStep<S, S> implements Bypas
@Override
public MemoryComputeKey<TraverserSet<S>> getMemoryComputeKey() {
- return MemoryComputeKey.of(this.getId(), new RangeGlobalStep.RangeBiOperator(this.limit), false, true);
+ return MemoryComputeKey.of(this.getId(), new RangeGlobalStep.RangeBiOperator<>(this.limit), false, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderGlobalStep.java
index cfec540..1daa31a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderGlobalStep.java
@@ -138,13 +138,13 @@ public final class OrderGlobalStep<S, C extends Comparable> extends CollectingBa
@Override
public MemoryComputeKey<TraverserSet<S>> getMemoryComputeKey() {
if (null == this.chainedComparator)
- this.chainedComparator = new ChainedComparator(true, this.comparators);
- return MemoryComputeKey.of(this.getId(), new OrderBiOperator(this.chainedComparator, this.limit), false, true);
+ this.chainedComparator = new ChainedComparator<>(true, this.comparators);
+ return MemoryComputeKey.of(this.getId(), new OrderBiOperator<>(this.chainedComparator, this.limit), false, true);
}
////////////////
- public static final class OrderBiOperator implements BinaryOperator<TraverserSet>, Serializable {
+ public static final class OrderBiOperator<S> implements BinaryOperator<TraverserSet<S>>, Serializable {
private ChainedComparator chainedComparator;
private long limit;
@@ -153,13 +153,13 @@ public final class OrderGlobalStep<S, C extends Comparable> extends CollectingBa
// for serializers that need a no-arg constructor
}
- public OrderBiOperator(final ChainedComparator chainedComparator, final long limit) {
+ public OrderBiOperator(final ChainedComparator<S,?> chainedComparator, final long limit) {
this.chainedComparator = chainedComparator;
this.limit = limit;
}
@Override
- public TraverserSet apply(final TraverserSet setA, final TraverserSet setB) {
+ public TraverserSet<S> apply(final TraverserSet<S> setA, final TraverserSet<S> setB) {
setA.addAll(setB);
if (Long.MAX_VALUE != this.limit && setA.bulkSize() > this.limit) {
if (this.chainedComparator.isShuffle())
@@ -167,9 +167,9 @@ public final class OrderGlobalStep<S, C extends Comparable> extends CollectingBa
else
setA.sort(this.chainedComparator);
long counter = 0l;
- final Iterator<Traverser.Admin> traversers = setA.iterator();
+ final Iterator<Traverser.Admin<S>> traversers = setA.iterator();
while (traversers.hasNext()) {
- final Traverser.Admin traverser = traversers.next();
+ final Traverser.Admin<S> traverser = traversers.next();
if (counter > this.limit)
traversers.remove();
counter = counter + traverser.bulk();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileSideEffectStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileSideEffectStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileSideEffectStep.java
index 7a9523e..e2cf947 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileSideEffectStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileSideEffectStep.java
@@ -60,8 +60,8 @@ public final class ProfileSideEffectStep<S> extends SideEffectStep<S> implements
ret = super.next();
return ret;
} finally {
- if (ret == null) {
- finalizeMetrics();
+ if (!this.onGraphComputer && ret == null) {
+ ((DefaultTraversalMetrics) this.getTraversal().getSideEffects().get(this.sideEffectKey)).setMetrics(this.getTraversal(), false);
}
}
}
@@ -69,15 +69,17 @@ public final class ProfileSideEffectStep<S> extends SideEffectStep<S> implements
@Override
public boolean hasNext() {
boolean ret = super.hasNext();
- if (!ret) {
- finalizeMetrics();
+ if (!this.onGraphComputer && !ret) {
+ ((DefaultTraversalMetrics) this.getTraversal().getSideEffects().get(this.sideEffectKey)).setMetrics(this.getTraversal(), false);
}
return ret;
}
- private void finalizeMetrics() {
- final DefaultTraversalMetrics tm = this.getTraversal().getSideEffects().get(this.sideEffectKey);
- tm.setMetrics(traversal, onGraphComputer);
+ @Override
+ public DefaultTraversalMetrics generateFinalResult(final DefaultTraversalMetrics tm) {
+ if (this.onGraphComputer)
+ tm.setMetrics(this.getTraversal(), true);
+ return tm;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/CollectingBarrierStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/CollectingBarrierStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/CollectingBarrierStep.java
index f3bd457..f9c85a2 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/CollectingBarrierStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/CollectingBarrierStep.java
@@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
import java.util.Collections;
import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.function.BinaryOperator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -135,6 +136,6 @@ public abstract class CollectingBarrierStep<S> extends AbstractStep<S, S> implem
@Override
public MemoryComputeKey<TraverserSet<S>> getMemoryComputeKey() {
- return MemoryComputeKey.of(this.getId(), Operator.addAll, false, true);
+ return MemoryComputeKey.of(this.getId(), (BinaryOperator) Operator.addAll, false, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ProfileStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ProfileStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ProfileStep.java
index 26ba878..840fac0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ProfileStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ProfileStep.java
@@ -18,19 +18,24 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.util;
+import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
+import org.apache.tinkerpop.gremlin.process.traversal.step.MemoryComputing;
import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling;
import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
import java.util.NoSuchElementException;
+import java.util.function.BinaryOperator;
/**
* @author Bob Briody (http://bobbriody.com)
*/
-public final class ProfileStep<S> extends AbstractStep<S, S> {
+public final class ProfileStep<S> extends AbstractStep<S, S> implements MemoryComputing<MutableMetrics>, GraphComputing {
private MutableMetrics metrics;
+ private boolean onGraphComputer = false;
public ProfileStep(final Traversal.Admin traversal) {
super(traversal);
@@ -51,8 +56,16 @@ public final class ProfileStep<S> extends AbstractStep<S, S> {
} finally {
if (ret != null) {
metrics.finish(ret.bulk());
+ if(this.onGraphComputer) {
+ this.getTraversal().getSideEffects().add(this.getId(), metrics);
+ this.metrics = null;
+ }
} else {
metrics.stop();
+ if(this.onGraphComputer) {
+ this.getTraversal().getSideEffects().add(this.getId(), metrics);
+ this.metrics = null;
+ }
}
}
}
@@ -81,4 +94,30 @@ public final class ProfileStep<S> extends AbstractStep<S, S> {
}
}
+ @Override
+ public MemoryComputeKey<MutableMetrics> getMemoryComputeKey() {
+ return MemoryComputeKey.of(this.getId(), ProfileBiOperator.instance(), false, true);
+ }
+
+ @Override
+ public void onGraphComputer() {
+ this.onGraphComputer = true;
+ }
+
+ /////
+
+ public static class ProfileBiOperator implements BinaryOperator<MutableMetrics> {
+
+ private static final ProfileBiOperator INSTANCE = new ProfileBiOperator();
+
+ @Override
+ public MutableMetrics apply(final MutableMetrics metrics, final MutableMetrics metrics2) {
+ metrics.aggregate(metrics2);
+ return metrics;
+ }
+
+ public static final ProfileBiOperator instance() {
+ return INSTANCE;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/SupplyingBarrierStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/SupplyingBarrierStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/SupplyingBarrierStep.java
index 9cccb67..51ae06d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/SupplyingBarrierStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/SupplyingBarrierStep.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementExce
import java.util.Iterator;
import java.util.NoSuchElementException;
+import java.util.function.BinaryOperator;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -107,6 +108,6 @@ public abstract class SupplyingBarrierStep<S, E> extends AbstractStep<S, E> impl
@Override
public MemoryComputeKey<Boolean> getMemoryComputeKey() {
- return MemoryComputeKey.of(this.getId(), Operator.and, false, true);
+ return MemoryComputeKey.of(this.getId(), (BinaryOperator) Operator.and, false, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
index 85ce73b..8199f3a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
@@ -50,7 +50,7 @@ public final class ComputerVerificationStrategy extends AbstractTraversalStrateg
private static final ComputerVerificationStrategy INSTANCE = new ComputerVerificationStrategy();
private static final Set<Class<?>> UNSUPPORTED_STEPS = new HashSet<>(Arrays.asList(
- InjectStep.class, Mutating.class, SubgraphStep.class, ProfileStep.class, ProfileSideEffectStep.class, ComputerResultStep.class
+ InjectStep.class, Mutating.class, SubgraphStep.class, ComputerResultStep.class
));
private ComputerVerificationStrategy() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversalMetrics.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversalMetrics.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversalMetrics.java
index a2d2812..740cec8 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversalMetrics.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/DefaultTraversalMetrics.java
@@ -25,7 +25,14 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
/**
@@ -182,18 +189,25 @@ public final class DefaultTraversalMetrics implements TraversalMetrics, Serializ
}
public void setMetrics(Traversal.Admin traversal, boolean onGraphComputer) {
- addTopLevelMetrics(traversal);
+ addTopLevelMetrics(traversal, onGraphComputer);
handleNestedTraversals(traversal, null, onGraphComputer);
computeTotals();
}
- private void addTopLevelMetrics(Traversal.Admin traversal) {
+ private void addTopLevelMetrics(Traversal.Admin traversal, final boolean onGraphComputer) {
final List<ProfileStep> profileSteps = TraversalHelper.getStepsOfClass(ProfileStep.class, traversal);
for (int ii = 0; ii < profileSteps.size(); ii++) {
// The index is necessary to ensure that step order is preserved after a merge.
- final MutableMetrics stepMetrics = profileSteps.get(ii).getMetrics();
- this.indexToLabelMap.put(ii, stepMetrics.getId());
- this.metrics.put(stepMetrics.getId(), stepMetrics);
+ final ProfileStep step = profileSteps.get(ii);
+ if (onGraphComputer) {
+ final MutableMetrics stepMetrics = traversal.getSideEffects().get(step.getId());
+ this.indexToLabelMap.put(ii, stepMetrics.getId());
+ this.metrics.put(stepMetrics.getId(), stepMetrics);
+ } else {
+ final MutableMetrics stepMetrics = step.getMetrics();
+ this.indexToLabelMap.put(ii, stepMetrics.getId());
+ this.metrics.put(stepMetrics.getId(), stepMetrics);
+ }
}
}
@@ -202,20 +216,22 @@ public final class DefaultTraversalMetrics implements TraversalMetrics, Serializ
for (int ii = 0; ii < traversal.getSteps().size(); ii++) {
Step step = (Step) traversal.getSteps().get(ii);
if (step instanceof ProfileStep) {
- final MutableMetrics metrics = ((ProfileStep) step).getMetrics();
- if (!onGraphComputer) {
+ final MutableMetrics metrics = onGraphComputer ?
+ traversal.getSideEffects().get(step.getId()) :
+ ((ProfileStep) step).getMetrics();
+ if (!onGraphComputer && null != metrics) {
// save duration before we adjust
long tempDur = metrics.getDuration(TimeUnit.NANOSECONDS);
// adjust duration
metrics.setDuration(metrics.getDuration(TimeUnit.NANOSECONDS) - prevDur, TimeUnit.NANOSECONDS);
prevDur = tempDur;
}
- if (parentMetrics != null) {
+ if (parentMetrics != null && null != metrics) {
parentMetrics.addNested(metrics);
}
} else if (step instanceof TraversalParent) {
for (Traversal.Admin<?, ?> t : ((TraversalParent) step).getLocalChildren()) {
- handleNestedTraversals(t, ((ProfileStep) step.getNextStep()).getMetrics(), onGraphComputer);
+ handleNestedTraversals(t, ((ProfileStep) step.getNextStep()).getMetrics(), false);
}
for (Traversal.Admin<?, ?> t : ((TraversalParent) step).getGlobalChildren()) {
handleNestedTraversals(t, ((ProfileStep) step.getNextStep()).getMetrics(), onGraphComputer);
@@ -223,4 +239,29 @@ public final class DefaultTraversalMetrics implements TraversalMetrics, Serializ
}
}
}
+
+ /* private void handleNestedTraversals(Traversal.Admin traversal, MutableMetrics parentMetrics, boolean onGraphComputer) {
+ for (int ii = 0; ii < traversal.getSteps().size(); ii++) {
+ Step step = (Step) traversal.getSteps().get(ii);
+ if (step instanceof ProfileStep) {
+ if (onGraphComputer) {
+ if (parentMetrics != null) {
+ parentMetrics.addNested(traversal.getSideEffects().get(step.getId()));
+ }
+ } else {
+ final MutableMetrics metrics = ((ProfileStep) step).getMetrics();
+ if (parentMetrics != null && null != metrics) {
+ parentMetrics.addNested(metrics);
+ }
+ }
+ } else if (step instanceof TraversalParent) {
+ for (Traversal.Admin<?, ?> t : ((TraversalParent) step).getLocalChildren()) {
+ handleNestedTraversals(t, ((ProfileStep) step.getNextStep()).getMetrics(), false);
+ }
+ for (Traversal.Admin<?, ?> t : ((TraversalParent) step).getGlobalChildren()) {
+ handleNestedTraversals(t, ((ProfileStep) step.getNextStep()).getMetrics(), onGraphComputer);
+ }
+ }
+ }
+ }*/
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProfileTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProfileTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProfileTest.java
index d8d281d..d0ce91c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProfileTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ProfileTest.java
@@ -21,8 +21,11 @@ package org.apache.tinkerpop.gremlin.process.traversal.step.map;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
-import org.apache.tinkerpop.gremlin.process.IgnoreEngine;
-import org.apache.tinkerpop.gremlin.process.traversal.*;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
@@ -44,7 +47,9 @@ import java.util.concurrent.TimeUnit;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.GRATEFUL;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.both;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
/**
* @author Bob Briody (http://bobbriody.com)
@@ -80,7 +85,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
*/
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_out_out_profile_simple() {
final Traversal<Vertex, TraversalMetrics> traversal = get_g_V_out_out_profile();
printTraversalForm(traversal);
@@ -102,7 +107,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_out_out_profile_simpleXmetrics_keyX() {
final Traversal<Vertex, Vertex> traversal = get_g_V_out_out_profileXmetrics_keyX();
printTraversalForm(traversal);
@@ -232,7 +237,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile() {
final Traversal<Vertex, TraversalMetrics> traversal = get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile();
printTraversalForm(traversal);
@@ -243,7 +248,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profileXmetrics_keyX() {
final Traversal<Vertex, Vertex> traversal = get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profileXmetrics_keyX();
printTraversalForm(traversal);
@@ -290,7 +295,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_repeat_both_modern_profile() {
final Traversal<Vertex, TraversalMetrics> traversal = get_g_V_repeatXbothX_timesX3X_profile();
printTraversalForm(traversal);
@@ -301,7 +306,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_V_repeat_both_modern_profileXmetrics_keyX() {
final Traversal<Vertex, Vertex> traversal = get_g_V_repeatXbothX_timesX3X_profileXmetrics_keyX();
printTraversalForm(traversal);
@@ -382,7 +387,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void testProfileStrategyCallback() {
final Traversal<Vertex, TraversalMetrics> t = get_g_V_out_out_profile();
MockStep mockStep = new MockStep(t.asAdmin());
@@ -390,14 +395,14 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
TraversalMetrics traversalMetrics = t.next();
assertTrue(mockStep.callbackCalled);
- if (!TraversalHelper.onGraphComputer(t.asAdmin())) {
+ if (!onGraphComputer(t.asAdmin())) {
assertEquals(100, traversalMetrics.getMetrics(3).getCount("bogusCount").longValue());
}
}
@Test
@LoadGraphWith(MODERN)
- @IgnoreEngine(TraversalEngine.Type.COMPUTER)
+ //@IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void testProfileStrategyCallbackSideEffect() {
final Traversal<Vertex, Vertex> t = get_g_V_out_out_profileXmetrics_keyX();
MockStep mockStep = new MockStep(t.asAdmin());
@@ -405,7 +410,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
t.iterate();
assertTrue(mockStep.callbackCalled);
- if (!TraversalHelper.onGraphComputer(t.asAdmin())) {
+ if (!onGraphComputer(t.asAdmin())) {
final TraversalMetrics traversalMetrics = t.asAdmin().getSideEffects().<TraversalMetrics>get(METRICS_KEY);
assertEquals(100, traversalMetrics.getMetrics(3).getCount("bogusCount").longValue());
}
@@ -427,6 +432,10 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
traversal.iterate();
}
+ private static boolean onGraphComputer(final Traversal.Admin<?, ?> traversal) {
+ return !TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, TraversalHelper.getRootTraversal(traversal)).isEmpty();
+ }
+
/**
* Traversals
*/
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/58615c04/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
index 6dbedc2..4425e65 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
@@ -57,13 +57,14 @@ public class TinkerGraphPlayTest {
GraphTraversalSource g = graph.traversal().withComputer();//GraphTraversalSource.computer());
//System.out.println(g.V().outE("knows").identity().inV().count().is(P.eq(5)).explain());
//System.out.println(g.V().hasLabel("person").fold().order(Scope.local).by("age").toList());
- final Traversal<?,?> traversal = g.V().out("created").group("m").by(T.label).pageRank(1.0).by("pageRank").by(__.inE()).times(1).in("created").group("m").by("pageRank").cap("m");
-
- System.out.println(traversal.asAdmin().clone().toString());
- final Traversal<?,?> clone = traversal.asAdmin().clone();
- clone.asAdmin().applyStrategies();
- System.out.println(clone);
- System.out.println(traversal.asAdmin().clone().toList());
+ final Traversal<?,?> traversal = g.V().where(__.in("created").count().is(1l)).<String>values("name").profile("m");
+
+ //System.out.println(traversal.asAdmin().clone().toString());
+ // final Traversal<?,?> clone = traversal.asAdmin().clone();
+ // clone.asAdmin().applyStrategies();
+ // System.out.println(clone);
+ System.out.println(traversal.asAdmin().toList());
+ System.out.println(traversal.asAdmin().getSideEffects().get("m") + " ");
//System.out.println(g.V().pageRank().order().by(PageRankVertexProgram.PAGE_RANK).valueMap().toList());
}