You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2017/01/09 14:54:19 UTC
[07/17] tinkerpop git commit: discovering various synchronization
bottlenecks in TinkerGraphComputer. Also,
realized some dumb things I was doing in TraversalVertexProgram. Its crazy,
for this benchmark that @dkuppitz and i have, if I don't touch vertex.
discovering various synchronization bottlenecks in TinkerGraphComputer. Also, realized some dumb things I was doing in TraversalVertexProgram. Its crazy, for this benchmark that @dkuppitz and i have, if I don't touch vertex.properties that are compute keys: millisecond return times. If I do, seconds return times.... Need to figure out how to partition TinkerGraphView... Perhaps thread local..dah.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/8d961285
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/8d961285
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/8d961285
Branch: refs/heads/tp32
Commit: 8d961285f687107ffc5c1d8a3bca7c78ea833adf
Parents: cef1979
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Jan 4 10:32:21 2017 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Jan 5 17:00:05 2017 -0700
----------------------------------------------------------------------
.../traversal/TraversalVertexProgram.java | 16 +++++++---
.../computer/traversal/WorkerExecutor.java | 2 +-
.../computer/TinkerGraphComputerView.java | 32 +++++++++-----------
.../groovy/TinkerGraphGroovyPlayTest.groovy | 2 +-
4 files changed, 27 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index 1a54721..b82e265 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -32,7 +32,6 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ComputerResultStep;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy;
import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
import org.apache.tinkerpop.gremlin.process.computer.util.SingleMessenger;
@@ -51,6 +50,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal;
@@ -251,8 +251,14 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
((Barrier) this.traversalMatrix.getStepById(stepId)).done();
}
// define halted traversers
- final TraverserSet<Object> haltedTraversers = vertex.<TraverserSet<Object>>property(HALTED_TRAVERSERS).orElse(new TraverserSet<>());
- vertex.property(VertexProperty.Cardinality.single, HALTED_TRAVERSERS, haltedTraversers);
+ final VertexProperty<TraverserSet<Object>> property = vertex.property(HALTED_TRAVERSERS);
+ final TraverserSet<Object> haltedTraversers;
+ if (property.isPresent()) {
+ haltedTraversers = property.value();
+ } else {
+ haltedTraversers = new TraverserSet<>();
+ vertex.property(VertexProperty.Cardinality.single, HALTED_TRAVERSERS, haltedTraversers);
+ }
//////////////////
if (memory.isInitialIteration()) { // ITERATION 1
final TraverserSet<Object> activeTraversers = new TraverserSet<>();
@@ -282,9 +288,9 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
activeTraversers.add((Traverser.Admin) traverser);
});
}
- memory.add(VOTE_TO_HALT, activeTraversers.isEmpty() || WorkerExecutor.execute(vertex, new SingleMessenger<>(messenger, activeTraversers), this.traversalMatrix, memory, this.returnHaltedTraversers, this.haltedTraverserStrategy));
+ memory.add(VOTE_TO_HALT, activeTraversers.isEmpty() || WorkerExecutor.execute(vertex, new SingleMessenger<>(messenger, activeTraversers), this.traversalMatrix, memory, this.returnHaltedTraversers, haltedTraversers, this.haltedTraverserStrategy));
} else // ITERATION 1+
- memory.add(VOTE_TO_HALT, WorkerExecutor.execute(vertex, messenger, this.traversalMatrix, memory, this.returnHaltedTraversers, this.haltedTraverserStrategy));
+ memory.add(VOTE_TO_HALT, WorkerExecutor.execute(vertex, messenger, this.traversalMatrix, memory, this.returnHaltedTraversers, haltedTraversers, this.haltedTraverserStrategy));
// save space by not having an empty halted traversers property
if (this.returnHaltedTraversers || haltedTraversers.isEmpty())
vertex.<TraverserSet>property(HALTED_TRAVERSERS).remove();
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
index 2571e7b..e6e73d0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
@@ -59,10 +59,10 @@ final class WorkerExecutor {
final TraversalMatrix<?, ?> traversalMatrix,
final Memory memory,
final boolean returnHaltedTraversers,
+ final TraverserSet<Object> haltedTraversers,
final HaltedTraverserStrategy haltedTraverserStrategy) {
final TraversalSideEffects traversalSideEffects = traversalMatrix.getTraversal().getSideEffects();
final AtomicBoolean voteToHalt = new AtomicBoolean(true);
- final TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS);
final TraverserSet<Object> activeTraversers = new TraverserSet<>();
final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
index 43090fe..43998fb 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
@@ -43,8 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -104,11 +102,14 @@ public final class TinkerGraphComputerView {
}
public List<Property> getProperties(final TinkerVertex vertex) {
- final Stream<Property> a = TinkerHelper.getProperties(vertex).values().stream().flatMap(list -> list.stream());
- final Stream<Property> b = this.computeProperties.containsKey(vertex) ?
- this.computeProperties.get(vertex).values().stream().flatMap(list -> list.stream()) :
- Stream.empty();
- return Stream.concat(a, b).collect(Collectors.toList());
+ final List<Property> list = new ArrayList<>();
+ for (final List<VertexProperty> properties : TinkerHelper.getProperties(vertex).values()) {
+ list.addAll(properties);
+ }
+ for (final List<VertexProperty<?>> properties : this.computeProperties.getOrDefault(vertex, Collections.emptyMap()).values()) {
+ list.addAll(properties);
+ }
+ return list;
}
public void removeProperty(final TinkerVertex vertex, final String key, final VertexProperty property) {
@@ -131,8 +132,9 @@ public final class TinkerGraphComputerView {
// remove all transient properties from the vertices
for (final VertexComputeKey computeKey : this.computeKeys.values()) {
if (computeKey.isTransient()) {
- final List<VertexProperty<?>> toRemove = this.computeProperties.values().stream().flatMap(map -> map.getOrDefault(computeKey.getKey(), Collections.emptyList()).stream()).collect(Collectors.toList());
- toRemove.forEach(VertexProperty::remove);
+ for (final Map<String, List<VertexProperty<?>>> properties : this.computeProperties.values()) {
+ properties.remove(computeKey.getKey());
+ }
}
}
}
@@ -211,22 +213,16 @@ public final class TinkerGraphComputerView {
}
private void addValue(final Vertex vertex, final String key, final VertexProperty property) {
- final Map<String, List<VertexProperty<?>>> elementProperties = this.computeProperties.computeIfAbsent(vertex, k -> new ConcurrentHashMap<>());
+ final Map<String, List<VertexProperty<?>>> elementProperties = this.computeProperties.computeIfAbsent(vertex, k -> new HashMap<>());
elementProperties.compute(key, (k, v) -> {
- if (null == v) v = Collections.synchronizedList(new ArrayList<>());
+ if (null == v) v = new ArrayList<>();
v.add(property);
return v;
});
}
private void removeValue(final Vertex vertex, final String key, final VertexProperty property) {
- this.computeProperties.computeIfPresent(vertex, (k, v) -> {
- v.computeIfPresent(key, (k1, v1) -> {
- v1.remove(property);
- return v1;
- });
- return v;
- });
+ this.computeProperties.<List<Map<String, VertexProperty<?>>>>getOrDefault(vertex, Collections.emptyMap()).get(key).remove(property);
}
private List<VertexProperty<?>> getValue(final Vertex vertex, final String key) {
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy
index d277977..a736f3d 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy
@@ -40,7 +40,7 @@ class TinkerGraphGroovyPlayTest {
def a = graph.traversal().withComputer(Computer.compute())
def r = new Random(123)
- (1..1725403).each {
+ (1..1000000).each {
def vid = ["a", "b", "c", "d"].collectEntries { [it, r.nextInt() % 400000] }
graph.addVertex(T.id, vid)
}; []