You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/29 18:54:11 UTC
[1/2] incubator-tinkerpop git commit: now that
TraversalVertexProgram.terminate() is so complex,
a bug in GiraphMemory popped up. Created PassthroughMemory as a fix for
Giraph. Basically,
its a wrapper around GiraphMemory where all set() operations go in
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1166 200348513 -> fb349865d
now that TraversalVertexProgram.terminate() is so complex, a bug in GiraphMemory popped up. Created PassthroughMemory as a fix for Giraph. Basically, its a wrapper around GiraphMemory where all set() operations go into a local map and to the Giraph aggregator. However, all get()s are from the local map -- since get() doesn't work on the same iteration as terminate for Giraph.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/7d38cee2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/7d38cee2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/7d38cee2
Branch: refs/heads/TINKERPOP-1166
Commit: 7d38cee2395980e467972f342c8ee0b5cf878f04
Parents: 2003485
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Feb 29 10:42:16 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Feb 29 10:42:16 2016 -0700
----------------------------------------------------------------------
.../giraph/process/computer/GiraphMemory.java | 5 +-
.../process/computer/PassthroughMemory.java | 100 +++++++++++++++++++
.../traversal/TraversalVertexProgram.java | 2 +-
.../computer/traversal/TraverserExecutor.java | 4 +-
.../process/computer/util/MapMemory.java | 2 +-
5 files changed, 105 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
index 752ba41..56428f3 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
@@ -31,7 +31,6 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.javatuples.Pair;
@@ -92,7 +91,7 @@ public final class GiraphMemory extends MasterCompute implements Memory {
this.vertexProgram.setup(this);
} else {
// a hack to get the last iteration memory values to stick
- final MapMemory memory = new MapMemory(this);
+ final PassThroughMemory memory = new PassThroughMemory(this);
if (this.vertexProgram.terminate(memory)) { // terminate
final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
if (null != outputLocation) {
@@ -114,8 +113,6 @@ public final class GiraphMemory extends MasterCompute implements Memory {
}
this.haltComputation();
}
- // to ensure proper terminate that is not the "hack" above
- this.vertexProgram.terminate(this);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
new file mode 100644
index 0000000..998110f
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class PassThroughMemory implements Memory.Admin, Serializable {
+
+ private final GiraphMemory giraphMemory;
+ private long runtime = 0l;
+ private int iteration = -1;
+ private final Map<String, Object> memoryMap = new HashMap<>();
+
+ public PassThroughMemory(final GiraphMemory giraphMemory) {
+ this.giraphMemory = giraphMemory;
+ giraphMemory.keys().forEach(key -> this.memoryMap.put(key, giraphMemory.get(key)));
+ this.iteration = giraphMemory.getIteration();
+ }
+
+ @Override
+ public Set<String> keys() {
+ return this.memoryMap.keySet();
+ }
+
+ @Override
+ public <R> R get(final String key) throws IllegalArgumentException {
+ final R r = (R) this.memoryMap.get(key);
+ if (null == r)
+ throw Memory.Exceptions.memoryDoesNotExist(key);
+ else
+ return r;
+ }
+
+ @Override
+ public void set(final String key, Object value) {
+ this.memoryMap.put(key, value);
+ this.giraphMemory.set(key, value);
+ }
+
+ @Override
+ public int getIteration() {
+ return this.iteration;
+ }
+
+ @Override
+ public long getRuntime() {
+ return this.runtime;
+ }
+
+ @Override
+ public void add(final String key, final Object value) {
+ this.giraphMemory.add(key, value);
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.memoryString(this);
+ }
+
+ @Override
+ public void incrIteration() {
+ this.iteration = this.iteration + 1;
+ }
+
+ @Override
+ public void setIteration(final int iteration) {
+ this.iteration = iteration;
+ }
+
+ @Override
+ public void setRuntime(long runtime) {
+ this.runtime = runtime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index b1ab50d..9829cff 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -286,7 +286,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
haltedTraversers.add((Traverser.Admin) traverser);
} else if (traverser.get() instanceof Attachable &&
!(traverser.get() instanceof Path) &&
- !TraversalHelper.isLocalElement(this.traversalMatrix.getStepById(traverser.getStepId()))) {
+ !TraversalHelper.isLocalElement(this.traversalMatrix.getStepById(traverser.getStepId()))) { // this is so that patterns like order().name work as expected.
remoteActiveTraversers.add((Traverser.Admin) traverser);
} else {
final Step<?, ?> currentStep = this.traversalMatrix.getStepById(traverser.getStepId());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
index 1025351..4c63c24 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
@@ -53,8 +53,8 @@ public final class TraverserExecutor {
final TraverserSet<Object> activeTraversers = new TraverserSet<>();
final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
- final TraverserSet<Object> maybeAliveTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
- final Iterator<Traverser.Admin<Object>> iterator = maybeAliveTraversers.iterator();
+ final TraverserSet<Object> maybeActiveTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
+ final Iterator<Traverser.Admin<Object>> iterator = maybeActiveTraversers.iterator();
while (iterator.hasNext()) {
final Traverser.Admin<Object> traverser = iterator.next();
if (vertex.equals(TraverserExecutor.getHostingVertex(traverser.get()))) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
index 74e1fc1..e8edbc1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
@@ -74,7 +74,7 @@ public final class MapMemory implements Memory.Admin, Serializable {
@Override
public void set(final String key, Object value) {
- // this.checkKeyValue(key, value);
+ this.checkKeyValue(key, value);
this.memoryMap.put(key, value);
}
[2/2] incubator-tinkerpop git commit: minor nothings. running full
integration tests on this branch and then will VOTE it out.
Posted by ok...@apache.org.
minor nothings. running full integration tests on this branch and then will VOTE it out.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/fb349865
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/fb349865
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/fb349865
Branch: refs/heads/TINKERPOP-1166
Commit: fb349865da343f7a0f0b80a4f5faaa5b16d8bc8a
Parents: 7d38cee
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Feb 29 10:53:51 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Feb 29 10:53:51 2016 -0700
----------------------------------------------------------------------
.../gremlin/giraph/process/computer/PassthroughMemory.java | 3 +--
.../apache/tinkerpop/gremlin/process/computer/util/MapMemory.java | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/fb349865/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
index 998110f..dc40e7b 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
@@ -22,7 +22,6 @@ package org.apache.tinkerpop.gremlin.giraph.process.computer;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -30,7 +29,7 @@ import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class PassThroughMemory implements Memory.Admin, Serializable {
+public final class PassThroughMemory implements Memory.Admin {
private final GiraphMemory giraphMemory;
private long runtime = 0l;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/fb349865/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
index e8edbc1..74e1fc1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
@@ -74,7 +74,7 @@ public final class MapMemory implements Memory.Admin, Serializable {
@Override
public void set(final String key, Object value) {
- this.checkKeyValue(key, value);
+ // this.checkKeyValue(key, value);
this.memoryMap.put(key, value);
}