You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/29 18:54:11 UTC

[1/2] incubator-tinkerpop git commit: now that TraversalVertexProgram.terminate() is so complex, a bug in GiraphMemory popped up. Created PassthroughMemory as a fix for Giraph. Basically, its a wrapper around GiraphMemory where all set() operations go in

Repository: incubator-tinkerpop
Updated Branches:
  refs/heads/TINKERPOP-1166 200348513 -> fb349865d


now that TraversalVertexProgram.terminate() is so complex, a bug in GiraphMemory popped up. Created PassthroughMemory as a fix for Giraph. Basically, its a wrapper around GiraphMemory where all set() operations go into a local map and to the Giraph aggregator. However, all get()s are from the local map -- since get() doesn't work on the same iteration as terminate for Giraph.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/7d38cee2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/7d38cee2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/7d38cee2

Branch: refs/heads/TINKERPOP-1166
Commit: 7d38cee2395980e467972f342c8ee0b5cf878f04
Parents: 2003485
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Feb 29 10:42:16 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Feb 29 10:42:16 2016 -0700

----------------------------------------------------------------------
 .../giraph/process/computer/GiraphMemory.java   |   5 +-
 .../process/computer/PassthroughMemory.java     | 100 +++++++++++++++++++
 .../traversal/TraversalVertexProgram.java       |   2 +-
 .../computer/traversal/TraverserExecutor.java   |   4 +-
 .../process/computer/util/MapMemory.java        |   2 +-
 5 files changed, 105 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
index 752ba41..56428f3 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
@@ -31,7 +31,6 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
 import org.apache.tinkerpop.gremlin.process.computer.Memory;
 import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
 import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
 import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
 import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
 import org.javatuples.Pair;
@@ -92,7 +91,7 @@ public final class GiraphMemory extends MasterCompute implements Memory {
             this.vertexProgram.setup(this);
         } else {
             // a hack to get the last iteration memory values to stick
-            final MapMemory memory = new MapMemory(this);
+            final PassThroughMemory memory = new PassThroughMemory(this);
             if (this.vertexProgram.terminate(memory)) { // terminate
                 final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
                 if (null != outputLocation) {
@@ -114,8 +113,6 @@ public final class GiraphMemory extends MasterCompute implements Memory {
                 }
                 this.haltComputation();
             }
-            // to ensure proper terminate that is not the "hack" above
-            this.vertexProgram.terminate(this);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
new file mode 100644
index 0000000..998110f
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class PassThroughMemory implements Memory.Admin, Serializable {
+
+    private final GiraphMemory giraphMemory;
+    private long runtime = 0l;
+    private int iteration = -1;
+    private final Map<String, Object> memoryMap = new HashMap<>();
+
+    public PassThroughMemory(final GiraphMemory giraphMemory) {
+        this.giraphMemory = giraphMemory;
+        giraphMemory.keys().forEach(key -> this.memoryMap.put(key, giraphMemory.get(key)));
+        this.iteration = giraphMemory.getIteration();
+    }
+
+    @Override
+    public Set<String> keys() {
+        return this.memoryMap.keySet();
+    }
+
+    @Override
+    public <R> R get(final String key) throws IllegalArgumentException {
+        final R r = (R) this.memoryMap.get(key);
+        if (null == r)
+            throw Memory.Exceptions.memoryDoesNotExist(key);
+        else
+            return r;
+    }
+
+    @Override
+    public void set(final String key, Object value) {
+        this.memoryMap.put(key, value);
+        this.giraphMemory.set(key, value);
+    }
+
+    @Override
+    public int getIteration() {
+        return this.iteration;
+    }
+
+    @Override
+    public long getRuntime() {
+        return this.runtime;
+    }
+
+    @Override
+    public void add(final String key, final Object value) {
+        this.giraphMemory.add(key, value);
+    }
+
+    @Override
+    public String toString() {
+        return StringFactory.memoryString(this);
+    }
+
+    @Override
+    public void incrIteration() {
+        this.iteration = this.iteration + 1;
+    }
+
+    @Override
+    public void setIteration(final int iteration) {
+        this.iteration = iteration;
+    }
+
+    @Override
+    public void setRuntime(long runtime) {
+        this.runtime = runtime;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index b1ab50d..9829cff 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -286,7 +286,7 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
                 haltedTraversers.add((Traverser.Admin) traverser);
             } else if (traverser.get() instanceof Attachable &&
                     !(traverser.get() instanceof Path) &&
-                    !TraversalHelper.isLocalElement(this.traversalMatrix.getStepById(traverser.getStepId()))) {
+                    !TraversalHelper.isLocalElement(this.traversalMatrix.getStepById(traverser.getStepId()))) {  // this is so that patterns like order().name work as expected.
                 remoteActiveTraversers.add((Traverser.Admin) traverser);
             } else {
                 final Step<?, ?> currentStep = this.traversalMatrix.getStepById(traverser.getStepId());

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
index 1025351..4c63c24 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraverserExecutor.java
@@ -53,8 +53,8 @@ public final class TraverserExecutor {
         final TraverserSet<Object> activeTraversers = new TraverserSet<>();
         final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
 
-        final TraverserSet<Object> maybeAliveTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
-        final Iterator<Traverser.Admin<Object>> iterator = maybeAliveTraversers.iterator();
+        final TraverserSet<Object> maybeActiveTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
+        final Iterator<Traverser.Admin<Object>> iterator = maybeActiveTraversers.iterator();
         while (iterator.hasNext()) {
             final Traverser.Admin<Object> traverser = iterator.next();
             if (vertex.equals(TraverserExecutor.getHostingVertex(traverser.get()))) {

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/7d38cee2/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
index 74e1fc1..e8edbc1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
@@ -74,7 +74,7 @@ public final class MapMemory implements Memory.Admin, Serializable {
 
     @Override
     public void set(final String key, Object value) {
-        // this.checkKeyValue(key, value);
+        this.checkKeyValue(key, value);
         this.memoryMap.put(key, value);
     }
 


[2/2] incubator-tinkerpop git commit: minor nothings. running full integration tests on this branch and then will VOTE it out.

Posted by ok...@apache.org.
minor nothings. running full integration tests on this branch and then will VOTE it out.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/fb349865
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/fb349865
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/fb349865

Branch: refs/heads/TINKERPOP-1166
Commit: fb349865da343f7a0f0b80a4f5faaa5b16d8bc8a
Parents: 7d38cee
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Feb 29 10:53:51 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Feb 29 10:53:51 2016 -0700

----------------------------------------------------------------------
 .../gremlin/giraph/process/computer/PassthroughMemory.java        | 3 +--
 .../apache/tinkerpop/gremlin/process/computer/util/MapMemory.java | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/fb349865/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
index 998110f..dc40e7b 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassthroughMemory.java
@@ -22,7 +22,6 @@ package org.apache.tinkerpop.gremlin.giraph.process.computer;
 import org.apache.tinkerpop.gremlin.process.computer.Memory;
 import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
 
-import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -30,7 +29,7 @@ import java.util.Set;
 /**
  * @author Marko A. Rodriguez (http://markorodriguez.com)
  */
-public final class PassThroughMemory implements Memory.Admin, Serializable {
+public final class PassThroughMemory implements Memory.Admin {
 
     private final GiraphMemory giraphMemory;
     private long runtime = 0l;

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/fb349865/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
index e8edbc1..74e1fc1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapMemory.java
@@ -74,7 +74,7 @@ public final class MapMemory implements Memory.Admin, Serializable {
 
     @Override
     public void set(final String key, Object value) {
-        this.checkKeyValue(key, value);
+        // this.checkKeyValue(key, value);
         this.memoryMap.put(key, value);
     }