You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/02/16 18:58:10 UTC
incubator-tinkerpop git commit:
GraphComputer.supportsNonSerializedObjects means that the GraphComputer
doesn't need to serialize parameters across the wire. Only
TinkerGraphComputer supports this and allows lambdas in its
TraversalVertexProgram traversa
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 6640db010 -> 1885300d7
GraphComputer.supportsNonSerializedObjects means that the GraphComputer doesn't need to serialize parameters across the wire. Only TinkerGraphComputer supports this and allows lambdas in its TraversalVertexProgram traversals.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1885300d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1885300d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1885300d
Branch: refs/heads/master
Commit: 1885300d7d1f9176d14aa44946810d0f1e87b11f
Parents: 6640db0
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Feb 16 10:58:10 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Feb 16 10:58:10 2015 -0700
----------------------------------------------------------------------
.../gremlin/process/computer/GraphComputer.java | 4 ++
.../computer/traversal/TraversalSupplier.java | 10 ++++-
.../traversal/TraversalVertexProgram.java | 9 ++++-
.../traversal/step/map/ComputerResultStep.java | 3 +-
.../mapreduce/TraverserMapReduce.java | 4 +-
.../process/computer/util/LambdaHolder.java | 6 +++
.../traversal/lambda/IdentityTraversal.java | 4 +-
.../structure/ExceptionCoverageTest.java | 1 +
.../computer/giraph/GiraphGraphComputer.java | 42 ++++++++++++--------
.../process/computer/TinkerGraphComputer.java | 1 -
.../tinkergraph/structure/TinkerGraphTest.java | 4 +-
11 files changed, 61 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
index a5c2f55..674cb32 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
@@ -130,6 +130,10 @@ public interface GraphComputer {
public default boolean supportsIsolation(final Isolation isolation) {
return true;
}
+
+ public default boolean supportsNonSerializableObjects() {
+ return true;
+ }
}
public static class Exceptions {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalSupplier.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalSupplier.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalSupplier.java
index 191e434..4925116 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalSupplier.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalSupplier.java
@@ -29,13 +29,19 @@ import java.util.function.Supplier;
public final class TraversalSupplier<S, E> implements Supplier<Traversal.Admin<S, E>>, Serializable {
private final Traversal.Admin<S, E> traversal;
+ private final boolean cloneOnGet;
- public TraversalSupplier(final Traversal.Admin<S, E> traversal) {
+ public TraversalSupplier(final Traversal.Admin<S, E> traversal, final boolean cloneOnGet) {
this.traversal = traversal;
+ this.cloneOnGet = cloneOnGet;
}
@Override
public Traversal.Admin<S, E> get() {
- return this.traversal;
+ try {
+ return this.cloneOnGet ? this.traversal.clone() : this.traversal;
+ } catch (final CloneNotSupportedException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
index cf9a00f..c706e77 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java
@@ -251,7 +251,14 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet<
}
public Builder traversal(final Traversal.Admin traversal) {
- LambdaHolder.storeState(this.configuration, LambdaHolder.Type.OBJECT, TRAVERSAL_SUPPLIER, new TraversalSupplier(traversal));
+ return this.traversal(traversal, true);
+ }
+
+ public Builder traversal(final Traversal.Admin traversal, boolean serialize) {
+ if (serialize)
+ LambdaHolder.storeState(this.configuration, LambdaHolder.Type.SERIALIZED_OBJECT, TRAVERSAL_SUPPLIER, new TraversalSupplier(traversal, false));
+ else
+ LambdaHolder.storeState(this.configuration, LambdaHolder.Type.OBJECT, TRAVERSAL_SUPPLIER, new TraversalSupplier(traversal, true));
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ComputerResultStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ComputerResultStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ComputerResultStep.java
index 2811e8d..e94822a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ComputerResultStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/ComputerResultStep.java
@@ -75,8 +75,7 @@ public final class ComputerResultStep<S> extends AbstractStep<S, S> {
if (this.byPass) return this.starts.next();
if (this.first && null == this.computerResult) {
try {
- final TraversalVertexProgram vertexProgram = TraversalVertexProgram.build().traversal(this.getTraversal()).create();
- populateTraversers(this.graphComputer.program(vertexProgram).submit().get());
+ populateTraversers(this.graphComputer.program(TraversalVertexProgram.build().traversal(this.getTraversal(), !this.graphComputer.features().supportsNonSerializableObjects()).create()).submit().get());
} catch (final Exception e) {
throw new IllegalStateException(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
index 0d35dc4..044e2ed 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce;
+import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.process.Step;
import org.apache.tinkerpop.gremlin.process.Traversal;
import org.apache.tinkerpop.gremlin.process.Traverser;
@@ -32,7 +33,6 @@ import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import org.apache.commons.configuration.Configuration;
import java.util.Comparator;
import java.util.Iterator;
@@ -62,7 +62,7 @@ public final class TraverserMapReduce extends StaticMapReduce<Comparable, Object
@Override
public void loadState(final Configuration configuration) {
this.traversal = TraversalVertexProgram.getTraversalSupplier(configuration).get();
- final Step endStep = this.traversal.getEndStep();
+ final Step endStep = this.traversal.getEndStep().getPreviousStep(); // don't get the ComputerResultStep
this.comparator = Optional.ofNullable(endStep instanceof ComparatorHolder ? GraphComputerHelper.chainComparators(((ComparatorHolder) endStep).getComparators()) : null);
this.reducer = Optional.ofNullable(endStep instanceof Reducing ? ((Reducing) endStep).getReducer() : null);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/LambdaHolder.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/LambdaHolder.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/LambdaHolder.java
index b9c89b5..50f62ad 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/LambdaHolder.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/LambdaHolder.java
@@ -28,6 +28,7 @@ public class LambdaHolder<S> {
public enum Type {
OBJECT,
+ SERIALIZED_OBJECT,
CLASS,
SCRIPT
}
@@ -73,6 +74,9 @@ public class LambdaHolder<S> {
lambdaHolder.configKeyPrefix = configKeyPrefix;
lambdaHolder.type = Type.valueOf(configuration.getString(lambdaHolder.configKeyPrefix.concat(DOT_TYPE)));
if (lambdaHolder.type.equals(Type.OBJECT)) {
+ lambdaHolder.configObject = configuration.getProperty(lambdaHolder.configKeyPrefix.concat(DOT_OBJECT));
+ lambdaHolder.realObject = lambdaHolder.configObject;
+ } else if (lambdaHolder.type.equals(Type.SERIALIZED_OBJECT)) {
lambdaHolder.configObject = VertexProgramHelper.deserialize(configuration, lambdaHolder.configKeyPrefix.concat(DOT_OBJECT));
lambdaHolder.realObject = lambdaHolder.configObject;
} else if (lambdaHolder.type.equals(Type.CLASS)) {
@@ -95,6 +99,8 @@ public class LambdaHolder<S> {
public void storeState(final Configuration configuration) {
configuration.setProperty(this.configKeyPrefix.concat(DOT_TYPE), this.type.name());
if (this.type.equals(Type.OBJECT))
+ configuration.setProperty(this.configKeyPrefix.concat(DOT_OBJECT), this.configObject);
+ else if (this.type.equals(Type.SERIALIZED_OBJECT))
VertexProgramHelper.serialize(this.configObject, configuration, this.configKeyPrefix.concat(DOT_OBJECT));
else if (this.type.equals(Type.CLASS))
configuration.setProperty(this.configKeyPrefix.concat(DOT_OBJECT), ((Class) this.configObject).getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/lambda/IdentityTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/lambda/IdentityTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/lambda/IdentityTraversal.java
index 7b85784..debe515 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/lambda/IdentityTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/lambda/IdentityTraversal.java
@@ -25,6 +25,8 @@ import org.apache.tinkerpop.gremlin.process.Traverser;
*/
public final class IdentityTraversal<S, E> extends AbstractLambdaTraversal<S, E> {
+ private static final String IDENTITY = "identity";
+
private S s;
@Override
@@ -39,6 +41,6 @@ public final class IdentityTraversal<S, E> extends AbstractLambdaTraversal<S, E>
@Override
public String toString() {
- return "";
+ return IDENTITY;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/structure/ExceptionCoverageTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/structure/ExceptionCoverageTest.java b/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/structure/ExceptionCoverageTest.java
index 7f3b08c..b8a417f 100644
--- a/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/structure/ExceptionCoverageTest.java
+++ b/gremlin-test/src/test/java/org/apache/tinkerpop/gremlin/structure/ExceptionCoverageTest.java
@@ -69,6 +69,7 @@ public class ExceptionCoverageTest {
add("org.apache.tinkerpop.gremlin.process.computer.GraphComputer$Exceptions#adjacentElementPropertiesCanNotBeRead");
add("org.apache.tinkerpop.gremlin.process.computer.GraphComputer$Exceptions#adjacentElementPropertiesCanNotBeWritten");
add("org.apache.tinkerpop.gremlin.process.computer.GraphComputer$Exceptions#adjacentVerticesCanNotBeQueried");
+ add("org.apache.tinkerpop.gremlin.process.computer.GraphComputer$Exceptions#supportsNonSerializableObjects");
}};
// register test classes here that contain @ExceptionCoverage annotations
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
index ca7835b..e382699 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
@@ -18,6 +18,22 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.FileConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.job.GiraphJob;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io.GiraphVertexInputFormat;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io.GiraphVertexOutputFormat;
@@ -36,22 +52,6 @@ import org.apache.tinkerpop.gremlin.process.computer.util.GraphComputerHelper;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.FileConfiguration;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -231,4 +231,14 @@ public class GiraphGraphComputer extends Configured implements GraphComputer, To
throw e;
}
}
+
+ @Override
+ public Features features() {
+ return new Features() {
+ @Override
+ public boolean supportsNonSerializableObjects() {
+ return false;
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index a6fe9cd..a466bb8 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -182,5 +182,4 @@ public class TinkerGraphComputer implements GraphComputer {
return this.iterator.hasNext() ? this.iterator.next() : null;
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1885300d/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
index 344688c..3f37ba2 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
@@ -139,8 +139,8 @@ public class TinkerGraphTest {
public void testPlay3() throws Exception {
Graph g = TinkerFactory.createModern();
g.engine(ComputerTraversalEngine.instance());
- //Traversal t = g.V().label().groupCount();
- Traversal t = ComputerTestHelper.compute("g.V().out('created').groupCount().by('name').values('name')", g);
+ Traversal t = g.V().map(v -> v.get().value("name"));
+ //Traversal t = ComputerTestHelper.compute("g.V().out('created').groupCount().by('name').values('name')", g);
System.out.println(t);
t.forEachRemaining(System.out::println);
System.out.println(t);