You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/10/26 14:18:33 UTC
[01/29] tinkerpop git commit: added Pick.none and Pick.any to the
CoreImports. Added it to both GraphSONModule and GryoMapper. Added two new
test cases which use any and none -- one in ChooseTest and one in BranchTest.
Added none/any to GroovyTranslator [Forced Update!]
Repository: tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1389 b3364a5fe -> db8e8c106 (forced update)
added Pick.none and Pick.any to the CoreImports. Added it to both GraphSONModule and GryoMapper. Added two new test cases which use any and none -- one in ChooseTest and one in BranchTest. Added none/any to GroovyTranslator and PythonTranslator. All good in the hood.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/0fec46db
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/0fec46db
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/0fec46db
Branch: refs/heads/TINKERPOP-1389
Commit: 0fec46db21a3af0604a85b9b4bed4e47db8d1a70
Parents: 1148e82
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Oct 21 13:07:18 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Oct 21 13:07:18 2016 -0600
----------------------------------------------------------------------
.../structure/io/graphson/GraphSONModule.java | 4 +++
.../gremlin/structure/io/gryo/GryoMapper.java | 4 ++-
.../tinkerpop/gremlin/util/CoreImports.java | 6 +++-
.../step/branch/GroovyBranchTest.groovy | 13 ++++++++-
.../step/branch/GroovyChooseTest.groovy | 10 +++++++
.../gremlin/groovy/jsr223/GroovyTranslator.java | 3 ++
.../gremlin/python/jsr223/PythonTranslator.java | 3 ++
.../jython/gremlin_python/process/traversal.py | 4 +++
.../traversal/step/branch/BranchTest.java | 29 ++++++++++++++++----
.../traversal/step/branch/ChooseTest.java | 19 +++++++++++++
10 files changed, 87 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONModule.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONModule.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONModule.java
index b361fac..a644d37 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONModule.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONModule.java
@@ -31,6 +31,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ConnectiveStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategy;
@@ -145,6 +146,7 @@ abstract class GraphSONModule extends TinkerPopJacksonModule {
Order.values(),
Pop.values(),
SackFunctions.Barrier.values(),
+ TraversalOptionParent.Pick.values(),
Scope.values(),
T.values()).flatMap(Stream::of).forEach(e -> put(e.getClass(), e.getDeclaringClass().getSimpleName()));
Arrays.asList(
@@ -215,6 +217,7 @@ abstract class GraphSONModule extends TinkerPopJacksonModule {
Pop.class,
SackFunctions.Barrier.class,
Scope.class,
+ TraversalOptionParent.Pick.class,
T.class).forEach(e -> addSerializer(e, new GraphSONTraversalSerializersV2d0.EnumJacksonSerializer()));
addSerializer(P.class, new GraphSONTraversalSerializersV2d0.PJacksonSerializer());
addSerializer(Lambda.class, new GraphSONTraversalSerializersV2d0.LambdaJacksonSerializer());
@@ -245,6 +248,7 @@ abstract class GraphSONModule extends TinkerPopJacksonModule {
Pop.values(),
SackFunctions.Barrier.values(),
Scope.values(),
+ TraversalOptionParent.Pick.values(),
T.values()).flatMap(Stream::of).forEach(e -> addDeserializer(e.getClass(), new GraphSONTraversalSerializersV2d0.EnumJacksonDeserializer(e.getDeclaringClass())));
addDeserializer(P.class, new GraphSONTraversalSerializersV2d0.PJacksonDeserializer());
addDeserializer(Lambda.class, new GraphSONTraversalSerializersV2d0.LambdaJacksonDeserializer());
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 0a7d277..fef7288 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Path;
import org.apache.tinkerpop.gremlin.process.traversal.Pop;
import org.apache.tinkerpop.gremlin.process.traversal.SackFunctions;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.FoldStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupCountStep;
@@ -349,7 +350,8 @@ public final class GryoMapper implements Mapper<Kryo> {
add(GryoTypeReg.of(Column.class, 132));
add(GryoTypeReg.of(Pop.class, 133));
add(GryoTypeReg.of(SackFunctions.Barrier.class, 135));
- add(GryoTypeReg.of(HashSetSupplier.class, 136, new UtilSerializers.HashSetSupplierSerializer())); // ***LAST ID***
+ add(GryoTypeReg.of(TraversalOptionParent.Pick.class, 137)); // ***LAST ID***
+ add(GryoTypeReg.of(HashSetSupplier.class, 136, new UtilSerializers.HashSetSupplierSerializer()));
add(GryoTypeReg.of(TraverserSet.class, 58));
add(GryoTypeReg.of(Tree.class, 61));
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/CoreImports.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/CoreImports.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/CoreImports.java
index 1c76f0e..e6c64fd 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/CoreImports.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/CoreImports.java
@@ -44,19 +44,20 @@ import org.apache.tinkerpop.gremlin.process.traversal.Translator;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ConnectiveStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.LazyBarrierStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.finalization.MatchAlgorithmStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.finalization.ProfileStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.AdjacentToIncidentStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.FilterRankingStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.IdentityRemovalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.IncidentToAdjacentStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.LazyBarrierStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.MatchPredicateStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.OrderLimitStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.PathProcessorStrategy;
@@ -123,6 +124,8 @@ public final class CoreImports {
CLASS_IMPORTS.add(Pop.class);
CLASS_IMPORTS.add(Scope.class);
CLASS_IMPORTS.add(T.class);
+ CLASS_IMPORTS.add(TraversalOptionParent.class);
+ CLASS_IMPORTS.add(TraversalOptionParent.Pick.class);
CLASS_IMPORTS.add(P.class);
// remote
CLASS_IMPORTS.add(RemoteConnection.class);
@@ -203,6 +206,7 @@ public final class CoreImports {
Collections.addAll(ENUM_IMPORTS, Pop.values());
Collections.addAll(ENUM_IMPORTS, Scope.values());
Collections.addAll(ENUM_IMPORTS, T.values());
+ Collections.addAll(ENUM_IMPORTS, TraversalOptionParent.Pick.values());
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyBranchTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyBranchTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyBranchTest.groovy
index fe13e1d..e602f4e 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyBranchTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyBranchTest.groovy
@@ -35,7 +35,7 @@ public abstract class GroovyBranchTest {
}
@Override
- public Traversal<Vertex, Object> get_g_V_branchXlabelX_optionXperson__ageX_optionXsoftware__langX_optionXsoftware__nameX() {
+ public Traversal<Vertex, Object> get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX() {
new ScriptTraversal<>(g, "gremlin-groovy", """
g.V.branch{it.label == 'person' ? 'a' : 'b'}
.option('a', __.age)
@@ -43,5 +43,16 @@ public abstract class GroovyBranchTest {
.option('b', __.name)
""")
}
+
+ @Override
+ public Traversal<Vertex, Object> get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX_optionXany__labelX() {
+ new ScriptTraversal<>(g, "gremlin-groovy", """
+ g.V.branch(label().is("person").count)
+ .option(1L,__.age)
+ .option(0L,__.lang)
+ .option(0L,__.name)
+ .option(any,label())
+ """)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyChooseTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyChooseTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyChooseTest.groovy
index b92ab9b..99bb31b 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyChooseTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyChooseTest.groovy
@@ -43,5 +43,15 @@ public abstract class GroovyChooseTest {
public Traversal<Vertex, String> get_g_V_chooseXhasLabelXpersonX_and_outXcreatedX__outXknowsX__identityX_name() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.choose(hasLabel('person').and().out('created'), out('knows'), identity()).name")
}
+
+ @Override
+ public Traversal<Vertex, String> get_g_V_chooseXlabelX_optionXblah__outXknowsXX_optionXbleep__outXcreatedXX_optionXnone__identityX_name() {
+ new ScriptTraversal<>(g, "gremlin-groovy", """
+ g.V.choose(label())
+ .option("blah", out("knows"))
+ .option("bleep", out("created"))
+ .option(none, identity()).name
+ """)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
index 02e49b1..d102037 100644
--- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Translator;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.TraversalStrategyProxy;
import org.apache.tinkerpop.gremlin.process.traversal.util.ConnectiveP;
import org.apache.tinkerpop.gremlin.process.traversal.util.OrP;
@@ -151,6 +152,8 @@ public final class GroovyTranslator implements Translator.ScriptTranslator {
return "SackFunctions.Barrier." + object.toString();
else if (object instanceof VertexProperty.Cardinality)
return "VertexProperty.Cardinality." + object.toString();
+ else if (object instanceof TraversalOptionParent.Pick)
+ return "TraversalOptionParent.Pick." + object.toString();
else if (object instanceof Enum)
return ((Enum) object).getDeclaringClass().getSimpleName() + "." + object.toString();
else if (object instanceof Element)
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java b/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java
index bfabbb8..0739c92 100644
--- a/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java
+++ b/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.TraversalStrategyProxy;
import org.apache.tinkerpop.gremlin.process.traversal.util.ConnectiveP;
import org.apache.tinkerpop.gremlin.process.traversal.util.OrP;
@@ -183,6 +184,8 @@ public class PythonTranslator implements Translator.ScriptTranslator {
return "Cardinality." + SymbolHelper.toPython(object.toString());
else if (object instanceof SackFunctions.Barrier)
return "Barrier." + SymbolHelper.toPython(object.toString());
+ else if (object instanceof TraversalOptionParent.Pick)
+ return "Pick." + SymbolHelper.toPython(object.toString());
else if (object instanceof Enum)
return convertStatic(((Enum) object).getDeclaringClass().getSimpleName() + ".") + SymbolHelper.toPython(object.toString());
else if (object instanceof P)
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
index a56dd74..d30db35 100644
--- a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
+++ b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
@@ -118,6 +118,10 @@ statics.add_static('keyDecr', Order.keyDecr)
statics.add_static('valueDecr', Order.valueDecr)
statics.add_static('shuffle', Order.shuffle)
+Pick = Enum('Pick', 'any none')
+statics.add_static('any', Pick.any)
+statics.add_static('none', Pick.none)
+
Pop = Enum('Pop', 'all_ first last')
statics.add_static('first', Pop.first)
statics.add_static('last', Pop.last)
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/BranchTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/BranchTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/BranchTest.java
index 049358f..44a63bd 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/BranchTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/BranchTest.java
@@ -22,7 +22,6 @@ import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -32,6 +31,7 @@ import java.util.Arrays;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.label;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.values;
+import static org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent.Pick.any;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -41,7 +41,9 @@ public abstract class BranchTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Object> get_g_V_branchXlabel_eq_person__a_bX_optionXa__ageX_optionXb__langX_optionXb__nameX();
- public abstract Traversal<Vertex, Object> get_g_V_branchXlabelX_optionXperson__ageX_optionXsoftware__langX_optionXsoftware__nameX();
+ public abstract Traversal<Vertex, Object> get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX();
+
+ public abstract Traversal<Vertex, Object> get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX_optionXany__labelX();
@Test
@LoadGraphWith(MODERN)
@@ -53,12 +55,20 @@ public abstract class BranchTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- public void g_V_branchXlabelX_optionXperson__ageX_optionXsoftware__langX_optionXsoftware__nameX() {
- final Traversal<Vertex, Object> traversal = get_g_V_branchXlabelX_optionXperson__ageX_optionXsoftware__langX_optionXsoftware__nameX();
+ public void g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX() {
+ final Traversal<Vertex, Object> traversal = get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX();
printTraversalForm(traversal);
checkResults(Arrays.asList("java", "java", "lop", "ripple", 29, 27, 32, 35), traversal);
}
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX_optionXany__labelX() {
+ final Traversal<Vertex, Object> traversal = get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX_optionXany__labelX();
+ printTraversalForm(traversal);
+ checkResults(Arrays.asList("java", "java", "lop", "ripple", 29, 27, 32, 35, "person", "person", "person", "person", "software", "software"), traversal);
+ }
+
public static class Traversals extends BranchTest {
@Override
@@ -70,11 +80,20 @@ public abstract class BranchTest extends AbstractGremlinProcessTest {
}
@Override
- public Traversal<Vertex, Object> get_g_V_branchXlabelX_optionXperson__ageX_optionXsoftware__langX_optionXsoftware__nameX() {
+ public Traversal<Vertex, Object> get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX() {
return g.V().branch(label().is("person").count())
.option(1L, values("age"))
.option(0L, values("lang"))
.option(0L, values("name"));
}
+
+ @Override
+ public Traversal<Vertex, Object> get_g_V_branchXlabel_isXpersonX_countX_optionX1__ageX_optionX0__langX_optionX0__nameX_optionXany__labelX() {
+ return g.V().branch(label().is("person").count())
+ .option(1L, values("age"))
+ .option(0L, values("lang"))
+ .option(0L, values("name"))
+ .option(any, label());
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0fec46db/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/ChooseTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/ChooseTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/ChooseTest.java
index bf050e4..ce12daf 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/ChooseTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/branch/ChooseTest.java
@@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.MapHelper;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
@@ -35,6 +36,7 @@ import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.hasLabel;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.identity;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.in;
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.label;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.out;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.valueMap;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.values;
@@ -54,6 +56,7 @@ public abstract class ChooseTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, String> get_g_V_chooseXhasLabelXpersonX_and_outXcreatedX__outXknowsX__identityX_name();
+ public abstract Traversal<Vertex, String> get_g_V_chooseXlabelX_optionXblah__outXknowsXX_optionXbleep__outXcreatedXX_optionXnone__identityX_name();
@Test
@LoadGraphWith(MODERN)
@@ -89,6 +92,14 @@ public abstract class ChooseTest extends AbstractGremlinProcessTest {
checkResults(Arrays.asList("lop", "ripple", "josh", "vadas", "vadas"), traversal);
}
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_chooseXlabelX_optionXblah__outXknowsXX_optionXbleep__outXcreatedXX_optionXnone__identityX_name() {
+ final Traversal<Vertex, String> traversal = get_g_V_chooseXlabelX_optionXblah__outXknowsXX_optionXbleep__outXcreatedXX_optionXnone__identityX_name();
+ printTraversalForm(traversal);
+ checkResults(Arrays.asList("marko", "vadas", "peter", "josh", "lop", "ripple"), traversal);
+ }
+
public static class Traversals extends ChooseTest {
@Override
@@ -107,5 +118,13 @@ public abstract class ChooseTest extends AbstractGremlinProcessTest {
public Traversal<Vertex, String> get_g_V_chooseXhasLabelXpersonX_and_outXcreatedX__outXknowsX__identityX_name() {
return g.V().choose(hasLabel("person").and().out("created"), out("knows"), identity()).values("name");
}
+
+ @Override
+ public Traversal<Vertex, String> get_g_V_chooseXlabelX_optionXblah__outXknowsXX_optionXbleep__outXcreatedXX_optionXnone__identityX_name() {
+ return g.V().choose(label())
+ .option("blah", out("knows"))
+ .option("bleep", out("created"))
+ .option(TraversalOptionParent.Pick.none, identity()).values("name");
+ }
}
}
\ No newline at end of file
[08/29] tinkerpop git commit: Add int feature requirement on
generator tests.
Posted by ok...@apache.org.
Add int feature requirement on generator tests.
These were requiring graphs to support integer and they were not marked as such. While most graphs support int, the Bolt protocol on neo4j does not so it caused some trouble over there. CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/5cefff0f
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/5cefff0f
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/5cefff0f
Branch: refs/heads/TINKERPOP-1389
Commit: 5cefff0f6365de1d31212e8337a27803ae14ad2c
Parents: 157dc3f
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Oct 25 11:31:46 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Oct 25 11:33:49 2016 -0400
----------------------------------------------------------------------
.../gremlin/algorithm/generator/CommunityGeneratorTest.java | 4 ++++
.../gremlin/algorithm/generator/DistributionGeneratorTest.java | 4 ++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5cefff0f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/CommunityGeneratorTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/CommunityGeneratorTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/CommunityGeneratorTest.java
index a7d3414..45f9b40 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/CommunityGeneratorTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/CommunityGeneratorTest.java
@@ -20,6 +20,7 @@ package org.apache.tinkerpop.gremlin.algorithm.generator;
import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -84,6 +85,7 @@ public class CommunityGeneratorTest {
@Test
@FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
public void shouldGenerateDifferentGraph() throws Exception {
final Configuration configuration = graphProvider.newGraphConfiguration("g1", this.getClass(), name.getMethodName(), null);
final Graph graph1 = graphProvider.openTestGraph(configuration);
@@ -113,6 +115,7 @@ public class CommunityGeneratorTest {
@Test
@FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
public void shouldGenerateSameGraph() throws Exception {
final Configuration configuration = graphProvider.newGraphConfiguration("g1", this.getClass(), name.getMethodName(), null);
final Graph graph1 = graphProvider.openTestGraph(configuration);
@@ -194,6 +197,7 @@ public class CommunityGeneratorTest {
@Test
@FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
public void shouldProcessVerticesEdges() {
final Distribution dist = new NormalDistribution(2);
final CommunityGenerator generator = CommunityGenerator.build(graph)
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5cefff0f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/DistributionGeneratorTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/DistributionGeneratorTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/DistributionGeneratorTest.java
index 9dc33af..7d668ae 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/DistributionGeneratorTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/algorithm/generator/DistributionGeneratorTest.java
@@ -20,6 +20,7 @@ package org.apache.tinkerpop.gremlin.algorithm.generator;
import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -67,6 +68,7 @@ public class DistributionGeneratorTest {
@Test
@FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
public void shouldGenerateDifferentGraph() throws Exception {
int executions = 0;
boolean same = true;
@@ -110,6 +112,7 @@ public class DistributionGeneratorTest {
@Test
@FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
public void shouldGenerateSameGraph() throws Exception {
final Configuration configuration = graphProvider.newGraphConfiguration("g1", this.getClass(), name.getMethodName(), null);
final Graph g1 = graphProvider.openTestGraph(configuration);
@@ -158,6 +161,7 @@ public class DistributionGeneratorTest {
@Test
@FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
public void shouldProcessEdges() {
final Distribution dist = new NormalDistribution(2);
final DistributionGenerator generator = DistributionGenerator.build(graph)
[20/29] tinkerpop git commit: GryoSerializer uses HadoopPools so that
gryo pools are not constantly produced (object reused stylie). This have
increased the performance of GryoSerializer based jobs to that of the 3.2.x
line prior to bumping to Spark 2.0.
Posted by ok...@apache.org.
GryoSerializer uses HadoopPools so that gryo pools are not constantly produced (object reused stylie). This have increased the performance of GryoSerializer based jobs to that of the 3.2.x line prior to bumping to Spark 2.0.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/38b4cd18
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/38b4cd18
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/38b4cd18
Branch: refs/heads/TINKERPOP-1389
Commit: 38b4cd18bcb6d45fb75b712bc4b280169404b856
Parents: db3dfcb
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Sep 12 12:22:05 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:13:53 2016 -0600
----------------------------------------------------------------------
.../hadoop/structure/io/HadoopPools.java | 5 +++
.../spark/structure/io/gryo/GryoSerializer.java | 40 +++++---------------
2 files changed, 14 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/38b4cd18/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
index 5074ad5..392e97d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
@@ -52,6 +52,11 @@ public final class HadoopPools {
HadoopPools.initialize(ConfUtil.makeApacheConfiguration(configuration));
}
+ public synchronized static void initialize(final GryoPool gryoPool) {
+ GRYO_POOL = gryoPool;
+ INITIALIZED = true;
+ }
+
public static GryoPool getGryoPool() {
if (!INITIALIZED) {
HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using the default pool"); // TODO: this is necessary because we can't get the pool intialized in the Merger code of the Hadoop process.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/38b4cd18/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 6735fe5..00cb702 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -33,6 +33,7 @@ import org.apache.spark.serializer.SerializerInstance;
import org.apache.spark.storage.BlockManagerId;
import org.apache.spark.util.SerializableConfiguration;
import org.apache.spark.util.collection.CompactBuffer;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
@@ -49,9 +50,7 @@ import scala.collection.mutable.WrappedArray;
import scala.runtime.BoxedUnit;
import java.io.Serializable;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.List;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -61,14 +60,9 @@ public final class GryoSerializer extends Serializer implements Serializable {
//private final Option<String> userRegistrator;
private final int bufferSize;
private final int maxBufferSize;
- private final int poolSize;
- private final ArrayList<String> ioRegList = new ArrayList<>();
private final boolean referenceTracking;
private final boolean registrationRequired;
-
- private transient GryoPool gryoPool;
-
public GryoSerializer(final SparkConf sparkConfiguration) {
final long bufferSizeKb = sparkConfiguration.getSizeAsKb("spark.kryoserializer.buffer", "64k");
final long maxBufferSizeMb = sparkConfiguration.getSizeAsMb("spark.kryoserializer.buffer.max", "64m");
@@ -85,19 +79,10 @@ public final class GryoSerializer extends Serializer implements Serializable {
//this.userRegistrator = sparkConfiguration.getOption("spark.kryo.registrator");
}
}
- poolSize = sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
- List<Object> list = makeApacheConfiguration(sparkConfiguration).getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList());
- list.forEach(c -> {
- ioRegList.add(c.toString());
- }
- );
- }
-
- private GryoPool createPool(){
- List<Object> list = new ArrayList<>(ioRegList);
- return GryoPool.build().
- poolSize(poolSize).
- ioRegistries(list).
+ // create a GryoPool and store it in static HadoopPools
+ HadoopPools.initialize(GryoPool.build().
+ poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT)).
+ ioRegistries(makeApacheConfiguration(sparkConfiguration).getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).
initializeMapper(builder -> {
try {
builder.addCustom(Tuple2.class, new Tuple2Serializer())
@@ -122,13 +107,13 @@ public final class GryoSerializer extends Serializer implements Serializable {
.addCustom(SerializableConfiguration.class, new JavaSerializer())
.addCustom(VertexWritable.class, new VertexWritableSerializer())
.addCustom(ObjectWritable.class, new ObjectWritableSerializer())
- .referenceTracking(referenceTracking)
- .registrationRequired(registrationRequired);
+ .referenceTracking(this.referenceTracking)
+ .registrationRequired(this.registrationRequired);
// add these as we find ClassNotFoundExceptions
} catch (final ClassNotFoundException e) {
throw new IllegalStateException(e);
}
- }).create();
+ }).create());
}
public Output newOutput() {
@@ -136,14 +121,7 @@ public final class GryoSerializer extends Serializer implements Serializable {
}
public GryoPool getGryoPool() {
- if (gryoPool == null) {
- synchronized (this) {
- if (gryoPool == null) {
- gryoPool = createPool();
- }
- }
- }
- return this.gryoPool;
+ return HadoopPools.getGryoPool();
}
@Override
[28/29] tinkerpop git commit: minor nothing to GiraphGraphComputer.
Posted by ok...@apache.org.
minor nothing to GiraphGraphComputer.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/78936b17
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/78936b17
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/78936b17
Branch: refs/heads/TINKERPOP-1389
Commit: 78936b173780740e19cf07323e9a9f93ed06b53f
Parents: d26ec21
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Oct 25 10:33:18 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:16:26 2016 -0600
----------------------------------------------------------------------
.../gremlin/giraph/process/computer/GiraphGraphComputer.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/78936b17/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index 6ffd5ea..e21cd3b 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -84,7 +84,6 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
super(hadoopGraph);
- System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, HadoopPoolShimService.class.getCanonicalName()); // HadoopPools only with Giraph
final Configuration configuration = hadoopGraph.configuration();
configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
@@ -97,6 +96,9 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
+ if (null == this.giraphConfiguration.get(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null))
+ this.giraphConfiguration.set(KryoShimServiceLoader.KRYO_SHIM_SERVICE, HadoopPoolShimService.class.getCanonicalName());
+ System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, this.giraphConfiguration.get(KryoShimServiceLoader.KRYO_SHIM_SERVICE));
this.useWorkerThreadsInConfiguration = this.giraphConfiguration.getInt(GiraphConstants.MAX_WORKERS, -666) != -666 || this.giraphConfiguration.getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), -666) != -666;
}
[05/29] tinkerpop git commit: Merge branch 'tp31' into tp32
Posted by ok...@apache.org.
Merge branch 'tp31' into tp32
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/487b3bf1
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/487b3bf1
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/487b3bf1
Branch: refs/heads/TINKERPOP-1389
Commit: 487b3bf1f5c6d75e7ae5d2946d001a19a0f5a593
Parents: b262c7e 157dc3f
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Oct 25 14:52:35 2016 +0200
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Oct 25 14:52:35 2016 +0200
----------------------------------------------------------------------
bin/validate-distribution.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[25/29] tinkerpop git commit: UnshadedKryoShimService is now the
default for SparkGraphComputer. Users no longer have to System.setProperty()
as its all handled in the SparkGraphComputer constructor. I would really like
to get rid of the 'service model'
Posted by ok...@apache.org.
UnshadedKryoShimService is now the default for SparkGraphComputer. Users no longer have to System.setProperty() as its all handled in the SparkGraphComputer constructor. I would really like to get rid of the 'service model' (cc @dalaro) as UnshadedKryoShimService only works with Spark and thus, we can't have it leak over to Giraph (and other Hadoop-based graph computeters). Its leak-free right now, but in general, it would be good to not even have this as a priority/service thing. Trying to think how to do this....
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/d26ec21f
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/d26ec21f
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/d26ec21f
Branch: refs/heads/TINKERPOP-1389
Commit: d26ec21f639ab464aa4b5433bc6779a48f80cdf5
Parents: c3d1c44
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Oct 25 08:02:07 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:16:26 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 +
docs/src/upgrade/release-3.3.x.asciidoc | 21 ++++++++
.../process/computer/GiraphGraphComputer.java | 3 ++
hadoop-gremlin/conf/hadoop-graphson.properties | 2 +
.../conf/hadoop-grateful-gryo.properties | 25 +++++-----
hadoop-gremlin/conf/hadoop-gryo.properties | 6 ++-
hadoop-gremlin/conf/hadoop-script.properties | 5 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 2 +
.../process/computer/SparkGraphComputer.java | 11 +++++
.../unshaded/UnshadedKryoShimService.java | 22 +++++----
...tratorGraphComputerProcessIntegrateTest.java | 33 -------------
...alizerGraphComputerProcessIntegrateTest.java | 33 +++++++++++++
...SparkHadoopGraphGryoRegistratorProvider.java | 52 --------------------
.../SparkHadoopGraphGryoSerializerProvider.java | 45 +++++++++++++++++
.../computer/SparkHadoopGraphProvider.java | 17 +++++--
15 files changed, 163 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index a70cd3a..17fd5bd 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,8 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
TinkerPop 3.3.0 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Bumped to support Spark 2.0.0.
+* Added `UnshadedKryoShimService` as the new default serializer model for `SparkGraphComputer`.
* Replaced term `REST` with `HTTP` to remove any confusion as to the design of the API.
* Moved `gremlin-benchmark` under `gremlin-tools` module.
* Added `gremlin-tools` and its submodule `gremlin-coverage`.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/docs/src/upgrade/release-3.3.x.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.3.x.asciidoc b/docs/src/upgrade/release-3.3.x.asciidoc
index a026771..d1a19d9 100644
--- a/docs/src/upgrade/release-3.3.x.asciidoc
+++ b/docs/src/upgrade/release-3.3.x.asciidoc
@@ -31,3 +31,24 @@ Please see the link:https://github.com/apache/tinkerpop/blob/3.3.3/CHANGELOG.asc
Upgrading for Users
~~~~~~~~~~~~~~~~~~~
+
+SparkGraphComputer GryoRegistrator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Historically, `SparkGraphComputer` has used `GryoSerializer` to handle the serialization of objects in Spark. The reason
+this exists is because TinkerPop uses a shaded version of Kryo and thus, couldn't use the standard `KryoSerializer`-model
+provided by Spark. However, a "shim model" was created which allows for the shaded and unshaded versions of Kryo to
+interact with one another. To this end, `KryoSerializer` can now be used with a `GryoRegistrator`. The properties file
+for a `SparkGraphComputer` now looks as follows:
+
+```
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
+```
+
+If the old `GryoSerializer` model is desired, then the properties file should simply look as before:
+
+```
+spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
+```
+
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index b06b40a..6ffd5ea 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -44,6 +44,7 @@ import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorage;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.GraphFilterAware;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.InputOutputHelper;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
@@ -57,6 +58,7 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.util.Gremlin;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
@@ -82,6 +84,7 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
super(hadoopGraph);
+ System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, HadoopPoolShimService.class.getCanonicalName()); // HadoopPools only with Giraph
final Configuration configuration = hadoopGraph.configuration();
configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/hadoop-gremlin/conf/hadoop-graphson.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-graphson.properties b/hadoop-gremlin/conf/hadoop-graphson.properties
index 10025de..c37cf28 100644
--- a/hadoop-gremlin/conf/hadoop-graphson.properties
+++ b/hadoop-gremlin/conf/hadoop-graphson.properties
@@ -30,6 +30,8 @@ gremlin.hadoop.jarsInDistributedCache=true
# SparkGraphComputer Configuration #
####################################
spark.master=local[4]
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
#####################################
# GiraphGraphComputer Configuration #
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-grateful-gryo.properties b/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
index 17aeadd..92ed942 100644
--- a/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
+++ b/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
@@ -15,9 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-#
-# Hadoop Graph Configuration
-#
gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
gremlin.hadoop.graphReader=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
gremlin.hadoop.graphWriter=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
@@ -25,9 +22,17 @@ gremlin.hadoop.inputLocation=grateful-dead.kryo
gremlin.hadoop.outputLocation=output
gremlin.hadoop.jarsInDistributedCache=true
-#
-# GiraphGraphComputer Configuration
-#
+####################################
+# SparkGraphComputer Configuration #
+####################################
+spark.master=local[1]
+spark.executor.memory=1g
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
+
+#####################################
+# GiraphGraphComputer Configuration #
+#####################################
giraph.minWorkers=1
giraph.maxWorkers=1
giraph.useOutOfCoreGraph=true
@@ -37,11 +42,3 @@ mapred.reduce.child.java.opts=-Xmx1024m
giraph.numInputThreads=4
giraph.numComputeThreads=4
giraph.maxMessagesInMemory=100000
-
-#
-# SparkGraphComputer Configuration
-#
-spark.master=local[1]
-spark.executor.memory=1g
-spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
-
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/hadoop-gremlin/conf/hadoop-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-gryo.properties b/hadoop-gremlin/conf/hadoop-gryo.properties
index aaab24d..c156a98 100644
--- a/hadoop-gremlin/conf/hadoop-gryo.properties
+++ b/hadoop-gremlin/conf/hadoop-gryo.properties
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
gremlin.hadoop.graphReader=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
gremlin.hadoop.graphWriter=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
@@ -28,7 +29,9 @@ gremlin.hadoop.outputLocation=output
####################################
spark.master=local[4]
spark.executor.memory=1g
-spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
+# spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer (3.2.x model)
# gremlin.spark.graphStorageLevel=MEMORY_AND_DISK
# gremlin.spark.persistContext=true
# gremlin.spark.graphWriter=org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD
@@ -39,7 +42,6 @@ spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerial
# spark.eventLog.dir=/tmp/spark-event-logs
# spark.ui.killEnabled=true
-
#####################################
# GiraphGraphComputer Configuration #
#####################################
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/hadoop-gremlin/conf/hadoop-script.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-script.properties b/hadoop-gremlin/conf/hadoop-script.properties
index 0c27dd7..88bf6d9 100644
--- a/hadoop-gremlin/conf/hadoop-script.properties
+++ b/hadoop-gremlin/conf/hadoop-script.properties
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
gremlin.hadoop.graphReader=org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptInputFormat
gremlin.hadoop.graphWriter=org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat
@@ -28,7 +29,9 @@ gremlin.hadoop.outputLocation=output
####################################
spark.master=local[4]
spark.executor.memory=1g
-spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
+# spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer (3.2.x model)
# spark.kryo.registrationRequired=true
# spark.storage.memoryFraction=0.2
# spark.eventLog.enabled=true
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index 9c79b53..3ff8e2a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -61,6 +61,8 @@ public final class Constants {
public static final String GREMLIN_SPARK_SKIP_PARTITIONER = "gremlin.spark.skipPartitioner"; // don't partition the loadedGraphRDD
public static final String GREMLIN_SPARK_SKIP_GRAPH_CACHE = "gremlin.spark.skipGraphCache"; // don't cache the loadedGraphRDD (ignores graphStorageLevel)
public static final String SPARK_SERIALIZER = "spark.serializer";
+ public static final String SPARK_KRYO_REGISTRATOR = "spark.kryo.registrator";
+ public static final String SPARK_KRYO_REGISTRATION_REQUIRED = "spark.kryo.registrationRequired";
public static String getGraphLocation(final String location) {
return location.endsWith("/") ? location + Constants.HIDDEN_G : location + "/" + Constants.HIDDEN_G;
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 36618a1..c7d0cfb 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -35,6 +35,7 @@ import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.launcher.SparkLauncher;
+import org.apache.spark.serializer.KryoSerializer;
import org.apache.spark.storage.StorageLevel;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
@@ -43,6 +44,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorage;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.GraphFilterAware;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
@@ -67,7 +69,9 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.OutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
@@ -106,6 +110,13 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
super(hadoopGraph);
this.sparkConfiguration = new HadoopConfiguration();
ConfigurationUtils.copy(this.hadoopGraph.configuration(), this.sparkConfiguration);
+ if (KryoSerializer.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_SERIALIZER, null)) &&
+ GryoRegistrator.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_KRYO_REGISTRATOR, null))) {
+ System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, UnshadedKryoShimService.class.getCanonicalName());
+ } else if (GryoSerializer.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_SERIALIZER, null)) &&
+ !this.sparkConfiguration.containsKey(Constants.SPARK_KRYO_REGISTRATOR)) {
+ System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, HadoopPoolShimService.class.getCanonicalName());
+ }
if (null != System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)) {
final String shimService = System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE);
this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
index 41e0001..0789d6a 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
@@ -30,6 +30,7 @@ import com.esotericsoftware.kryo.io.Output;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.SparkConf;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.IoRegistryAwareKryoSerializer;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService;
@@ -48,7 +49,8 @@ public class UnshadedKryoShimService implements KryoShimService {
private static volatile boolean initialized;
- public UnshadedKryoShimService() { }
+ public UnshadedKryoShimService() {
+ }
@Override
public Object readClassAndObject(final InputStream source) {
@@ -123,23 +125,23 @@ public class UnshadedKryoShimService implements KryoShimService {
sparkConf.set(GryoPool.CONFIG_IO_REGISTRY, regStr);
}
// Setting spark.serializer here almost certainly isn't necessary, but it doesn't hurt
- sparkConf.set("spark.serializer", IoRegistryAwareKryoSerializer.class.getCanonicalName());
+ sparkConf.set(Constants.SPARK_SERIALIZER, IoRegistryAwareKryoSerializer.class.getCanonicalName());
- final String registrator = conf.getString("spark.kryo.registrator");
+ final String registrator = conf.getString(Constants.SPARK_KRYO_REGISTRATOR);
if (null != registrator) {
- sparkConf.set("spark.kryo.registrator", registrator);
- log.info("Copied spark.kryo.registrator: {}", registrator);
+ sparkConf.set(Constants.SPARK_KRYO_REGISTRATOR, registrator);
+ log.info("Copied " + Constants.SPARK_KRYO_REGISTRATOR + ": {}", registrator);
} else {
- log.info("Not copying spark.kryo.registrator");
+ log.info("Not copying " + Constants.SPARK_KRYO_REGISTRATOR);
}
- // Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf)
- final int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
- GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
// Instantiate the spark.serializer
final IoRegistryAwareKryoSerializer ioReg = new IoRegistryAwareKryoSerializer(sparkConf);
- // Setup a pool backed by our spark.serializer instance
+ // Setup a pool backed by our spark.serializer instance
+ // Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf)
+ final int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
+ GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
for (int i = 0; i < poolSize; i++) {
KRYOS.add(ioReg.newKryo());
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java
deleted file mode 100644
index 29f627d..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(ProcessComputerSuite.class)
-@GraphProviderClass(provider = SparkHadoopGraphGryoRegistratorProvider.class, graph = HadoopGraph.class)
-public class SparkGryoRegistratorGraphComputerProcessIntegrateTest {
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoSerializerGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoSerializerGraphComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoSerializerGraphComputerProcessIntegrateTest.java
new file mode 100644
index 0000000..b04513c
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoSerializerGraphComputerProcessIntegrateTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(ProcessComputerSuite.class)
+@GraphProviderClass(provider = SparkHadoopGraphGryoSerializerProvider.class, graph = HadoopGraph.class)
+public class SparkGryoSerializerGraphComputerProcessIntegrateTest {
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
deleted file mode 100644
index fcebbd0..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process.computer;
-
-import org.apache.spark.serializer.KryoSerializer;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.spark.structure.Spark;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
-
-import java.util.Map;
-
-import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.KRYO_SHIM_SERVICE;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkHadoopGraphGryoRegistratorProvider extends SparkHadoopGraphProvider {
-
- public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- Spark.close();
- final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- // ensure the context doesn't stay open for the GryoSerializer tests to follow
- // this is primarily to ensure that the KryoShimService loaded specifically in these tests don't leak to the other tests
- config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false);
- config.put("spark.serializer", KryoSerializer.class.getCanonicalName());
- config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName());
- System.setProperty(KRYO_SHIM_SERVICE, UnshadedKryoShimService.class.getCanonicalName());
- KryoShimServiceLoader.load(true);
- System.clearProperty(KRYO_SHIM_SERVICE);
- return config;
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
new file mode 100644
index 0000000..9820b7b
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
+import org.apache.tinkerpop.gremlin.spark.structure.Spark;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
+
+import java.util.Map;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkHadoopGraphGryoSerializerProvider extends SparkHadoopGraphProvider {
+
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ if (this.getClass().equals(SparkHadoopGraphGryoSerializerProvider.class) &&
+ !HadoopPoolShimService.class.getCanonicalName().equals(System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)))
+ Spark.close();
+ final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
+ config.put(Constants.SPARK_SERIALIZER, GryoSerializer.class.getCanonicalName());
+ config.remove(Constants.SPARK_KRYO_REGISTRATOR);
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/d26ec21f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index d4201b5..c5b5083 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
+import org.apache.spark.launcher.SparkLauncher;
+import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.groovy.util.SugarTestHelper;
@@ -39,8 +41,10 @@ import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorageCheck;
import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.util.Map;
@@ -52,6 +56,10 @@ public class SparkHadoopGraphProvider extends HadoopGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ if (this.getClass().equals(SparkHadoopGraphProvider.class) &&
+ !UnshadedKryoShimService.class.getCanonicalName().equals(System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)))
+ Spark.close();
+
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
@@ -82,9 +90,10 @@ public class SparkHadoopGraphProvider extends HadoopGraphProvider {
}
config.put(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
- config.put("spark.master", "local[4]");
- config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
- config.put("spark.kryo.registrationRequired", true);
+ config.put(SparkLauncher.SPARK_MASTER, "local[4]");
+ config.put(Constants.SPARK_SERIALIZER, KryoSerializer.class.getCanonicalName());
+ config.put(Constants.SPARK_KRYO_REGISTRATOR, GryoRegistrator.class.getCanonicalName());
+ config.put(Constants.SPARK_KRYO_REGISTRATION_REQUIRED, true);
return config;
}
[04/29] tinkerpop git commit: CTR: Fixed a minor script bug that
popped up in the latest release process.
Posted by ok...@apache.org.
CTR: Fixed a minor script bug that popped up in the latest release process.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/157dc3f7
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/157dc3f7
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/157dc3f7
Branch: refs/heads/TINKERPOP-1389
Commit: 157dc3f784b3f798947bafaa214b87cc6c8e3ae3
Parents: 52ae785
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Oct 25 14:50:13 2016 +0200
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Oct 25 14:50:13 2016 +0200
----------------------------------------------------------------------
bin/validate-distribution.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/157dc3f7/bin/validate-distribution.sh
----------------------------------------------------------------------
diff --git a/bin/validate-distribution.sh b/bin/validate-distribution.sh
index 4daaf47..2148610 100755
--- a/bin/validate-distribution.sh
+++ b/bin/validate-distribution.sh
@@ -117,11 +117,11 @@ if [ "${TYPE}" = "SOURCE" ]; then
cd ${DIR_NAME}
echo -n "* building project ... "
LOG_FILE="${TMP_DIR}/mvn-clean-install-${VERSION}.log"
-mvn clean install -q 2>&1 > "${LOG_FILE}" || {
+mvn clean install -q > "${LOG_FILE}" 2>&1 || {
echo "failed"
echo
tail -n50 "${LOG_FILE}"
- echo -e "\n\e[1mThe full log file can be inspected under `pwd`/${LOG_FILE}.\e[0m\n"
+ echo -e "\n\e[1mThe full log file can be inspected under ${LOG_FILE}.\e[0m\n"
exit 1
}
echo "OK"
[07/29] tinkerpop git commit: Setup changelog/upgrade docs for 3.1.6
Posted by ok...@apache.org.
Setup changelog/upgrade docs for 3.1.6
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/1f84ad39
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/1f84ad39
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/1f84ad39
Branch: refs/heads/TINKERPOP-1389
Commit: 1f84ad39d5a9804951340f42d9e79d7d92433ce7
Parents: 5cefff0
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Oct 25 11:33:08 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Oct 25 11:33:49 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 5 +++++
docs/src/upgrade/release-3.1.x-incubating.asciidoc | 7 +++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1f84ad39/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 4c0ca63..551a2be 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -22,6 +22,11 @@ TinkerPop 3.1.0 (A 187 On The Undercover Gremlinz)
image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/images/gremlin-gangster.png[width=185]
+[[release-3-1-6]]
+TinkerPop 3.1.6 (Release Date: NOT OFFICIALLY RELEASED YET)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
[[release-3-1-5]]
TinkerPop 3.1.5 (Release Date: October 17, 2016)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1f84ad39/docs/src/upgrade/release-3.1.x-incubating.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.1.x-incubating.asciidoc b/docs/src/upgrade/release-3.1.x-incubating.asciidoc
index 5779dc8..925b21b 100644
--- a/docs/src/upgrade/release-3.1.x-incubating.asciidoc
+++ b/docs/src/upgrade/release-3.1.x-incubating.asciidoc
@@ -22,6 +22,13 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
*A 187 On The Undercover Gremlinz*
+TinkerPop 3.1.6
+---------------
+
+*Release Date: NOT OFFICIALLY RELEASED YET*
+
+Please see the link:https://github.com/apache/tinkerpop/blob/3.1.6/CHANGELOG.asciidoc#release-3-1-6[changelog] for a complete list of all the modifications that are part of this release.
+
TinkerPop 3.1.5
---------------
[29/29] tinkerpop git commit: Avoid starting VP worker iterations
that never end
Posted by ok...@apache.org.
Avoid starting VP worker iterations that never end
SparkExecutor.executeVertexProgramIteration was written in such a way
that an empty RDD partition would cause it to invoke
VertexProgram.workerIterationStart without ever invoking
VertexProgram.workerIterationEnd. This seems like a contract
violation. I have at least one VP that relies on
workerIterationStart|End to allocate and release resources. Failing
to invoke End like this causes a leak in that VP, as it would for any
VP that uses that resource management pattern.
(cherry picked from commit 36e1159a80f539b8bd4a884e5c1cf304ec52c4f9;
this is the same change, except it tracks a switch between Spark 1.6
and 2.0 away from functions that manipulate iterables to those that
manipulate iterators)
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/19e0f2fc
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/19e0f2fc
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/19e0f2fc
Branch: refs/heads/TINKERPOP-1389
Commit: 19e0f2fc1a4662364d06ff8b32095e32cce93bf3
Parents: 78936b1
Author: Dan LaRocque <da...@hopcount.org>
Authored: Tue Oct 25 19:37:17 2016 -0500
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:16:26 2016 -0600
----------------------------------------------------------------------
.../gremlin/spark/process/computer/SparkExecutor.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/19e0f2fc/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index 8d32b36..e9372d0 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -91,9 +91,15 @@ public final class SparkExecutor {
// for each partition of vertices emit a view and their outgoing messages
.mapPartitionsToPair(partitionIterator -> {
KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration);
+
+ // if the partition is empty, return without starting a new VP iteration
+ if (!partitionIterator.hasNext())
+ return Collections.emptyIterator();
+
final VertexProgram<M> workerVertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(graphComputerConfiguration), vertexProgramConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
final SparkMessenger<M> messenger = new SparkMessenger<>();
+
workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
return IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
final StarGraph.StarVertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
[14/29] tinkerpop git commit: Merge branch 'tp31' into tp32
Posted by ok...@apache.org.
Merge branch 'tp31' into tp32
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/16c7c244
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/16c7c244
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/16c7c244
Branch: refs/heads/TINKERPOP-1389
Commit: 16c7c2449d45a6090e036a2b62244db75244adae
Parents: 2d9bb20 781ff3a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Oct 26 09:33:49 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Oct 26 09:33:49 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../tinkerpop/gremlin/FeatureRequirementSet.java | 2 +-
.../apache/tinkerpop/gremlin/structure/EdgeTest.java | 13 ++++---------
3 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/16c7c244/CHANGELOG.asciidoc
----------------------------------------------------------------------
[18/29] tinkerpop git commit: jesus. that was a depedency hell like
only spark can provide. got SparkServer working with GremlinConsole and HDFS.
Phew... Also, bumped to Hadoop 2.7.3 (from 2.7.2).
Posted by ok...@apache.org.
jesus. that was a depedency hell like only spark can provide. got SparkServer working with GremlinConsole and HDFS. Phew... Also, bumped to Hadoop 2.7.3 (from 2.7.2).
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/5983a5e4
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/5983a5e4
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/5983a5e4
Branch: refs/heads/TINKERPOP-1389
Commit: 5983a5e49d70c1a12cf21bcd96c7ef7f6a1951f0
Parents: 38b4cd1
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Sep 12 18:04:30 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:13:53 2016 -0600
----------------------------------------------------------------------
hadoop-gremlin/pom.xml | 16 ++++++++++------
pom.xml | 2 +-
spark-gremlin/pom.xml | 43 ++++++++++++++++++++++++++++++++++++++++---
3 files changed, 51 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5983a5e4/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 46d318b..b573852 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -27,12 +27,16 @@ limitations under the License.
<name>Apache TinkerPop :: Hadoop Gremlin</name>
<dependencyManagement>
<dependencies>
- <dependency>
- <!-- see: https://github.com/apache/hadoop/pull/84 -->
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.7.1.Final</version>
- </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.0.29.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.8.0.Final</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<dependencies>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5983a5e4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c2cc89d..a6b2a92 100644
--- a/pom.xml
+++ b/pom.xml
@@ -141,7 +141,7 @@ limitations under the License.
<skipIntegrationTests>true</skipIntegrationTests>
<skipPerformanceTests>true</skipPerformanceTests>
<slf4j.version>1.7.21</slf4j.version>
- <hadoop.version>2.7.2</hadoop.version>
+ <hadoop.version>2.7.3</hadoop.version>
<java.tuples.version>1.2</java.tuples.version>
<javadoc-plugin.version>2.10.1</javadoc-plugin.version>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/5983a5e4/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 625579e..02a3a05 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -112,7 +112,7 @@
<!-- SPARK -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
+ <artifactId>spark-core_2.11</artifactId>
<version>2.0.0</version>
<exclusions>
<!-- self conflicts -->
@@ -129,6 +129,10 @@
<artifactId>scala-reflect</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-xml_2.11</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
@@ -208,13 +212,32 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- consistent dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>2.10.5</version>
+ <version>2.11.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-xml_2.11</artifactId>
+ <version>1.0.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -236,6 +259,16 @@
<artifactId>snappy-java</artifactId>
<version>1.1.1.7</version>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.0.29.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.8.0.Final</version>
+ </dependency>
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@@ -247,6 +280,10 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -378,7 +415,7 @@
<configuration>
<archive>
<manifestEntries>
- <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:2.7.2
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:2.7.3
</Gremlin-Plugin-Dependencies>
<!-- deletes the servlet-api jar from the path after install - causes conflicts -->
<Gremlin-Plugin-Paths>servlet-api-2.5.jar=</Gremlin-Plugin-Paths>
[22/29] tinkerpop git commit: if the System.getProperty() has a
shim-service set,
then it is also set as a DRIVER_JAVA_OPTIONS and an EXECUTOR_JAVA_OPTIONS in
SparkGraphComputer (this is necessary for Spark 2.0). Also,
some is.testing class registrations
Posted by ok...@apache.org.
if the System.getProperty() has a shim-service set, then it is also set as a DRIVER_JAVA_OPTIONS and an EXECUTOR_JAVA_OPTIONS in SparkGraphComputer (this is necessary for Spark 2.0). Also, some is.testing class registrations are now also needed outside of testing -- Spark 2.0 anonomaly.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/c3d1c44e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/c3d1c44e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/c3d1c44e
Branch: refs/heads/TINKERPOP-1389
Commit: c3d1c44e326c4f4db763039ff6b1abfc6d5bac6d
Parents: 508e1fd
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Oct 24 15:21:23 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:14:21 2016 -0600
----------------------------------------------------------------------
.../spark/process/computer/SparkGraphComputer.java | 8 ++++++++
.../spark/structure/io/gryo/GryoRegistrator.java | 12 +++++-------
2 files changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c3d1c44e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 40598c0..36618a1 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -70,6 +70,7 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.File;
import java.io.IOException;
@@ -105,6 +106,13 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
super(hadoopGraph);
this.sparkConfiguration = new HadoopConfiguration();
ConfigurationUtils.copy(this.hadoopGraph.configuration(), this.sparkConfiguration);
+ if (null != System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)) {
+ final String shimService = System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE);
+ this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/c3d1c44e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
index ffd731a..9495375 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -178,13 +178,11 @@ public class GryoRegistrator implements KryoRegistrator {
// duplication, but it would be a bit cumbersome to do so without disturbing
// the ordering of the existing entries in that constructor, since not all
// of the entries are for TinkerPop (and the ordering is significant).
- if (Boolean.valueOf(System.getProperty("is.testing", "false"))) {
- try {
- m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
- m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
- } catch (final ClassNotFoundException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
+ try {
+ m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
+ m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
+ } catch (final ClassNotFoundException e) {
+ throw new IllegalStateException(e.getMessage(), e);
}
m.put(WrappedArray.ofRef.class, null);
m.put(MessagePayload.class, null);
[06/29] tinkerpop git commit: Merge branch 'tp32'
Posted by ok...@apache.org.
Merge branch 'tp32'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/2ae615f2
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/2ae615f2
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/2ae615f2
Branch: refs/heads/TINKERPOP-1389
Commit: 2ae615f20839fae82cd9d622fa8e87057353be4c
Parents: 42090f2 487b3bf
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Oct 25 14:53:02 2016 +0200
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Oct 25 14:53:02 2016 +0200
----------------------------------------------------------------------
bin/validate-distribution.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[19/29] tinkerpop git commit: TINKERPOP-1389 Support Spark 2.0
Posted by ok...@apache.org.
TINKERPOP-1389 Support Spark 2.0
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/db3dfcb4
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/db3dfcb4
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/db3dfcb4
Branch: refs/heads/TINKERPOP-1389
Commit: db3dfcb4aff776a8c10db393ad5d1811607eba30
Parents: 1484c8d
Author: yucx <yu...@cn.ibm.com>
Authored: Thu Sep 1 23:11:58 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:13:53 2016 -0600
----------------------------------------------------------------------
giraph-gremlin/pom.xml | 2 +-
hadoop-gremlin/pom.xml | 2 +-
spark-gremlin/pom.xml | 13 +++++-
.../spark/process/computer/SparkExecutor.java | 16 ++++----
.../SparkStarBarrierInterceptor.java | 10 ++---
.../spark/structure/io/gryo/GryoSerializer.java | 43 ++++++++++++++++----
6 files changed, 59 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db3dfcb4/giraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml
index 102f3a9..25d96ec 100644
--- a/giraph-gremlin/pom.xml
+++ b/giraph-gremlin/pom.xml
@@ -127,7 +127,7 @@ limitations under the License.
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <version>3.0.1</version>
+ <version>3.1.0</version>
</dependency>
<!-- TEST -->
<dependency>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db3dfcb4/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 5416f18..46d318b 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -128,7 +128,7 @@ limitations under the License.
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
- <version>3.0.1</version>
+ <version>3.1.0</version>
</dependency>
<!-- TEST -->
<dependency>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db3dfcb4/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index f68d76b..625579e 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -30,6 +30,11 @@
<name>Apache TinkerPop :: Spark Gremlin</name>
<dependencies>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>14.0.1</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>${project.version}</version>
@@ -55,6 +60,10 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
@@ -104,7 +113,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>1.6.1</version>
+ <version>2.0.0</version>
<exclusions>
<!-- self conflicts -->
<exclusion>
@@ -210,7 +219,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.4.4</version>
+ <version>2.6.5</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db3dfcb4/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index 8dd2381..8d32b36 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -18,7 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
-import com.google.common.base.Optional;
+import org.apache.spark.api.java.Optional;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function2;
@@ -65,7 +65,7 @@ public final class SparkExecutor {
public static JavaPairRDD<Object, VertexWritable> applyGraphFilter(final JavaPairRDD<Object, VertexWritable> graphRDD, final GraphFilter graphFilter) {
return graphRDD.mapPartitionsToPair(partitionIterator -> {
final GraphFilter gFilter = graphFilter.clone();
- return () -> IteratorUtils.filter(partitionIterator, tuple -> (tuple._2().get().applyGraphFilter(gFilter)).isPresent());
+ return IteratorUtils.filter(partitionIterator, tuple -> (tuple._2().get().applyGraphFilter(gFilter)).isPresent());
}, true);
}
@@ -95,7 +95,7 @@ public final class SparkExecutor {
final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
final SparkMessenger<M> messenger = new SparkMessenger<>();
workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
- return () -> IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
+ return IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
final StarGraph.StarVertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
final boolean hasViewAndMessages = vertexViewIncoming._2()._2().isPresent(); // if this is the first iteration, then there are no views or messages
final List<DetachedVertexProperty<Object>> previousView = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getView() : memory.isInitialIteration() ? new ArrayList<>() : Collections.emptyList();
@@ -133,7 +133,7 @@ public final class SparkExecutor {
/////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////
final PairFlatMapFunction<Tuple2<Object, ViewOutgoingPayload<M>>, Object, Payload> messageFunction =
- tuple -> () -> IteratorUtils.concat(
+ tuple -> IteratorUtils.concat(
IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())), // emit the view payload
IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))));
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(vertexProgramConfiguration), vertexProgramConfiguration).getMessageCombiner().orElse(null);
@@ -172,7 +172,7 @@ public final class SparkExecutor {
newViewIncomingRDD
.foreachPartition(partitionIterator -> {
KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration);
- }); // need to complete a task so its BSP and the memory for this iteration is updated
+ }); // need to complete a task so its BSP and the memory for this iteration is updated�
return newViewIncomingRDD;
}
@@ -207,7 +207,7 @@ public final class SparkExecutor {
final Configuration graphComputerConfiguration) {
JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> {
KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration);
- return () -> new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator);
+ return new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator);
});
if (mapReduce.getMapKeySort().isPresent())
mapRDD = mapRDD.sortByKey(mapReduce.getMapKeySort().get(), true, 1);
@@ -218,7 +218,7 @@ public final class SparkExecutor {
final Configuration graphComputerConfiguration) {
return mapRDD.mapPartitionsToPair(partitionIterator -> {
KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration);
- return () -> new CombineIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator);
+ return new CombineIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator);
});
}
@@ -227,7 +227,7 @@ public final class SparkExecutor {
final Configuration graphComputerConfiguration) {
JavaPairRDD<OK, OV> reduceRDD = mapOrCombineRDD.groupByKey().mapPartitionsToPair(partitionIterator -> {
KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration);
- return () -> new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator);
+ return new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator);
});
if (mapReduce.getReduceKeySort().isPresent())
reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get(), true, 1);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db3dfcb4/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
index 8585e0d..dc22d47 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/interceptor/SparkStarBarrierInterceptor.java
@@ -85,13 +85,11 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
.filter(vertexWritable -> ElementHelper.idExists(vertexWritable.get().id(), graphStepIds)) // ensure vertex ids are in V(x)
.flatMap(vertexWritable -> {
if (identityTraversal) // g.V.count()-style (identity)
- return () -> IteratorUtils.of(traversal.getTraverserGenerator().generate(vertexWritable.get(), (Step) graphStep, 1l));
+ return IteratorUtils.of(traversal.getTraverserGenerator().generate(vertexWritable.get(), (Step) graphStep, 1l));
else { // add the vertex to head of the traversal
- return () -> { // and iterate it for its results
- final Traversal.Admin<Vertex, ?> clone = traversal.clone(); // need a unique clone for each vertex to isolate the computation
+ final Traversal.Admin<Vertex, ?> clone = traversal.clone(); // need a unique clone for each vertex to isolate the computation
clone.getStartStep().addStart(clone.getTraverserGenerator().generate(vertexWritable.get(), graphStep, 1l));
return (Step) clone.getEndStep();
- };
}
});
// USE SPARK DSL FOR THE RESPECTIVE END REDUCING BARRIER STEP OF THE TRAVERSAL
@@ -133,14 +131,14 @@ public final class SparkStarBarrierInterceptor implements SparkVertexProgramInte
result = ((GroupStep) endStep).generateFinalResult(nextRDD.
mapPartitions(partitions -> {
final GroupStep<Object, Object, Object> clone = (GroupStep) endStep.clone();
- return () -> IteratorUtils.map(partitions, clone::projectTraverser);
+ return IteratorUtils.map(partitions, clone::projectTraverser);
}).fold(((GroupStep<Object, Object, Object>) endStep).getSeedSupplier().get(), biOperator::apply));
} else if (endStep instanceof GroupCountStep) {
final GroupCountStep.GroupCountBiOperator<Object> biOperator = GroupCountStep.GroupCountBiOperator.instance();
result = nextRDD
.mapPartitions(partitions -> {
final GroupCountStep<Object, Object> clone = (GroupCountStep) endStep.clone();
- return () -> IteratorUtils.map(partitions, clone::projectTraverser);
+ return IteratorUtils.map(partitions, clone::projectTraverser);
})
.fold(((GroupCountStep<Object, Object>) endStep).getSeedSupplier().get(), biOperator::apply);
} else
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db3dfcb4/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 28a4d55..6735fe5 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -24,7 +24,7 @@ import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.SparkConf;
import org.apache.spark.api.python.PythonBroadcast;
-import org.apache.spark.broadcast.HttpBroadcast;
+import org.apache.spark.broadcast.TorrentBroadcast;
import org.apache.spark.network.util.ByteUnit;
import org.apache.spark.scheduler.CompressedMapStatus;
import org.apache.spark.scheduler.HighlyCompressedMapStatus;
@@ -48,24 +48,32 @@ import scala.Tuple3;
import scala.collection.mutable.WrappedArray;
import scala.runtime.BoxedUnit;
+import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class GryoSerializer extends Serializer {
+public final class GryoSerializer extends Serializer implements Serializable {
//private final Option<String> userRegistrator;
private final int bufferSize;
private final int maxBufferSize;
+ private final int poolSize;
+ private final ArrayList<String> ioRegList = new ArrayList<>();
+ private final boolean referenceTracking;
+ private final boolean registrationRequired;
- private final GryoPool gryoPool;
+
+ private transient GryoPool gryoPool;
public GryoSerializer(final SparkConf sparkConfiguration) {
final long bufferSizeKb = sparkConfiguration.getSizeAsKb("spark.kryoserializer.buffer", "64k");
final long maxBufferSizeMb = sparkConfiguration.getSizeAsMb("spark.kryoserializer.buffer.max", "64m");
- final boolean referenceTracking = sparkConfiguration.getBoolean("spark.kryo.referenceTracking", true);
- final boolean registrationRequired = sparkConfiguration.getBoolean("spark.kryo.registrationRequired", false);
+ referenceTracking = sparkConfiguration.getBoolean("spark.kryo.referenceTracking", true);
+ registrationRequired = sparkConfiguration.getBoolean("spark.kryo.registrationRequired", false);
if (bufferSizeKb >= ByteUnit.GiB.toKiB(2L)) {
throw new IllegalArgumentException("spark.kryoserializer.buffer must be less than 2048 mb, got: " + bufferSizeKb + " mb.");
} else {
@@ -77,9 +85,19 @@ public final class GryoSerializer extends Serializer {
//this.userRegistrator = sparkConfiguration.getOption("spark.kryo.registrator");
}
}
- this.gryoPool = GryoPool.build().
- poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT)).
- ioRegistries(makeApacheConfiguration(sparkConfiguration).getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).
+ poolSize = sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
+ List<Object> list = makeApacheConfiguration(sparkConfiguration).getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList());
+ list.forEach(c -> {
+ ioRegList.add(c.toString());
+ }
+ );
+ }
+
+ private GryoPool createPool(){
+ List<Object> list = new ArrayList<>(ioRegList);
+ return GryoPool.build().
+ poolSize(poolSize).
+ ioRegistries(list).
initializeMapper(builder -> {
try {
builder.addCustom(Tuple2.class, new Tuple2Serializer())
@@ -91,7 +109,7 @@ public final class GryoSerializer extends Serializer {
.addCustom(CompressedMapStatus.class)
.addCustom(BlockManagerId.class)
.addCustom(HighlyCompressedMapStatus.class, new ExternalizableSerializer()) // externalizable implemented so its okay
- .addCustom(HttpBroadcast.class)
+ .addCustom(TorrentBroadcast.class)
.addCustom(PythonBroadcast.class)
.addCustom(BoxedUnit.class)
.addCustom(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer())
@@ -118,6 +136,13 @@ public final class GryoSerializer extends Serializer {
}
public GryoPool getGryoPool() {
+ if (gryoPool == null) {
+ synchronized (this) {
+ if (gryoPool == null) {
+ gryoPool = createPool();
+ }
+ }
+ }
return this.gryoPool;
}
[11/29] tinkerpop git commit: Deprecated OptIn "performance" tests
Posted by ok...@apache.org.
Deprecated OptIn "performance" tests
This should have been done a long time ago on 3.2.1, but it was missed CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/2d9bb204
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/2d9bb204
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/2d9bb204
Branch: refs/heads/TINKERPOP-1389
Commit: 2d9bb2047271ccf453c6aedbf756322fe87f535e
Parents: 285ff35
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Oct 25 16:13:51 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Oct 25 16:13:51 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/upgrade/release-3.2.x-incubating.asciidoc | 16 ++++++++++++++++
.../apache/tinkerpop/gremlin/structure/Graph.java | 15 +++++++++++++++
3 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2d9bb204/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index a281c38..73bfa1e 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
TinkerPop 3.2.4 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Deprecated the "performance" tests in `OptIn`.
* Fixed a severe bug where `GraphComputer` strategies are not being loaded until the second use of the traversal source.
[[release-3-2-3]]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2d9bb204/docs/src/upgrade/release-3.2.x-incubating.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.2.x-incubating.asciidoc b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
index 3ed46b2..0fba8f5 100644
--- a/docs/src/upgrade/release-3.2.x-incubating.asciidoc
+++ b/docs/src/upgrade/release-3.2.x-incubating.asciidoc
@@ -29,6 +29,22 @@ TinkerPop 3.2.4
Please see the link:https://github.com/apache/tinkerpop/blob/3.2.4/CHANGELOG.asciidoc#release-3-2-4[changelog] for a complete list of all the modifications that are part of this release.
+Upgrading for Providers
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Graph Database Providers
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Deprecated Performance OptIn
+++++++++++++++++++++++++++++
+
+In 3.2.1, all `junit-benchmark` performance tests were deprecated. At that time, the `OptIn` representations of these
+tests should have been deprecated as well, but they were not. That omission has been remedied now. Specifically, the
+following fields were deprecated:
+
+* `OptIn.SUITE_GROOVY_ENVIRONMENT_PERFORMANCE`
+* `OptIn.SUITE_PROCESS_PERFORMANCE`
+* `OptIn.SUITE_STRUCTURE_PERFORMANCE`
TinkerPop 3.2.3
---------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2d9bb204/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
index 01379f4..ed3f12d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
@@ -1200,14 +1200,29 @@ public interface Graph extends AutoCloseable, Host {
public @interface OptIn {
public static String SUITE_STRUCTURE_STANDARD = "org.apache.tinkerpop.gremlin.structure.StructureStandardSuite";
public static String SUITE_STRUCTURE_INTEGRATE = "org.apache.tinkerpop.gremlin.structure.StructureIntegrateSuite";
+
+ /**
+ * @deprecated As of release 3.2.4, effectively replaced by the gremlin-benchmarks module.
+ */
+ @Deprecated
public static String SUITE_STRUCTURE_PERFORMANCE = "org.apache.tinkerpop.gremlin.structure.StructurePerformanceSuite";
public static String SUITE_PROCESS_COMPUTER = "org.apache.tinkerpop.gremlin.process.ProcessComputerSuite";
public static String SUITE_PROCESS_STANDARD = "org.apache.tinkerpop.gremlin.process.ProcessStandardSuite";
+
+ /**
+ * @deprecated As of release 3.2.4, effectively replaced by the gremlin-benchmarks module.
+ */
+ @Deprecated
public static String SUITE_PROCESS_PERFORMANCE = "org.apache.tinkerpop.gremlin.process.ProcessPerformanceSuite";
public static String SUITE_GROOVY_PROCESS_STANDARD = "org.apache.tinkerpop.gremlin.process.GroovyProcessStandardSuite";
public static String SUITE_GROOVY_PROCESS_COMPUTER = "org.apache.tinkerpop.gremlin.process.GroovyProcessComputerSuite";
public static String SUITE_GROOVY_ENVIRONMENT = "org.apache.tinkerpop.gremlin.groovy.GroovyEnvironmentSuite";
public static String SUITE_GROOVY_ENVIRONMENT_INTEGRATE = "org.apache.tinkerpop.gremlin.groovy.GroovyEnvironmentIntegrateSuite";
+
+ /**
+ * @deprecated As of release 3.2.4, effectively replaced by the gremlin-benchmarks module.
+ */
+ @Deprecated
public static String SUITE_GROOVY_ENVIRONMENT_PERFORMANCE = "org.apache.tinkerpop.gremlin.groovy.GroovyEnvironmentPerformanceSuite";
/**
[16/29] tinkerpop git commit: Merge branch 'TINKERPOP-1507' of
https://github.com/apache/tinkerpop into tp32
Posted by ok...@apache.org.
Merge branch 'TINKERPOP-1507' of https://github.com/apache/tinkerpop into tp32
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ea8cd65c
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ea8cd65c
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ea8cd65c
Branch: refs/heads/TINKERPOP-1389
Commit: ea8cd65ca933b36cbe3d47352d73d96ded3a7ad4
Parents: 16c7c24 2817c01
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Oct 26 08:12:02 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:12:02 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/dev/io/graphson.asciidoc | 1 +
.../structure/io/graphson/GraphSONModule.java | 4 +++
.../gremlin/structure/io/gryo/GryoMapper.java | 4 ++-
.../tinkerpop/gremlin/util/CoreImports.java | 6 +++-
.../step/branch/GroovyBranchTest.groovy | 13 ++++++++-
.../step/branch/GroovyChooseTest.groovy | 10 +++++++
.../gremlin/groovy/jsr223/GroovyTranslator.java | 3 ++
.../gremlin/python/jsr223/PythonTranslator.java | 3 ++
.../jython/gremlin_python/process/traversal.py | 4 +++
.../traversal/step/branch/BranchTest.java | 29 ++++++++++++++++----
.../traversal/step/branch/ChooseTest.java | 19 +++++++++++++
12 files changed, 89 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ea8cd65c/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index dc8da2e,d2cd065..74bfcef
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -26,7 -26,7 +26,8 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.2.4 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Deprecated the "performance" tests in `OptIn`.
+ * Added `Pick.none` and `Pick.any` to the serializers and importers.
* Fixed a severe bug where `GraphComputer` strategies are not being loaded until the second use of the traversal source.
[[release-3-2-3]]
[10/29] tinkerpop git commit: Merge branch 'tp32'
Posted by ok...@apache.org.
Merge branch 'tp32'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/f161583e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/f161583e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/f161583e
Branch: refs/heads/TINKERPOP-1389
Commit: f161583e7f2804b43ce6f6b7cbb588dce9fe4f67
Parents: 2ae615f 285ff35
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Oct 25 11:34:50 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Oct 25 11:34:50 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 5 +++++
docs/src/upgrade/release-3.1.x-incubating.asciidoc | 7 +++++++
.../gremlin/algorithm/generator/CommunityGeneratorTest.java | 4 ++++
.../algorithm/generator/DistributionGeneratorTest.java | 4 ++++
4 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f161583e/CHANGELOG.asciidoc
----------------------------------------------------------------------
[17/29] tinkerpop git commit: Merge branch 'tp32'
Posted by ok...@apache.org.
Merge branch 'tp32'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/1484c8de
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/1484c8de
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/1484c8de
Branch: refs/heads/TINKERPOP-1389
Commit: 1484c8deb2438517b7fc12ec8a8ed154732cf47f
Parents: 7a4b338 ea8cd65
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Oct 26 08:12:17 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:12:17 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/dev/io/graphson.asciidoc | 1 +
.../structure/io/graphson/GraphSONModule.java | 4 +++
.../gremlin/structure/io/gryo/GryoMapper.java | 4 ++-
.../tinkerpop/gremlin/util/CoreImports.java | 6 +++-
.../step/branch/GroovyBranchTest.groovy | 13 ++++++++-
.../step/branch/GroovyChooseTest.groovy | 10 +++++++
.../gremlin/groovy/jsr223/GroovyTranslator.java | 3 ++
.../gremlin/python/jsr223/PythonTranslator.java | 3 ++
.../jython/gremlin_python/process/traversal.py | 4 +++
.../traversal/step/branch/BranchTest.java | 29 ++++++++++++++++----
.../traversal/step/branch/ChooseTest.java | 19 +++++++++++++
12 files changed, 89 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/1484c8de/CHANGELOG.asciidoc
----------------------------------------------------------------------
[02/29] tinkerpop git commit: Merge branch 'tp32' into TINKERPOP-1507
Posted by ok...@apache.org.
Merge branch 'tp32' into TINKERPOP-1507
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/fc6ce291
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/fc6ce291
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/fc6ce291
Branch: refs/heads/TINKERPOP-1389
Commit: fc6ce2917b5850e3538acd098030371a21e4a441
Parents: 0fec46d b262c7e
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Oct 25 06:05:25 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Oct 25 06:05:25 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 4 ++--
docs/src/dev/developer/release.asciidoc | 4 ++++
docs/src/reference/the-traversal.asciidoc | 4 ++--
.../decoration/VertexProgramStrategy.java | 5 +++++
.../server/GremlinServerIntegrateTest.java | 18 ------------------
5 files changed, 13 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
[27/29] tinkerpop git commit: added SystemUtil to gremlin-core which
is able to generate Configurations from System.properties. Useful in Hadoop
GraphComputer settings where JVMs can have IO configuration information in
them. HadoopPools will now try and
Posted by ok...@apache.org.
added SystemUtil to gremlin-core which is able to generate Configurations from System.properties. Useful in Hadoop GraphComputer settings where JVMs can have IO configuration information in them. HadoopPools will now try and generate a Configuraiton from System.properties if no configuration is explicitly provided. Likewise for KryoShimServices (via KryoShimServiceLoader). SparkGraphComputer will spawn its driver and executors using gremlin. and spark. System properties taken from the SparkConf. Lots of nick nack cleanups to the shim service scene. cc/ @dalaro.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/982d2207
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/982d2207
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/982d2207
Branch: refs/heads/TINKERPOP-1389
Commit: 982d2207bc5840f0f3c83de480990af073e80122
Parents: 8367ab3
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Oct 26 07:18:21 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:16:26 2016 -0600
----------------------------------------------------------------------
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 108 ++++++++-----------
.../tinkerpop/gremlin/util/SystemUtil.java | 55 ++++++++++
.../tinkerpop/gremlin/util/SystemUtilTest.java | 89 +++++++++++++++
.../hadoop/structure/io/HadoopPools.java | 19 +---
.../process/computer/SparkGraphComputer.java | 18 +++-
.../unshaded/UnshadedKryoShimService.java | 56 ++++------
...SparkGremlinGryoSerializerIntegrateTest.java | 33 ++++++
7 files changed, 259 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index f9e4c2e..7b67328 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -19,6 +19,8 @@
package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationUtils;
+import org.apache.tinkerpop.gremlin.util.SystemUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,8 +49,12 @@ public class KryoShimServiceLoader {
public static final String KRYO_SHIM_SERVICE = "gremlin.io.kryoShimService";
public static void applyConfiguration(final Configuration configuration) {
- KryoShimServiceLoader.configuration = configuration;
- load(true);
+ if (null == KryoShimServiceLoader.configuration ||
+ !ConfigurationUtils.toString(KryoShimServiceLoader.configuration).equals(ConfigurationUtils.toString(configuration))) {
+ KryoShimServiceLoader.configuration = configuration;
+ load(true);
+ } else
+ load(false);
}
/**
@@ -62,78 +68,54 @@ public class KryoShimServiceLoader {
* @return the shim service
*/
public static KryoShimService load(final boolean forceReload) {
-
- if (null != cachedShimService && !forceReload) {
+ // if the service is loaded and doesn't need reloading, simply return in
+ if (null != cachedShimService && !forceReload)
return cachedShimService;
- }
-
- final ArrayList<KryoShimService> services = new ArrayList<>();
- final ServiceLoader<KryoShimService> sl = ServiceLoader.load(KryoShimService.class);
-
- KryoShimService result = null;
+ // if the configuration is null, try and load the configuration from System.properties
+ if (null == configuration)
+ configuration = SystemUtil.getSystemPropertiesConfiguration("gremlin", true);
+ // get all of the shim services
+ final ArrayList<KryoShimService> services = new ArrayList<>();
+ final ServiceLoader<KryoShimService> serviceLoader = ServiceLoader.load(KryoShimService.class);
synchronized (KryoShimServiceLoader.class) {
- if (forceReload) {
- sl.reload();
- }
-
- for (KryoShimService kss : sl) {
+ if (forceReload) serviceLoader.reload();
+ for (final KryoShimService kss : serviceLoader) {
services.add(kss);
}
}
-
- String shimClass = null != configuration && configuration.containsKey(KRYO_SHIM_SERVICE) ?
- configuration.getString(KRYO_SHIM_SERVICE) :
- System.getProperty(KRYO_SHIM_SERVICE);
-
- if (null != shimClass) {
- for (KryoShimService kss : services) {
- if (kss.getClass().getCanonicalName().equals(shimClass)) {
+ // if a shim service class is specified in the configuration, use it -- else, priority-based
+ if (configuration.containsKey(KRYO_SHIM_SERVICE)) {
+ for (final KryoShimService kss : services) {
+ if (kss.getClass().getCanonicalName().equals(configuration.getString(KRYO_SHIM_SERVICE))) {
log.info("Set {} provider to {} ({}) from system property {}={}",
KryoShimService.class.getSimpleName(), kss, kss.getClass(),
- KRYO_SHIM_SERVICE, shimClass);
- result = kss;
+ KRYO_SHIM_SERVICE, configuration.getString(KRYO_SHIM_SERVICE));
+ cachedShimService = kss;
+ break;
}
}
} else {
Collections.sort(services, KryoShimServiceComparator.INSTANCE);
-
- for (KryoShimService kss : services) {
+ for (final KryoShimService kss : services) {
log.debug("Found Kryo shim service class {} (priority {})", kss.getClass(), kss.getPriority());
}
-
if (0 != services.size()) {
- result = services.get(services.size() - 1);
-
+ cachedShimService = services.get(services.size() - 1);
log.info("Set {} provider to {} ({}) because its priority value ({}) is the best available",
- KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
+ KryoShimService.class.getSimpleName(), cachedShimService, cachedShimService.getClass(), cachedShimService.getPriority());
}
}
-
- if (null == result) {
+ // no shim service was available
+ if (null == cachedShimService)
throw new IllegalStateException("Unable to load KryoShimService");
- }
-
- final Configuration userConf = configuration;
-
- if (null != userConf) {
- log.info("Configuring {} provider {} with user-provided configuration",
- KryoShimService.class.getSimpleName(), result);
- result.applyConfiguration(userConf);
- }
- return cachedShimService = result;
- }
-
- /**
- * Equivalent to {@link #load(boolean)} with the parameter {@code true}.
- *
- * @return the (possibly cached) shim service
- */
- public static KryoShimService load() {
- return load(false);
+ // once the shim service is defined, configure it
+ log.info("Configuring {} provider {} with user-provided configuration", KryoShimService.class.getSimpleName(), cachedShimService.getClass().getCanonicalName());
+ cachedShimService.applyConfiguration(configuration);
+ return cachedShimService;
}
/**
@@ -141,16 +123,13 @@ public class KryoShimServiceLoader {
* where the {@code output} parameter is an internally-created {@link ByteArrayOutputStream}. Returns
* the byte array underlying that stream.
*
- * @param o an object for which the instance and class are serialized
+ * @param object an object for which the instance and class are serialized
* @return the serialized form
*/
- public static byte[] writeClassAndObjectToBytes(final Object o) {
- final KryoShimService shimService = load();
-
+ public static byte[] writeClassAndObjectToBytes(final Object object) {
+ final KryoShimService shimService = load(false);
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- shimService.writeClassAndObject(o, baos);
-
+ shimService.writeClassAndObject(object, baos);
return baos.toByteArray();
}
@@ -158,14 +137,13 @@ public class KryoShimServiceLoader {
* A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#readClassAndObject},
* where the {@code input} parameter is {@code source}. Returns the deserialized object.
*
- * @param source an input stream containing data for a serialized object class and instance
- * @param <T> the type to which the deserialized object is cast as it is returned
+ * @param inputStream an input stream containing data for a serialized object class and instance
+ * @param <T> the type to which the deserialized object is cast as it is returned
* @return the deserialized object
*/
- public static <T> T readClassAndObject(final InputStream source) {
- final KryoShimService shimService = load();
-
- return (T) shimService.readClassAndObject(source);
+ public static <T> T readClassAndObject(final InputStream inputStream) {
+ final KryoShimService shimService = load(false);
+ return (T) shimService.readClassAndObject(inputStream);
}
/**
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/SystemUtil.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/SystemUtil.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/SystemUtil.java
new file mode 100644
index 0000000..bc2c5ff
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/SystemUtil.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.util;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+
+import java.util.Map;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SystemUtil {
+
+ private SystemUtil() {
+ }
+
+ /**
+ * Generate a {@link Configuration} from the {@link System#getProperties}.
+ * Only those properties with specified prefix key are aggregated.
+ * If the prefix and a . should be removed, then trim prefix.
+ *
+ * @param prefix the prefix of the keys to include in the configuration
+ * @param trimPrefix whether to trim the prefix + . from the key
+ * @return a configuration generated from the System properties
+ */
+ public static Configuration getSystemPropertiesConfiguration(final String prefix, final boolean trimPrefix) {
+ final BaseConfiguration apacheConfiguration = new BaseConfiguration();
+ apacheConfiguration.setDelimiterParsingDisabled(true);
+ for (final Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
+ final String key = entry.getKey().toString();
+ final Object value = entry.getValue();
+ if (key.startsWith(prefix + "."))
+ apacheConfiguration.setProperty(trimPrefix ? key.substring(prefix.length() + 1) : key, value);
+ }
+ return apacheConfiguration;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/util/SystemUtilTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/util/SystemUtilTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/util/SystemUtilTest.java
new file mode 100644
index 0000000..e5dc705
--- /dev/null
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/util/SystemUtilTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.util;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SystemUtilTest {
+
+ @Test
+ public void shouldLoadSystemProperties() {
+ System.setProperty("blah.aa", "1");
+ System.setProperty("blah.b", "true");
+ System.setProperty("blah.c", "three");
+ System.setProperty("bleep.d", "false");
+ Configuration configuration = SystemUtil.getSystemPropertiesConfiguration("blah", false);
+ assertEquals(3, IteratorUtils.count(configuration.getKeys()));
+ assertEquals(1, configuration.getInt("blah.aa"));
+ assertTrue(configuration.getBoolean("blah.b"));
+ assertEquals("three", configuration.getProperty("blah.c"));
+ assertFalse(configuration.containsKey("d") || configuration.containsKey("bleep.d"));
+ System.clearProperty("blah.aa");
+ System.clearProperty("blah.b");
+ System.clearProperty("blah.c");
+ System.clearProperty("bleep.d");
+ }
+
+ @Test
+ public void shouldTrimSystemPropertyPrefixes() {
+ System.setProperty("blah.a", "1");
+ System.setProperty("blah.bbb", "true");
+ System.setProperty("blah.c", "three");
+ System.setProperty("bleep.d", "false");
+ Configuration configuration = SystemUtil.getSystemPropertiesConfiguration("blah", true);
+ assertEquals(3, IteratorUtils.count(configuration.getKeys()));
+ assertEquals(1, configuration.getInt("a"));
+ assertTrue(configuration.getBoolean("bbb"));
+ assertEquals("three", configuration.getProperty("c"));
+ assertFalse(configuration.containsKey("d") || configuration.containsKey("bleep.d"));
+ System.clearProperty("blah.a");
+ System.clearProperty("blah.bbb");
+ System.clearProperty("blah.c");
+ System.clearProperty("bleep.d");
+ }
+
+ @Test
+ public void shouldTrimSystemPropertyPrefixesAndNoMore() {
+ System.setProperty("blah.a.x", "1");
+ System.setProperty("blah.b.y", "true");
+ System.setProperty("blah.cc.zzz", "three");
+ System.setProperty("bleep.d.d", "false");
+ Configuration configuration = SystemUtil.getSystemPropertiesConfiguration("blah", true);
+ assertEquals(3, IteratorUtils.count(configuration.getKeys()));
+ assertEquals(1, configuration.getInt("a.x"));
+ assertTrue(configuration.getBoolean("b.y"));
+ assertEquals("three", configuration.getProperty("cc.zzz"));
+ assertFalse(configuration.containsKey("d") || configuration.containsKey("bleep.d"));
+ assertFalse(configuration.containsKey("d.d") || configuration.containsKey("bleep.d.d"));
+ System.clearProperty("blah.a.x");
+ System.clearProperty("blah.b.y");
+ System.clearProperty("blah.cc.zzz");
+ System.clearProperty("bleep.d.d");
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
index 25bc8b4..eb9a884 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
@@ -18,11 +18,12 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationUtils;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
+import org.apache.tinkerpop.gremlin.util.SystemUtil;
import java.util.Collections;
@@ -31,8 +32,6 @@ import java.util.Collections;
*/
public final class HadoopPools {
- private static final Configuration EMPTY_CONFIGURATION = new BaseConfiguration();
-
private HadoopPools() {
}
@@ -61,17 +60,9 @@ public final class HadoopPools {
public static GryoPool getGryoPool() {
if (!INITIALIZED) {
- /*if (null != System.getProperty("configuration", null)) {
- try {
- HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using the System properties configuration");
- initialize((Configuration) Serializer.deserializeObject(System.getProperty("configuration").getBytes()));
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- } else {*/
- HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using the default pool");
- initialize(EMPTY_CONFIGURATION);
- //}
+ final Configuration configuration = SystemUtil.getSystemPropertiesConfiguration("gremlin", true);
+ HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using system properties configuration: " + ConfigurationUtils.toString(configuration));
+ initialize(configuration);
}
return GRYO_POOL;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index d345100..81ebaaa 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -70,7 +70,6 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
@@ -115,10 +114,19 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
UnshadedKryoShimService.class.getCanonicalName() :
HadoopPoolShimService.class.getCanonicalName();
this.sparkConfiguration.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, shimService);
- this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
- (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
- this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
- (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ ///////////
+ final StringBuilder params = new StringBuilder();
+ this.sparkConfiguration.getKeys().forEachRemaining(key -> {
+ if (key.startsWith("gremlin") || key.startsWith("spark")) {
+ params.append(" -D").append("gremlin.").append(key).append("=").append(this.sparkConfiguration.getProperty(key));
+ }
+ });
+ if (params.length() > 0) {
+ this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + params.toString()).trim());
+ this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + params.toString()).trim());
+ }
KryoShimServiceLoader.applyConfiguration(this.sparkConfiguration);
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
index 0789d6a..2b0efda 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
@@ -44,56 +44,43 @@ import java.util.concurrent.LinkedBlockingQueue;
public class UnshadedKryoShimService implements KryoShimService {
private static final Logger log = LoggerFactory.getLogger(UnshadedKryoShimService.class);
-
private static final LinkedBlockingQueue<Kryo> KRYOS = new LinkedBlockingQueue<>();
-
private static volatile boolean initialized;
- public UnshadedKryoShimService() {
- }
-
@Override
- public Object readClassAndObject(final InputStream source) {
-
+ public Object readClassAndObject(final InputStream inputStream) {
final LinkedBlockingQueue<Kryo> kryos = initialize();
-
Kryo k = null;
try {
k = kryos.take();
-
- return k.readClassAndObject(new Input(source));
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
+ return k.readClassAndObject(new Input(inputStream));
+ } catch (final InterruptedException e) {
+ throw new IllegalStateException(e);
} finally {
try {
kryos.put(k);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
+ } catch (final InterruptedException e) {
+ throw new IllegalStateException(e);
}
}
}
@Override
- public void writeClassAndObject(final Object o, OutputStream sink) {
-
+ public void writeClassAndObject(final Object object, OutputStream outputStream) {
final LinkedBlockingQueue<Kryo> kryos = initialize();
-
Kryo k = null;
try {
k = kryos.take();
-
- final Output kryoOutput = new Output(sink);
-
- k.writeClassAndObject(kryoOutput, o);
-
+ final Output kryoOutput = new Output(outputStream);
+ k.writeClassAndObject(kryoOutput, object);
kryoOutput.flush();
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
+ } catch (final InterruptedException e) {
+ throw new IllegalStateException(e);
} finally {
try {
kryos.put(k);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
+ } catch (final InterruptedException e) {
+ throw new IllegalStateException(e);
}
}
}
@@ -104,15 +91,15 @@ public class UnshadedKryoShimService implements KryoShimService {
}
@Override
- public void applyConfiguration(final Configuration conf) {
- initialize(conf);
+ public void applyConfiguration(final Configuration configuration) {
+ initialize(configuration);
}
private LinkedBlockingQueue<Kryo> initialize() {
return initialize(new BaseConfiguration());
}
- private LinkedBlockingQueue<Kryo> initialize(final Configuration conf) {
+ private LinkedBlockingQueue<Kryo> initialize(final Configuration configuration) {
// DCL is safe in this case due to volatility
if (!initialized) {
synchronized (UnshadedKryoShimService.class) {
@@ -120,14 +107,13 @@ public class UnshadedKryoShimService implements KryoShimService {
final SparkConf sparkConf = new SparkConf();
// Copy the user's IoRegistry from the param conf to the SparkConf we just created
- final String regStr = conf.getString(GryoPool.CONFIG_IO_REGISTRY);
- if (null != regStr) { // SparkConf rejects null values with NPE, so this has to be checked before set(...)
+ final String regStr = configuration.getString(GryoPool.CONFIG_IO_REGISTRY, null);
+ if (null != regStr) // SparkConf rejects null values with NPE, so this has to be checked before set(...)
sparkConf.set(GryoPool.CONFIG_IO_REGISTRY, regStr);
- }
+
// Setting spark.serializer here almost certainly isn't necessary, but it doesn't hurt
sparkConf.set(Constants.SPARK_SERIALIZER, IoRegistryAwareKryoSerializer.class.getCanonicalName());
-
- final String registrator = conf.getString(Constants.SPARK_KRYO_REGISTRATOR);
+ final String registrator = configuration.getString(Constants.SPARK_KRYO_REGISTRATOR);
if (null != registrator) {
sparkConf.set(Constants.SPARK_KRYO_REGISTRATOR, registrator);
log.info("Copied " + Constants.SPARK_KRYO_REGISTRATOR + ": {}", registrator);
@@ -140,7 +126,7 @@ public class UnshadedKryoShimService implements KryoShimService {
// Setup a pool backed by our spark.serializer instance
// Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf)
- final int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
+ final int poolSize = configuration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
for (int i = 0; i < poolSize; i++) {
KRYOS.add(ioReg.newKryo());
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/982d2207/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/SparkGremlinGryoSerializerIntegrateTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/SparkGremlinGryoSerializerIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/SparkGremlinGryoSerializerIntegrateTest.java
new file mode 100644
index 0000000..6523592
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/SparkGremlinGryoSerializerIntegrateTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphGryoSerializerProvider;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(SparkGremlinSuite.class)
+@GraphProviderClass(provider = SparkHadoopGraphGryoSerializerProvider.class, graph = HadoopGraph.class)
+public class SparkGremlinGryoSerializerIntegrateTest {
+}
[15/29] tinkerpop git commit: Merge branch 'tp32'
Posted by ok...@apache.org.
Merge branch 'tp32'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7a4b3385
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7a4b3385
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7a4b3385
Branch: refs/heads/TINKERPOP-1389
Commit: 7a4b33855714235573a786715c742bef6d8add61
Parents: 69af7c2 16c7c24
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Oct 26 09:33:59 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Oct 26 09:33:59 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../tinkerpop/gremlin/FeatureRequirementSet.java | 2 +-
.../apache/tinkerpop/gremlin/structure/EdgeTest.java | 13 ++++---------
3 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7a4b3385/CHANGELOG.asciidoc
----------------------------------------------------------------------
[26/29] tinkerpop git commit: using tinkerpop as the prefix for
System property configurations. Tweaked the KryoShimServiceLoader's logging
information so its easier to see whats going on --- doing cluster testing.
Posted by ok...@apache.org.
using tinkerpop as the prefix for System property configurations. Tweaked the KryoShimServiceLoader's logging information so its easier to see whats going on --- doing cluster testing.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/db8e8c10
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/db8e8c10
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/db8e8c10
Branch: refs/heads/TINKERPOP-1389
Commit: db8e8c10677c4c4d584d000a172976e806207c1e
Parents: 982d220
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Oct 26 08:04:38 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:16:26 2016 -0600
----------------------------------------------------------------------
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 21 +++++++++++---------
.../hadoop/structure/io/HadoopPools.java | 2 +-
.../process/computer/SparkGraphComputer.java | 2 +-
3 files changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db8e8c10/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index 7b67328..0051204 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -67,14 +67,14 @@ public class KryoShimServiceLoader {
* before selecting a new service to return
* @return the shim service
*/
- public static KryoShimService load(final boolean forceReload) {
+ private static KryoShimService load(final boolean forceReload) {
// if the service is loaded and doesn't need reloading, simply return in
if (null != cachedShimService && !forceReload)
return cachedShimService;
// if the configuration is null, try and load the configuration from System.properties
if (null == configuration)
- configuration = SystemUtil.getSystemPropertiesConfiguration("gremlin", true);
+ configuration = SystemUtil.getSystemPropertiesConfiguration("tinkerpop", true);
// get all of the shim services
final ArrayList<KryoShimService> services = new ArrayList<>();
@@ -89,9 +89,10 @@ public class KryoShimServiceLoader {
if (configuration.containsKey(KRYO_SHIM_SERVICE)) {
for (final KryoShimService kss : services) {
if (kss.getClass().getCanonicalName().equals(configuration.getString(KRYO_SHIM_SERVICE))) {
- log.info("Set {} provider to {} ({}) from system property {}={}",
- KryoShimService.class.getSimpleName(), kss, kss.getClass(),
- KRYO_SHIM_SERVICE, configuration.getString(KRYO_SHIM_SERVICE));
+ log.info("Set KryoShimService to {} because of configuration {}={}",
+ kss.getClass().getSimpleName(),
+ KRYO_SHIM_SERVICE,
+ configuration.getString(KRYO_SHIM_SERVICE));
cachedShimService = kss;
break;
}
@@ -99,12 +100,12 @@ public class KryoShimServiceLoader {
} else {
Collections.sort(services, KryoShimServiceComparator.INSTANCE);
for (final KryoShimService kss : services) {
- log.debug("Found Kryo shim service class {} (priority {})", kss.getClass(), kss.getPriority());
+ log.debug("Found KryoShimService: {} (priority {})", kss.getClass().getCanonicalName(), kss.getPriority());
}
if (0 != services.size()) {
cachedShimService = services.get(services.size() - 1);
- log.info("Set {} provider to {} ({}) because its priority value ({}) is the best available",
- KryoShimService.class.getSimpleName(), cachedShimService, cachedShimService.getClass(), cachedShimService.getPriority());
+ log.info("Set KryoShimService to {} because its priority value ({}) is the best available",
+ cachedShimService.getClass().getSimpleName(), cachedShimService.getPriority());
}
}
@@ -113,7 +114,9 @@ public class KryoShimServiceLoader {
throw new IllegalStateException("Unable to load KryoShimService");
// once the shim service is defined, configure it
- log.info("Configuring {} provider {} with user-provided configuration", KryoShimService.class.getSimpleName(), cachedShimService.getClass().getCanonicalName());
+ log.info("Configuring KryoShimService {} with following configuration: {}",
+ cachedShimService.getClass().getCanonicalName(),
+ ConfigurationUtils.toString(configuration));
cachedShimService.applyConfiguration(configuration);
return cachedShimService;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db8e8c10/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
index eb9a884..939887f 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
@@ -60,7 +60,7 @@ public final class HadoopPools {
public static GryoPool getGryoPool() {
if (!INITIALIZED) {
- final Configuration configuration = SystemUtil.getSystemPropertiesConfiguration("gremlin", true);
+ final Configuration configuration = SystemUtil.getSystemPropertiesConfiguration("tinkerpop", true);
HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using system properties configuration: " + ConfigurationUtils.toString(configuration));
initialize(configuration);
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/db8e8c10/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 81ebaaa..af158a0 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -118,7 +118,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
final StringBuilder params = new StringBuilder();
this.sparkConfiguration.getKeys().forEachRemaining(key -> {
if (key.startsWith("gremlin") || key.startsWith("spark")) {
- params.append(" -D").append("gremlin.").append(key).append("=").append(this.sparkConfiguration.getProperty(key));
+ params.append(" -D").append("tinkerpop.").append(key).append("=").append(this.sparkConfiguration.getProperty(key));
}
});
if (params.length() > 0) {
[21/29] tinkerpop git commit: @dalaro provided an update to
KryoShimServiceLoader via a gist as the LOG message was not so clear.
Posted by ok...@apache.org.
@dalaro provided an update to KryoShimServiceLoader via a gist as the LOG message was not so clear.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/508e1fd3
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/508e1fd3
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/508e1fd3
Branch: refs/heads/TINKERPOP-1389
Commit: 508e1fd3f9b17f3f57e78c607de20c20a8e0b9c4
Parents: 6a6fa7b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Oct 24 13:31:58 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:14:21 2016 -0600
----------------------------------------------------------------------
.../structure/io/gryo/kryoshim/KryoShimServiceLoader.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/508e1fd3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index 4227edc..c026130 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -19,8 +19,6 @@
package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,6 +104,9 @@ public class KryoShimServiceLoader {
if (0 != services.size()) {
result = services.get(services.size() - 1);
+
+ log.info("Set {} provider to {} ({}) because its priority value ({}) is the best available",
+ KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
}
}
@@ -114,9 +115,6 @@ public class KryoShimServiceLoader {
throw new IllegalStateException("Unable to load KryoShimService");
}
- log.info("Set {} provider to {} ({}) because its priority value ({}) is the highest available",
- KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
-
final Configuration userConf = conf;
if (null != userConf) {
[13/29] tinkerpop git commit: Fixed test requirements in EdgeTest CTR
Posted by ok...@apache.org.
Fixed test requirements in EdgeTest CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/781ff3a0
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/781ff3a0
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/781ff3a0
Branch: refs/heads/TINKERPOP-1389
Commit: 781ff3a06907d6ebcfa54ab2985449c764474226
Parents: 1f84ad3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Oct 26 09:33:06 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Oct 26 09:33:06 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../tinkerpop/gremlin/FeatureRequirementSet.java | 2 +-
.../apache/tinkerpop/gremlin/structure/EdgeTest.java | 13 ++++---------
3 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/781ff3a0/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 551a2be..0dafd1c 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
TinkerPop 3.1.6 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Minor fixes to various test feature requirements in `gremlin-test`.
[[release-3-1-5]]
TinkerPop 3.1.5 (Release Date: October 17, 2016)
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/781ff3a0/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/FeatureRequirementSet.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/FeatureRequirementSet.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/FeatureRequirementSet.java
index 3941584..6b0e2f2 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/FeatureRequirementSet.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/FeatureRequirementSet.java
@@ -56,7 +56,7 @@ public @interface FeatureRequirementSet {
add(FeatureRequirement.Factory.create(Graph.Features.EdgeFeatures.FEATURE_ADD_EDGES, Graph.Features.EdgeFeatures.class));
add(FeatureRequirement.Factory.create(Graph.Features.VertexFeatures.FEATURE_ADD_PROPERTY, Graph.Features.VertexFeatures.class));
add(FeatureRequirement.Factory.create(Graph.Features.VertexPropertyFeatures.FEATURE_STRING_VALUES, Graph.Features.VertexPropertyFeatures.class));
- add(FeatureRequirement.Factory.create(Graph.Features.VertexPropertyFeatures.FEATURE_STRING_VALUES, Graph.Features.EdgePropertyFeatures.class));
+ add(FeatureRequirement.Factory.create(Graph.Features.EdgePropertyFeatures.FEATURE_STRING_VALUES, Graph.Features.EdgePropertyFeatures.class));
add(FeatureRequirement.Factory.create(Graph.Features.EdgeFeatures.FEATURE_ADD_PROPERTY, Graph.Features.EdgeFeatures.class));
}};
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/781ff3a0/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/EdgeTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/EdgeTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/EdgeTest.java
index 320afb4..b7c521d 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/EdgeTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/EdgeTest.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.structure;
import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
import org.apache.tinkerpop.gremlin.ExceptionCoverage;
import org.apache.tinkerpop.gremlin.FeatureRequirement;
+import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.junit.Test;
@@ -214,11 +215,7 @@ public class EdgeTest {
}
@Test
- @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_ADD_EDGES)
- @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_ADD_PROPERTY)
- @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_REMOVE_PROPERTY)
- @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = Graph.Features.VertexFeatures.FEATURE_ADD_VERTICES)
- @FeatureRequirement(featureClass = Graph.Features.EdgePropertyFeatures.class, feature = FEATURE_STRING_VALUES)
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
public void shouldGetPropertyKeysOnEdge() {
final Vertex v = graph.addVertex();
final Edge e = v.addEdge("friend", v, "name", "marko", "location", "desert", "status", "dope");
@@ -272,10 +269,8 @@ public class EdgeTest {
}
@Test
- @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = Graph.Features.VertexFeatures.FEATURE_ADD_VERTICES)
- @FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = Graph.Features.VertexFeatures.FEATURE_ADD_PROPERTY)
- @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_ADD_EDGES)
- @FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_ADD_PROPERTY)
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.VertexPropertyFeatures.class, feature = Graph.Features.VertexPropertyFeatures.FEATURE_INTEGER_VALUES)
@FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_REMOVE_EDGES)
public void shouldNotHaveAConcurrentModificationExceptionWhenIteratingAndRemovingAddingEdges() {
final Vertex v1 = graph.addVertex("name", "marko");
[12/29] tinkerpop git commit: Merge branch 'tp32'
Posted by ok...@apache.org.
Merge branch 'tp32'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/69af7c2e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/69af7c2e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/69af7c2e
Branch: refs/heads/TINKERPOP-1389
Commit: 69af7c2e1cf6501452ee9f8cd1595fac1c852ee0
Parents: f161583 2d9bb20
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Oct 25 16:14:29 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Oct 25 16:14:29 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/upgrade/release-3.2.x-incubating.asciidoc | 16 ++++++++++++++++
.../apache/tinkerpop/gremlin/structure/Graph.java | 15 +++++++++++++++
3 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/69af7c2e/CHANGELOG.asciidoc
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/69af7c2e/docs/src/upgrade/release-3.2.x-incubating.asciidoc
----------------------------------------------------------------------
[23/29] tinkerpop git commit: had to go back down to Hadoop 2.7.2
because Hadoop 2.7.3 changed a method signature that GiraphJob calls and thus,
Giraph fails. Dar. Cleaned up pom.xmls a bit and updated CHANGELOG. This
branch is ready to be tested on the
Posted by ok...@apache.org.
had to go back down to Hadoop 2.7.2 because Hadoop 2.7.3 changed a method signature that GiraphJob calls and thus, Giraph fails. Dar. Cleaned up pom.xmls a bit and updated CHANGELOG. This branch is ready to be tested on the Blade cluster against the Friendster dataset.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/6a6fa7bb
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/6a6fa7bb
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/6a6fa7bb
Branch: refs/heads/TINKERPOP-1389
Commit: 6a6fa7bb11743edb9a5a5cda3607e7f12af79f27
Parents: 5983a5e
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 13 08:20:22 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:14:21 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +++
giraph-gremlin/pom.xml | 2 +-
hadoop-gremlin/pom.xml | 18 +++++++-----------
pom.xml | 2 +-
spark-gremlin/pom.xml | 2 +-
5 files changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6a6fa7bb/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index afc8b6a..a70cd3a 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -112,6 +112,9 @@ This release also includes changes from <<release-3-1-5, 3.1.5>>.
* Produced better errors in `readGraph` of `GryoReader` and `GraphSONReader` if a `Vertex` cannot be found in the cache on edge loading.
* VertexPrograms can now declare traverser requirements, e.g. to have access to the path when used with `.program()`.
* New build options for `gremlin-python` where `-DglvPython` is no longer required.
+=======
+* Bumped to support Spark 2.0.0.
+>>>>>>> had to go back down to Hadoop 2.7.2 because Hadoop 2.7.3 changed a method signature that GiraphJob calls and thus, Giraph fails. Dar. Cleaned up pom.xmls a bit and updated CHANGELOG. This branch is ready to be tested on the Blade cluster against the Friendster dataset.
* Added missing `InetAddress` to GraphSON extension module.
* Added new recipe for "Pagination".
* Added new recipe for "Recommendation".
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6a6fa7bb/giraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml
index 25d96ec..9a21f36 100644
--- a/giraph-gremlin/pom.xml
+++ b/giraph-gremlin/pom.xml
@@ -235,7 +235,7 @@ limitations under the License.
<configuration>
<archive>
<manifestEntries>
- <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:2.7.2
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:${hadoop.version}
</Gremlin-Plugin-Dependencies>
</manifestEntries>
</archive>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6a6fa7bb/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index b573852..927b2af 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -25,20 +25,16 @@ limitations under the License.
</parent>
<artifactId>hadoop-gremlin</artifactId>
<name>Apache TinkerPop :: Hadoop Gremlin</name>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.0.29.Final</version>
- </dependency>
+ <dependencyManagement>
+ <dependencies>
<dependency>
+ <!-- see: https://github.com/apache/hadoop/pull/84 -->
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
- <version>3.8.0.Final</version>
+ <version>3.7.1.Final</version>
</dependency>
- </dependencies>
- </dependencyManagement>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@@ -180,7 +176,7 @@ limitations under the License.
<configuration>
<archive>
<manifestEntries>
- <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:2.7.2
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:${hadoop.version}
</Gremlin-Plugin-Dependencies>
</manifestEntries>
</archive>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6a6fa7bb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a6b2a92..c2cc89d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -141,7 +141,7 @@ limitations under the License.
<skipIntegrationTests>true</skipIntegrationTests>
<skipPerformanceTests>true</skipPerformanceTests>
<slf4j.version>1.7.21</slf4j.version>
- <hadoop.version>2.7.3</hadoop.version>
+ <hadoop.version>2.7.2</hadoop.version>
<java.tuples.version>1.2</java.tuples.version>
<javadoc-plugin.version>2.10.1</javadoc-plugin.version>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6a6fa7bb/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 02a3a05..7dc8504 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -415,7 +415,7 @@
<configuration>
<archive>
<manifestEntries>
- <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:2.7.3
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:${hadoop.version}
</Gremlin-Plugin-Dependencies>
<!-- deletes the servlet-api jar from the path after install - causes conflicts -->
<Gremlin-Plugin-Paths>servlet-api-2.5.jar=</Gremlin-Plugin-Paths>
[03/29] tinkerpop git commit: added Pick to the io graphson.asciidoc
docs. updated CHANGELOG.
Posted by ok...@apache.org.
added Pick to the io graphson.asciidoc docs. updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/2817c01c
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/2817c01c
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/2817c01c
Branch: refs/heads/TINKERPOP-1389
Commit: 2817c01c1ea1a8fdb17c8f3bd8a9149585926b0d
Parents: fc6ce29
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Oct 25 06:08:00 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Oct 25 06:08:00 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/dev/io/graphson.asciidoc | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2817c01c/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 4dee4c2..d2cd065 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
TinkerPop 3.2.4 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `Pick.none` and `Pick.any` to the serializers and importers.
* Fixed a severe bug where `GraphComputer` strategies are not being loaded until the second use of the traversal source.
[[release-3-2-3]]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2817c01c/docs/src/dev/io/graphson.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/io/graphson.asciidoc b/docs/src/dev/io/graphson.asciidoc
index 511a0fc..5858b95 100644
--- a/docs/src/dev/io/graphson.asciidoc
+++ b/docs/src/dev/io/graphson.asciidoc
@@ -149,6 +149,7 @@ file.withWriter { writer ->
writer.write(toJson(Operator.sum, "Operator"))
writer.write(toJson(Order.incr, "Order"))
writer.write(toJson(Pop.all, "Pop"))
+ writer.write(toJson(Pick.any, "Pick"))
writer.write(toJson(org.apache.tinkerpop.gremlin.util.function.Lambda.function("{ it.get() }"), "Lambda"))
tm = g.V().hasLabel('person').out().out().tree().profile().next()
metrics = new org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics(tm.getMetrics(0));
[09/29] tinkerpop git commit: Merge branch 'tp31' into tp32
Posted by ok...@apache.org.
Merge branch 'tp31' into tp32
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/285ff35e
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/285ff35e
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/285ff35e
Branch: refs/heads/TINKERPOP-1389
Commit: 285ff35e26d01f4a228d936d07e4e047933f0522
Parents: 487b3bf 1f84ad3
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Oct 25 11:34:29 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Oct 25 11:34:29 2016 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 5 +++++
docs/src/upgrade/release-3.1.x-incubating.asciidoc | 7 +++++++
.../gremlin/algorithm/generator/CommunityGeneratorTest.java | 4 ++++
.../algorithm/generator/DistributionGeneratorTest.java | 4 ++++
4 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/285ff35e/CHANGELOG.asciidoc
----------------------------------------------------------------------
[24/29] tinkerpop git commit: Decoupled GryoRecordReader/Writer from
KryoShimService. The shim service should ONLY be for inter-process
communication -- not input/output formats. cc/ @dalaro
Posted by ok...@apache.org.
Decoupled GryoRecordReader/Writer from KryoShimService. The shim service should ONLY be for inter-process communication -- not input/output formats. cc/ @dalaro
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/8367ab36
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/8367ab36
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/8367ab36
Branch: refs/heads/TINKERPOP-1389
Commit: 8367ab36e00a463a27268aafd708ffacec1e606d
Parents: 19e0f2f
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Oct 25 19:44:45 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Oct 26 08:16:26 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphGraphComputer.java | 3 --
.../io/gryo/kryoshim/KryoShimService.java | 4 --
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 17 +++----
.../structure/io/HadoopPoolShimService.java | 47 ++++++--------------
.../hadoop/structure/io/HadoopPools.java | 15 ++++++-
.../structure/io/HadoopPoolsConfigurable.java | 2 +-
.../structure/io/gryo/GryoRecordReader.java | 3 +-
.../structure/io/gryo/GryoRecordWriter.java | 4 +-
.../process/computer/SparkGraphComputer.java | 28 +++++-------
.../SparkHadoopGraphGryoSerializerProvider.java | 4 +-
.../computer/SparkHadoopGraphProvider.java | 6 +--
11 files changed, 53 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
index e21cd3b..1be548a 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -96,9 +96,6 @@ public final class GiraphGraphComputer extends AbstractHadoopGraphComputer imple
this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
- if (null == this.giraphConfiguration.get(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null))
- this.giraphConfiguration.set(KryoShimServiceLoader.KRYO_SHIM_SERVICE, HadoopPoolShimService.class.getCanonicalName());
- System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, this.giraphConfiguration.get(KryoShimServiceLoader.KRYO_SHIM_SERVICE));
this.useWorkerThreadsInConfiguration = this.giraphConfiguration.getInt(GiraphConstants.MAX_WORKERS, -666) != -666 || this.giraphConfiguration.getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), -666) != -666;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
index b8880a4..4d3ece5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
@@ -85,10 +85,6 @@ public interface KryoShimService {
/**
* Attempt to incorporate the supplied configuration in future read/write calls.
- * <p>
- * This method is a wart that exists essentially just to support the old
- * {@link HadoopPools#initialize(Configuration)} use-case.
- * <p>
* This method is not guaranteed to have any effect on an instance of this interface
* after {@link #writeClassAndObject(Object, OutputStream)} or {@link #readClassAndObject(InputStream)}
* has been invoked on that particular instance.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index c026130..f9e4c2e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -35,8 +35,7 @@ import java.util.ServiceLoader;
public class KryoShimServiceLoader {
private static volatile KryoShimService cachedShimService;
-
- private static volatile Configuration conf;
+ private static volatile Configuration configuration;
private static final Logger log = LoggerFactory.getLogger(KryoShimServiceLoader.class);
@@ -47,8 +46,8 @@ public class KryoShimServiceLoader {
*/
public static final String KRYO_SHIM_SERVICE = "gremlin.io.kryoShimService";
- public static void applyConfiguration(final Configuration conf) {
- KryoShimServiceLoader.conf = conf;
+ public static void applyConfiguration(final Configuration configuration) {
+ KryoShimServiceLoader.configuration = configuration;
load(true);
}
@@ -84,7 +83,9 @@ public class KryoShimServiceLoader {
}
}
- String shimClass = System.getProperty(KRYO_SHIM_SERVICE);
+ String shimClass = null != configuration && configuration.containsKey(KRYO_SHIM_SERVICE) ?
+ configuration.getString(KRYO_SHIM_SERVICE) :
+ System.getProperty(KRYO_SHIM_SERVICE);
if (null != shimClass) {
for (KryoShimService kss : services) {
@@ -115,7 +116,7 @@ public class KryoShimServiceLoader {
throw new IllegalStateException("Unable to load KryoShimService");
}
- final Configuration userConf = conf;
+ final Configuration userConf = configuration;
if (null != userConf) {
log.info("Configuring {} provider {} with user-provided configuration",
@@ -136,7 +137,7 @@ public class KryoShimServiceLoader {
}
/**
- * A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#writeClassAndObject(Output, Object)},
+ * A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#writeClassAndObject},
* where the {@code output} parameter is an internally-created {@link ByteArrayOutputStream}. Returns
* the byte array underlying that stream.
*
@@ -154,7 +155,7 @@ public class KryoShimServiceLoader {
}
/**
- * A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#readClassAndObject(Input)},
+ * A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#readClassAndObject},
* where the {@code input} parameter is {@code source}. Returns the deserialized object.
*
* @param source an input stream containing data for a serialized object class and instance
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
index 3fad4fd..a52eac4 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -24,52 +24,33 @@ import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
public class HadoopPoolShimService implements KryoShimService {
- public Object readClassAndObject(final InputStream source) {
-
- Kryo k = null;
-
- try {
- k = HadoopPools.getGryoPool().takeKryo();
-
- return k.readClassAndObject(new Input(source));
- } finally {
- if (null != k) {
- HadoopPools.getGryoPool().offerKryo(k);
- }
- }
+ @Override
+ public Object readClassAndObject(final InputStream inputStream) {
+ return HadoopPools.getGryoPool().readWithKryo(kryo -> kryo.readClassAndObject(new Input(inputStream)));
}
- public void writeClassAndObject(final Object o, final OutputStream sink) {
-
- Kryo k = null;
-
- try {
- k = HadoopPools.getGryoPool().takeKryo();
-
- final Output output = new Output(sink);
-
- k.writeClassAndObject(output, o);
-
+ @Override
+ public void writeClassAndObject(final Object object, final OutputStream outputStream) {
+ HadoopPools.getGryoPool().writeWithKryo(kryo -> {
+ final Output output = new Output(outputStream);
+ kryo.writeClassAndObject(output, object);
output.flush();
- } finally {
- if (null != k) {
- HadoopPools.getGryoPool().offerKryo(k);
- }
- }
+ });
}
@Override
- public int getPriority() {
- return 0;
+ public void applyConfiguration(final Configuration configuration) {
+ HadoopPools.initialize(configuration);
}
@Override
- public void applyConfiguration(final Configuration conf) {
- HadoopPools.initialize(conf);
+ public int getPriority() {
+ return 0;
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
index 392e97d..25bc8b4 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
@@ -31,6 +31,8 @@ import java.util.Collections;
*/
public final class HadoopPools {
+ private static final Configuration EMPTY_CONFIGURATION = new BaseConfiguration();
+
private HadoopPools() {
}
@@ -59,8 +61,17 @@ public final class HadoopPools {
public static GryoPool getGryoPool() {
if (!INITIALIZED) {
- HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using the default pool"); // TODO: this is necessary because we can't get the pool intialized in the Merger code of the Hadoop process.
- initialize(new BaseConfiguration());
+ /*if (null != System.getProperty("configuration", null)) {
+ try {
+ HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using the System properties configuration");
+ initialize((Configuration) Serializer.deserializeObject(System.getProperty("configuration").getBytes()));
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ } else {*/
+ HadoopGraph.LOGGER.warn("The " + HadoopPools.class.getSimpleName() + " has not been initialized, using the default pool");
+ initialize(EMPTY_CONFIGURATION);
+ //}
}
return GRYO_POOL;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
index 0e5f135..f05aae9 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
@@ -30,7 +30,7 @@ public interface HadoopPoolsConfigurable extends Configurable {
@Override
public default void setConf(final Configuration configuration) {
- KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration));
+ HadoopPools.initialize(configuration);
}
@Override
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
index a1daddf..d7ed46b 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
@@ -37,7 +37,6 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.VertexTerminator;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -72,7 +71,7 @@ public final class GryoRecordReader extends RecordReader<NullWritable, VertexWri
final Configuration configuration = context.getConfiguration();
if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null)
this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER);
- KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration));
+ HadoopPools.initialize(configuration);
this.gryoReader = HadoopPools.getGryoPool().takeReader();
long start = split.getStart();
final Path file = split.getPath();
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
index 2ea3394..67a8339 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
@@ -25,10 +25,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -45,7 +43,7 @@ public final class GryoRecordWriter extends RecordWriter<NullWritable, VertexWri
public GryoRecordWriter(final DataOutputStream outputStream, final Configuration configuration) {
this.outputStream = outputStream;
this.hasEdges = configuration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, true);
- KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration));
+ HadoopPools.initialize(configuration);
this.gryoWriter = HadoopPools.getGryoPool().takeWriter();
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index c7d0cfb..d345100 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -110,20 +110,16 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
super(hadoopGraph);
this.sparkConfiguration = new HadoopConfiguration();
ConfigurationUtils.copy(this.hadoopGraph.configuration(), this.sparkConfiguration);
- if (KryoSerializer.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_SERIALIZER, null)) &&
- GryoRegistrator.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_KRYO_REGISTRATOR, null))) {
- System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, UnshadedKryoShimService.class.getCanonicalName());
- } else if (GryoSerializer.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_SERIALIZER, null)) &&
- !this.sparkConfiguration.containsKey(Constants.SPARK_KRYO_REGISTRATOR)) {
- System.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, HadoopPoolShimService.class.getCanonicalName());
- }
- if (null != System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)) {
- final String shimService = System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE);
- this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
- (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
- this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
- (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
- }
+ final String shimService = KryoSerializer.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_SERIALIZER, null)) &&
+ GryoRegistrator.class.getCanonicalName().equals(this.sparkConfiguration.getString(Constants.SPARK_KRYO_REGISTRATOR, null)) ?
+ UnshadedKryoShimService.class.getCanonicalName() :
+ HadoopPoolShimService.class.getCanonicalName();
+ this.sparkConfiguration.setProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, shimService);
+ this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
+ (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim());
+ KryoShimServiceLoader.applyConfiguration(this.sparkConfiguration);
}
@Override
@@ -154,8 +150,8 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
final long startTime = System.currentTimeMillis();
// apache and hadoop configurations that are used throughout the graph computer computation
final org.apache.commons.configuration.Configuration graphComputerConfiguration = new HadoopConfiguration(this.sparkConfiguration);
- if (!graphComputerConfiguration.containsKey(Constants.SPARK_SERIALIZER))
- graphComputerConfiguration.setProperty(Constants.SPARK_SERIALIZER, GryoSerializer.class.getCanonicalName());
+ // TODO !! if (!graphComputerConfiguration.containsKey(Constants.SPARK_SERIALIZER))
+ // graphComputerConfiguration.setProperty(Constants.SPARK_SERIALIZER, GryoSerializer.class.getCanonicalName());
graphComputerConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, this.persist.equals(GraphComputer.Persist.EDGES));
final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(graphComputerConfiguration);
final Storage fileSystemStorage = FileSystemStorage.open(hadoopConfiguration);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
index 9820b7b..19b9121 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoSerializerProvider.java
@@ -34,9 +34,7 @@ import java.util.Map;
public final class SparkHadoopGraphGryoSerializerProvider extends SparkHadoopGraphProvider {
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- if (this.getClass().equals(SparkHadoopGraphGryoSerializerProvider.class) &&
- !HadoopPoolShimService.class.getCanonicalName().equals(System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)))
- Spark.close();
+ Spark.close();
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
config.put(Constants.SPARK_SERIALIZER, GryoSerializer.class.getCanonicalName());
config.remove(Constants.SPARK_KRYO_REGISTRATOR);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8367ab36/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index c5b5083..878fd1e 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -42,9 +42,7 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorageCheck;
import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.util.Map;
@@ -56,9 +54,7 @@ public class SparkHadoopGraphProvider extends HadoopGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- if (this.getClass().equals(SparkHadoopGraphProvider.class) &&
- !UnshadedKryoShimService.class.getCanonicalName().equals(System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)))
- Spark.close();
+ Spark.close();
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast