You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2015/11/03 17:48:50 UTC
[03/14] incubator-tinkerpop git commit: merged master/.
merged master/.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/16b50052
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/16b50052
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/16b50052
Branch: refs/heads/TINKERPOP3-909
Commit: 16b50052eb27ebb365341dc3b6e90608b07a71e6
Parents: 3d2d6a6 9300485
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Oct 29 17:29:02 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Oct 29 17:29:02 2015 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 4 +
CONTRIBUTING.asciidoc | 246 +----------------
RELEASE.asciidoc | 172 ------------
docs/src/developer-contributing.asciidoc | 272 +++++++++++++++++++
docs/src/developer-meetings.asciidoc | 64 +++++
docs/src/developer-release.asciidoc | 171 ++++++++++++
docs/src/developer.asciidoc | 31 +++
docs/src/gremlin-applications.asciidoc | 3 +-
docs/src/the-traversal.asciidoc | 25 ++
.../upgrade-release-3.1.x-incubating.asciidoc | 65 ++++-
docs/static/images/gremlin-standing-strong.png | Bin 0 -> 30985 bytes
.../process/computer/GiraphGraphComputer.java | 7 +
.../GephiTraversalVisualizationStrategy.groovy | 2 +-
.../gremlin/process/computer/GraphComputer.java | 16 +-
.../traversal/TraversalVertexProgram.java | 4 +-
.../traversal/dsl/graph/GraphTraversal.java | 7 +-
.../dsl/graph/GraphTraversalSource.java | 10 +-
.../gremlin/process/traversal/dsl/graph/__.java | 9 +-
.../process/traversal/step/map/GraphStep.java | 149 ++++++++++
.../traversal/step/sideEffect/GraphStep.java | 101 -------
.../step/util/TraversalComparator.java | 5 +-
.../strategy/decoration/ConnectiveStrategy.java | 3 +-
.../strategy/decoration/ElementIdStrategy.java | 2 +-
.../strategy/decoration/PartitionStrategy.java | 2 +-
.../strategy/decoration/SubgraphStrategy.java | 2 +-
.../finalization/LazyBarrierStrategy.java | 2 +-
.../ComputerVerificationStrategy.java | 39 ++-
.../process/traversal/util/TraversalHelper.java | 3 +
.../traversal/step/map/GraphStepTest.java | 41 +++
.../ElementIdStrategyTraverseTest.java | 4 +-
.../PartitionStrategyTraverseTest.java | 4 +-
.../SubgraphStrategyTraverseTest.java | 5 +-
.../traversal/step/map/GroovyGraphTest.groovy | 50 ++++
.../traversal/step/map/GroovyOrderTest.groovy | 10 +
.../process/GroovyProcessComputerSuite.java | 2 +
.../process/GroovyProcessStandardSuite.java | 2 +
.../handler/NioGremlinBinaryRequestDecoder.java | 5 +
.../handler/WsGremlinBinaryRequestDecoder.java | 5 +
.../handler/WsGremlinCloseRequestDecoder.java | 5 +
.../handler/WsGremlinTextRequestDecoder.java | 5 +
.../server/GremlinDriverIntegrateTest.java | 72 ++++-
.../server/GremlinServerIntegrateTest.java | 100 +++++++
.../gremlin/process/ProcessComputerSuite.java | 2 +
.../gremlin/process/ProcessStandardSuite.java | 2 +
.../process/computer/GraphComputerTest.java | 5 +
.../traversal/step/map/AddVertexTest.java | 2 +-
.../process/traversal/step/map/GraphTest.java | 109 ++++++++
.../process/traversal/step/map/OrderTest.java | 58 +++-
.../step/sideEffect/Neo4jGraphStep.java | 10 +-
.../optimization/Neo4jGraphStepStrategy.java | 13 +-
.../gremlin/neo4j/structure/Neo4jGraph.java | 1 +
.../traversal/strategy/Neo4jStrategySuite.java | 44 +++
.../traversal/strategy/Neo4jStrategyTest.java | 32 +++
.../Neo4jGraphStepStrategyTest.java | 76 ++++++
pom.xml | 34 ++-
.../process/computer/SparkGraphComputer.java | 16 +-
.../io/SparkContextPersistenceTest.java | 14 +-
.../process/computer/TinkerGraphComputer.java | 5 +
.../step/sideEffect/TinkerGraphStep.java | 10 +-
.../optimization/TinkerGraphStepStrategy.java | 13 +-
.../tinkergraph/structure/TinkerFactory.java | 8 +-
.../tinkergraph/structure/TinkerGraph.java | 59 +++-
.../tinkergraph/TinkerGraphProvider.java | 14 +-
.../strategy/TinkerGraphStrategySuite.java | 44 +++
.../strategy/TinkerGraphStrategyTest.java | 32 +++
.../TinkerGraphStepStrategyTest.java | 33 ++-
.../structure/TinkerGraphIdManagerTest.java | 18 +-
.../structure/TinkerGraphPlayTest.java | 11 +
.../tinkergraph/structure/TinkerGraphTest.java | 16 +-
69 files changed, 1751 insertions(+), 656 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/16b50052/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 7134f26,c3593c1..ef2ae6f
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@@ -80,20 -88,17 +90,20 @@@ public final class SparkGraphComputer e
public Future<ComputerResult> submit() {
super.validateStatePriorToExecution();
// apache and hadoop configurations that are used throughout the graph computer computation
- final org.apache.commons.configuration.Configuration apacheConfiguration = new HadoopConfiguration(this.hadoopGraph.configuration());
- this.sparkConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(GraphComputer.Persist.EDGES));
- final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(this.sparkConfiguration);
- if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
- try {
- final String inputLocation = FileSystem.get(hadoopConfiguration).getFileStatus(new Path(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION))).getPath().toString();
- this.sparkConfiguration.setProperty(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, inputLocation);
- hadoopConfiguration.set(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, inputLocation);
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
++ final org.apache.commons.configuration.Configuration apacheConfiguration = new HadoopConfiguration(this.sparkConfiguration);
+ apacheConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(GraphComputer.Persist.EDGES));
+ final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(apacheConfiguration);
+ if (null == hadoopConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD_NAME, null)) {
+ if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
+ try {
+ final String inputLocation = FileSystem.get(hadoopConfiguration).getFileStatus(new Path(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION))).getPath().toString();
+ apacheConfiguration.setProperty(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, inputLocation);
+ hadoopConfiguration.set(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, inputLocation);
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
}
- }
+ } // else WARN that both an INPUT_FORMAT and INPUT_RDD_NAME were provided?
// create the completable future
return CompletableFuture.<ComputerResult>supplyAsync(() -> {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/16b50052/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/SparkContextPersistenceTest.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/SparkContextPersistenceTest.java
index e19d89f,0000000..cc0957f
mode 100644,000000..100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/SparkContextPersistenceTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/SparkContextPersistenceTest.java
@@@ -1,126 -1,0 +1,126 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.process.computer.util.SparkHelper;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
- import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.junit.Test;
+
- import static org.junit.Assert.*;
++import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkContextPersistenceTest {
+
+ @Test
+ public void shouldPersistRDDAcrossJobs() throws Exception {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", "org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer");
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, NullOutputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD_NAME, "a-random-name-for-testing");
+ configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
+ Graph graph = GraphFactory.open(configuration);
+ graph.compute(SparkGraphComputer.class)
+ .result(GraphComputer.ResultGraph.NEW)
+ .persist(GraphComputer.Persist.NOTHING)
+ .program(TraversalVertexProgram.build()
+ .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)),
+ "gremlin-groovy",
+ "g.V()").create(graph)).submit().get();
+ ////////
+ SparkConf sparkConfiguration = new SparkConf();
+ sparkConfiguration.setAppName("shouldPersistRDDAcrossJobs");
+ ConfUtil.makeHadoopConfiguration(configuration).forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
+ JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ assertTrue(SparkHelper.getPersistedRDD(sparkContext, "a-random-name-for-testing").isPresent());
+ ///////
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD_NAME, "a-random-name-for-testing");
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD_NAME, null);
+ graph = GraphFactory.open(configuration);
+ graph.compute(SparkGraphComputer.class)
+ .result(GraphComputer.ResultGraph.NEW)
+ .persist(GraphComputer.Persist.NOTHING)
+ .program(TraversalVertexProgram.build()
+ .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)),
+ "gremlin-groovy",
+ "g.V()").create(graph)).submit().get();
+ }
+
+ @Test
+ public void testBulkLoaderVertexProgramChain() throws Exception {
+ final Configuration readConfiguration = new BaseConfiguration();
+ readConfiguration.setProperty("spark.master", "local[4]");
+ readConfiguration.setProperty("spark.serializer", "org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer");
+ readConfiguration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
+ readConfiguration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, NullOutputFormat.class.getCanonicalName());
+ readConfiguration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "target/test-output");
+ readConfiguration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ readConfiguration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD_NAME, "a-random-name-for-testing");
+ readConfiguration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
+ Graph graph = GraphFactory.open(readConfiguration);
+
+ ///////////////
+ final Configuration writeConfiguration = new BaseConfiguration();
+ writeConfiguration.setProperty(Graph.GRAPH, "org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph");
- writeConfiguration.setProperty(TinkerGraph.CONFIG_GRAPH_FORMAT, "gryo");
- writeConfiguration.setProperty(TinkerGraph.CONFIG_GRAPH_LOCATION, "target/test-output/tinkergraph.kryo");
++ writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
++ writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, "target/test-output/tinkergraph.kryo");
+ final Graph secondGraph = graph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.NOTHING).program(PageRankVertexProgram.build().create(graph)).submit().get().graph();
- secondGraph.configuration().setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD_NAME, "a-random-name-for-testing");
- secondGraph.configuration().setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD_NAME, null);
+ secondGraph.compute(SparkGraphComputer.class)
+ .persist(GraphComputer.Persist.NOTHING)
+ .workers(1)
++ .config(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD_NAME, "a-random-name-for-testing")
++ .config(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD_NAME, null)
+ .program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph(writeConfiguration).create(secondGraph))
+ .submit().get();
+ final Graph finalGraph = TinkerGraph.open();
+ finalGraph.io(IoCore.gryo()).readGraph("target/test-output/tinkergraph.kryo");
- assertEquals(6l,finalGraph.traversal().V().count().next().longValue());
++ assertEquals(6l, finalGraph.traversal().V().count().next().longValue());
+ }
+}