You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/11/05 02:56:50 UTC
[32/50] [abbrv] incubator-tinkerpop git commit: Merge branch 'master'
into TINKERPOP3-909
Merge branch 'master' into TINKERPOP3-909
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/558c04e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/558c04e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/558c04e7
Branch: refs/heads/TINKERPOP3-923
Commit: 558c04e7f3d51b12ddd06cd11fe4bbea5298843d
Parents: af4f5dc 06d747f
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Nov 3 17:47:37 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Nov 3 17:47:37 2015 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +-
docs/src/implementations.asciidoc | 36 +--
docs/src/the-traversal.asciidoc | 11 +-
.../upgrade-release-3.1.x-incubating.asciidoc | 18 +-
.../process/computer/GiraphGraphComputer.java | 11 +-
.../computer/io/GiraphVertexInputFormat.java | 70 ------
.../computer/io/GiraphVertexOutputFormat.java | 65 ------
.../process/computer/io/GiraphVertexReader.java | 67 ------
.../process/computer/io/GiraphVertexWriter.java | 57 -----
.../structure/io/GiraphVertexInputFormat.java | 70 ++++++
.../structure/io/GiraphVertexOutputFormat.java | 65 ++++++
.../giraph/structure/io/GiraphVertexReader.java | 67 ++++++
.../giraph/structure/io/GiraphVertexWriter.java | 57 +++++
.../gremlin/process/computer/GraphComputer.java | 5 +-
.../process/computer/GraphComputerTest.java | 2 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 1 -
.../structure/hdfs/HadoopElementIterator.java | 4 +-
.../hadoop/structure/io/InputOutputHelper.java | 22 ++
.../hadoop/structure/util/HadoopHelper.java | 50 -----
.../process/computer/SparkGraphComputer.java | 75 ++++---
.../process/computer/io/InputFormatRDD.java | 47 ----
.../spark/process/computer/io/InputRDD.java | 41 ----
.../process/computer/io/OutputFormatRDD.java | 49 -----
.../spark/process/computer/io/OutputRDD.java | 31 ---
.../spark/structure/io/InputFormatRDD.java | 47 ++++
.../spark/structure/io/InputOutputHelper.java | 81 +++++++
.../gremlin/spark/structure/io/InputRDD.java | 41 ++++
.../spark/structure/io/OutputFormatRDD.java | 49 +++++
.../gremlin/spark/structure/io/OutputRDD.java | 31 +++
.../spark/structure/io/PersistedInputRDD.java | 60 +++++
.../spark/structure/io/PersistedOutputRDD.java | 41 ++++
.../process/computer/io/ExampleInputRDD.java | 47 ----
.../process/computer/io/ExampleOutputRDD.java | 45 ----
.../process/computer/io/InputOutputRDDTest.java | 59 -----
.../spark/process/computer/io/InputRDDTest.java | 54 -----
.../process/computer/io/OutputRDDTest.java | 62 ------
.../spark/structure/io/ExampleInputRDD.java | 51 +++++
.../spark/structure/io/ExampleOutputRDD.java | 50 +++++
.../spark/structure/io/InputOutputRDDTest.java | 59 +++++
.../spark/structure/io/InputRDDTest.java | 54 +++++
.../spark/structure/io/OutputRDDTest.java | 62 ++++++
.../io/PersistedInputOutputRDDTest.java | 217 +++++++++++++++++++
.../process/computer/TinkerGraphComputer.java | 5 -
43 files changed, 1228 insertions(+), 811 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 84cb9f5,e79a0a0..5ad89e5
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -25,10 -25,10 +25,11 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.1.0 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Integrated `NumberHelper` in `SumStep`, `MinStep`, `MaxStep` and `MeanStep` (local and global step variants).
* Bumped to Neo4j 2.3.0.
+ * Added `PersistedInputRDD` and `PersistedOutputRDD` which enables `SparkGraphComputer` to store the graph RDD in the context between jobs (no HDFS serialization required).
* Renamed the `public static String` configuration variable names of TinkerGraph (deprecated old variables).
- * Added `GraphComputer.config(key,value)` to allow engine-specific configurations.
+ * Added `GraphComputer.configure(key,value)` to allow engine-specific configurations.
* `GraphStep` is no longer in the `sideEffect`-package and is now in `map`-package (breaking change).
* Added support for mid-traversal `V()`-steps (`GraphStep` semantics updated).
* Fixed `Number` handling in `Operator` enums. Prior this change a lot of operations on mixed `Number` types returned a wrong result (wrong data type).
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/docs/src/upgrade-release-3.1.x-incubating.asciidoc
----------------------------------------------------------------------
diff --cc docs/src/upgrade-release-3.1.x-incubating.asciidoc
index 00be151,9e5b50d..ada54f9
--- a/docs/src/upgrade-release-3.1.x-incubating.asciidoc
+++ b/docs/src/upgrade-release-3.1.x-incubating.asciidoc
@@@ -165,3 -172,11 +173,11 @@@ The `VendorOptimizationStrategy` has be
with revised terminology for what were formerly referred to as "vendors".
See link:https://issues.apache.org/jira/browse/TINKERPOP3-876[TINKERPOP3-876] for more information.
+
+ GraphComputer Updates
+ +++++++++++++++++++++
+
+ `GraphComputer.configure(String key, Object value)` is now a method (with default implementation).
+ This allows the user to specify engine-specific parameters to the underlying OLAP system. These parameters are not intended
+ to be cross engine supported. Moreover, if there are not parameters that can be altered (beyond the standard `GraphComputer`
-methods), then the provider's `GraphComputer` implementation should simply return and do nothing.
++methods), then the provider's `GraphComputer` implementation should simply return and do nothing.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 0000000,8d6c06e..e8bd44a
mode 000000,100644..100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@@ -1,0 -1,54 +1,54 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+ import org.apache.commons.configuration.BaseConfiguration;
+ import org.apache.commons.configuration.Configuration;
+ import org.apache.tinkerpop.gremlin.hadoop.Constants;
+ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+ import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+ import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+ import org.apache.tinkerpop.gremlin.structure.Graph;
+ import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+ import org.junit.Test;
+
+ import static org.junit.Assert.assertEquals;
+
+ /**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+ public class InputRDDTest {
+
+ @Test
+ public void shouldReadFromArbitraryRDD() {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "target/test-output");
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ ////////
+ Graph graph = GraphFactory.open(configuration);
- assertEquals(Double.valueOf(123.0d), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
++ assertEquals(123l, graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
+ assertEquals(Long.valueOf(4l), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().count().next());
+ }
+ }