You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2016/01/21 13:42:58 UTC
[1/2] incubator-tinkerpop git commit: added helper script that
distributes spark-gremlin jars across all servers in a Spark cluster
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 1a8e799d3 -> 18f5911e5
added helper script that distributes spark-gremlin jars across all servers in a Spark cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a9a5a5bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a9a5a5bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a9a5a5bb
Branch: refs/heads/master
Commit: a9a5a5bbc8acff745a1f82870e2cf0070af60cf8
Parents: b44253d
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Thu Jan 21 13:42:07 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Thu Jan 21 13:42:07 2016 +0100
----------------------------------------------------------------------
bin/init-tp-spark.sh | 75 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 75 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a9a5a5bb/bin/init-tp-spark.sh
----------------------------------------------------------------------
diff --git a/bin/init-tp-spark.sh b/bin/init-tp-spark.sh
new file mode 100755
index 0000000..7b6e5ba
--- /dev/null
+++ b/bin/init-tp-spark.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z $2 ]; then
+ echo
+ echo "Usage:"
+ echo
+ echo " $0 SPARK_HOME SPARK_NODE_1 SPARK_NODE_2 ..."
+ echo
+ echo "Example:"
+ echo
+ echo " $0 /usr/local/spark-1.5.2-bin-hadoop2.6 spark@master spark@slave1 spark@slave2 spark@slave3"
+ echo
+ exit 1
+fi
+
+if [ -z ${HADOOP_GREMLIN_LIBS} ]; then
+ echo
+ echo "HADOOP_GREMLIN_LIBS is not set; thus no jars to deploy."
+ echo
+ exit 1
+fi
+
+LOCAL_SPARK_GREMLIN_LIBS=$(tr ':' $'\n' <<< ${HADOOP_GREMLIN_LIBS} | grep 'spark-gremlin')
+LIB_DIRNAME="hadoop-gremlin-libs"
+TMP_DIR=/tmp/init-tp-spark
+TMP_DIR2=${TMP_DIR}/${LIB_DIRNAME}
+
+mkdir -p ${TMP_DIR2}
+cp -R ${LOCAL_SPARK_GREMLIN_LIBS}/*.jar ${TMP_DIR2}
+
+DIR=`dirname $0`
+
+SPARK_HOME=${1}
+SPARK_LIBS=${SPARK_HOME}/lib
+SPARK_NODES=${@:2}
+
+cat > ${TMP_DIR}/init-conf.sh <<EOF
+#!/bin/bash
+
+if [ ! -f \"${SPARK_HOME}/conf/spark-env.sh\" ]; then
+ cp ${SPARK_HOME}/conf/spark-env.sh.template ${SPARK_HOME}/conf/spark-env.sh
+fi
+grep -F GREMLIN_LIBS ${SPARK_HOME}/conf/spark-env.sh > /dev/null || {
+ echo >> ${SPARK_HOME}/conf/spark-env.sh
+ echo "GREMLIN_LIBS=\\\$(find ${SPARK_LIBS}/${LIB_DIRNAME} -name '*.jar' | paste -sd ':')" >> ${SPARK_HOME}/conf/spark-env.sh
+ echo "export SPARK_CLASSPATH=\\\${SPARK_CLASSPATH}:\\\${GREMLIN_LIBS}" >> ${SPARK_HOME}/conf/spark-env.sh
+}
+EOF
+
+for node in ${SPARK_NODES}
+do
+ rsync -az --delete ${TMP_DIR2} ${node}:${SPARK_LIBS}
+ ssh ${node} "bash -s" -- < ${TMP_DIR}/init-conf.sh
+done
+
+rm -rf ${TMP_DIR}
[2/2] incubator-tinkerpop git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-tinkerpop
Posted by dk...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-tinkerpop
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/18f5911e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/18f5911e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/18f5911e
Branch: refs/heads/master
Commit: 18f5911e5e22a2462178ceb0b8dfe776f13f93ab
Parents: a9a5a5b 1a8e799
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Thu Jan 21 13:42:41 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Thu Jan 21 13:42:41 2016 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 17 +
bin/process-docs.sh | 26 +-
docs/src/dev/developer/contributing.asciidoc | 10 +-
.../src/reference/gremlin-applications.asciidoc | 26 +-
docs/src/reference/implementations.asciidoc | 129 ++++---
docs/src/reference/the-graphcomputer.asciidoc | 9 +
docs/src/reference/the-traversal.asciidoc | 12 +-
.../tutorials/getting-started/index.asciidoc | 2 +
.../upgrade/release-3.1.x-incubating.asciidoc | 60 ++-
docs/static/images/gremlin-gym.png | Bin 273111 -> 694710 bytes
.../images/modern-edge-1-to-3-4-gremlin.png | Bin 0 -> 18042 bytes
.../process/computer/GiraphGraphComputer.java | 43 +--
.../GiraphHadoopGremlinIntegrateTest.java | 33 ++
.../computer/GiraphHadoopGraphProvider.java | 6 +-
.../GiraphHadoopGremlinPluginIntegrateTest.java | 33 --
.../bulkdumping/BulkDumperVertexProgram.java | 98 +++++
.../bulkloading/IncrementalBulkLoader.java | 2 +-
.../peerpressure/ClusterCountMapReduce.java | 7 +-
.../process/traversal/SackFunctions.java | 10 +-
.../dsl/graph/DefaultGraphTraversal.java | 14 -
.../traversal/dsl/graph/GraphTraversal.java | 36 +-
.../gremlin/process/traversal/dsl/graph/__.java | 363 +++++++++++++++----
.../lambda/AbstractLambdaTraversal.java | 33 +-
.../traversal/lambda/ConstantTraversal.java | 1 -
.../traversal/lambda/IdentityTraversal.java | 15 +-
.../process/traversal/lambda/LoopTraversal.java | 2 +-
.../process/traversal/lambda/TrueTraversal.java | 19 +-
.../traversal/step/map/SampleLocalStep.java | 68 ++--
.../process/traversal/step/util/EmptyStep.java | 2 +-
.../traverser/TraverserRequirement.java | 3 +-
.../traversal/util/DefaultTraversal.java | 17 +-
.../traversal/util/DependantMutableMetrics.java | 33 +-
.../process/traversal/util/EmptyTraversal.java | 2 +-
.../process/traversal/util/MutableMetrics.java | 3 +-
.../process/traversal/util/TraversalRing.java | 2 +-
.../tinkerpop/gremlin/structure/Graph.java | 15 +-
.../tinkerpop/gremlin/structure/io/Io.java | 8 +-
.../tinkerpop/gremlin/structure/io/Storage.java | 139 +++++++
.../structure/io/graphson/GraphSONReader.java | 4 +-
.../gremlin/structure/io/gryo/GryoMapper.java | 30 +-
.../gremlin/structure/io/gryo/GryoReader.java | 2 +-
.../structure/io/gryo/JavaTimeSerializers.java | 303 ++++++++++++++++
.../structure/util/AbstractTransaction.java | 8 +-
.../gremlin/structure/util/ElementHelper.java | 5 +-
.../gremlin/structure/util/StringFactory.java | 18 +-
.../step/filter/SampleGlobalStepTest.java | 22 ++
.../traversal/step/map/SampleLocalStepTest.java | 21 ++
.../structure/io/gryo/GryoMapperTest.java | 109 ++++++
.../step/sideEffect/GroovySackTest.groovy | 11 +-
.../process/GroovyProcessComputerSuite.java | 2 +
.../customizer/SimpleSandboxExtension.groovy | 5 +-
.../AbstractImportCustomizerProvider.java | 2 +
.../tinkerpop/gremlin/server/GraphManager.java | 14 +-
.../gremlin/server/channel/NioChannelizer.java | 11 +-
.../server/channel/WebSocketChannelizer.java | 12 +-
.../tinkerpop/gremlin/server/handler/Frame.java | 37 ++
.../handler/GremlinResponseFrameEncoder.java | 102 ++++++
.../gremlin/server/handler/IteratorHandler.java | 2 +
.../handler/NioGremlinResponseEncoder.java | 4 +
.../handler/NioGremlinResponseFrameEncoder.java | 47 +++
.../server/handler/OpExecutorHandler.java | 9 +
.../server/handler/OpSelectorHandler.java | 10 +-
.../handler/WsGremlinResponseEncoder.java | 6 +-
.../handler/WsGremlinResponseFrameEncoder.java | 44 +++
.../server/op/AbstractEvalOpProcessor.java | 128 +++++--
.../server/op/standard/StandardOpProcessor.java | 6 +-
.../server/GremlinDriverIntegrateTest.java | 12 +-
.../server/GremlinServerIntegrateTest.java | 30 ++
.../apache/tinkerpop/gremlin/GraphProvider.java | 3 +-
.../gremlin/process/GremlinProcessRunner.java | 2 +-
.../gremlin/process/ProcessComputerSuite.java | 2 +
.../process/computer/GraphComputerTest.java | 54 +--
.../BulkDumperVertexProgramTest.java | 104 ++++++
.../traversal/step/sideEffect/SackTest.java | 26 +-
.../ReadOnlyStrategyProcessTest.java | 3 +-
.../gremlin/structure/FeatureSupportTest.java | 2 +-
.../tinkerpop/gremlin/structure/io/IoTest.java | 82 +++++
.../gremlin/structure/io/IoVertexTest.java | 2 +-
.../conf/hadoop-grateful-gryo.properties | 6 +-
hadoop-gremlin/conf/hadoop-gryo.properties | 2 +
.../hadoop/groovy/plugin/HadoopLoader.groovy | 138 -------
.../tinkerpop/gremlin/hadoop/Constants.java | 22 ++
.../groovy/plugin/HadoopGremlinPlugin.java | 11 +-
.../process/computer/util/MapReduceHelper.java | 8 +-
.../hadoop/structure/HadoopConfiguration.java | 11 +-
.../gremlin/hadoop/structure/HadoopGraph.java | 4 +-
.../hadoop/structure/hdfs/HDFSTools.java | 113 ------
.../structure/hdfs/HadoopEdgeIterator.java | 83 -----
.../structure/hdfs/HadoopElementIterator.java | 74 ----
.../structure/hdfs/HadoopVertexIterator.java | 82 -----
.../hadoop/structure/hdfs/HiddenFileFilter.java | 44 ---
.../hadoop/structure/hdfs/TextIterator.java | 91 -----
.../hadoop/structure/io/FileSystemStorage.java | 246 +++++++++++++
.../hadoop/structure/io/HadoopEdgeIterator.java | 79 ++++
.../structure/io/HadoopElementIterator.java | 75 ++++
.../structure/io/HadoopVertexIterator.java | 78 ++++
.../hadoop/structure/io/HiddenFileFilter.java | 44 +++
.../hadoop/structure/io/InputOutputHelper.java | 6 +-
.../hadoop/structure/io/ObjectWritable.java | 7 +-
.../structure/io/ObjectWritableIterator.java | 12 +-
.../hadoop/structure/io/TextIterator.java | 91 +++++
.../structure/io/VertexWritableIterator.java | 10 +-
.../gremlin/hadoop/HadoopGremlinSuite.java | 36 ++
.../groovy/plugin/HadoopGremlinPluginCheck.java | 71 +---
.../hadoop/groovy/plugin/HadoopPluginSuite.java | 34 --
.../structure/io/AbstractStorageCheck.java | 145 ++++++++
.../structure/io/FileSystemStorageCheck.java | 88 +++++
.../neo4j/AbstractNeo4jGraphProvider.java | 4 +-
.../neo4j/MultiMetaNeo4jGraphProvider.java | 1 +
.../neo4j/NoMultiNoMetaNeo4jGraphProvider.java | 1 +
pom.xml | 7 +-
spark-gremlin/pom.xml | 2 +-
.../spark/groovy/plugin/SparkLoader.groovy | 68 ----
.../spark/groovy/plugin/SparkGremlinPlugin.java | 5 +-
.../spark/process/computer/SparkExecutor.java | 28 --
.../process/computer/SparkGraphComputer.java | 43 ++-
.../gremlin/spark/structure/Spark.java | 5 +-
.../spark/structure/io/InputFormatRDD.java | 15 +
.../spark/structure/io/InputOutputHelper.java | 3 -
.../gremlin/spark/structure/io/InputRDD.java | 21 +-
.../spark/structure/io/OutputFormatRDD.java | 30 +-
.../gremlin/spark/structure/io/OutputRDD.java | 25 ++
.../spark/structure/io/PersistedInputRDD.java | 14 +-
.../spark/structure/io/PersistedOutputRDD.java | 29 +-
.../spark/structure/io/SparkContextStorage.java | 164 +++++++++
.../spark/structure/io/gryo/GryoSerializer.java | 3 +
.../gremlin/spark/AbstractSparkTest.java | 30 ++
.../gremlin/spark/SparkGremlinSuite.java | 35 ++
.../gremlin/spark/SparkGremlinTest.java | 33 ++
.../gremlin/spark/SparkHadoopGremlinTest.java | 33 ++
.../process/computer/LocalPropertyTest.java | 2 +-
.../computer/SparkHadoopGraphProvider.java | 12 +-
.../groovy/plugin/SparkGremlinPluginTest.java | 126 -------
.../plugin/SparkHadoopGremlinPluginTest.java | 33 --
.../gremlin/spark/structure/SparkTest.java | 10 +-
.../io/PersistedInputOutputRDDTest.java | 72 +++-
.../structure/io/SparkContextStorageCheck.java | 74 ++++
.../tinkergraph/structure/TinkerIoRegistry.java | 1 +
.../tinkergraph/structure/TinkerGraphTest.java | 129 ++++++-
139 files changed, 3813 insertions(+), 1550 deletions(-)
----------------------------------------------------------------------