You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/02 18:57:33 UTC
incubator-tinkerpop git commit: finally made it through the
dependency maze. Spark and Giraph both cleanly work in hadoop-gremlin. Need
to go deeper into the SparkGraphComputer implementation to make sure its
perfect, but I think we have the pom.xml righ
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/spark 531b86a8f -> 0f0b60ba7
finally made it through the dependency maze. Spark and Giraph both cleanly work in hadoop-gremlin. Need to go deeper into the SparkGraphComputer implementation to make sure its perfect, but I think we have the pom.xml right.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/0f0b60ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/0f0b60ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/0f0b60ba
Branch: refs/heads/spark
Commit: 0f0b60ba71d90d5422b83bdf4c8bc38a0bc91a69
Parents: 531b86a
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Mar 2 10:57:40 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Mar 2 10:57:40 2015 -0700
----------------------------------------------------------------------
hadoop-gremlin/pom.xml | 135 ++++++-------------
.../computer/spark/SparkGraphComputer.java | 4 +-
2 files changed, 45 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/0f0b60ba/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 84497c9..21970d8 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -51,20 +51,12 @@ limitations under the License.
<artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-net</groupId>
- <artifactId>commons-net</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>jsp-api</artifactId>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -72,23 +64,7 @@ limitations under the License.
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-parent</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-parent</artifactId>
+ <artifactId>jsp-api-2.1</artifactId>
</exclusion>
</exclusions>
<!--<scope>provided</scope>-->
@@ -99,7 +75,7 @@ limitations under the License.
<artifactId>giraph-core</artifactId>
<version>1.0.0</version>
<exclusions>
- <!-- self-conflict -->
+ <!-- self conflicts -->
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -118,25 +94,14 @@ limitations under the License.
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
+ <!-- spark conflicts -->
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
@@ -150,92 +115,78 @@ limitations under the License.
<artifactId>spark-core_2.10</artifactId>
<version>1.2.1</version>
<exclusions>
+ <!-- self conflicts -->
<exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
</exclusion>
+ <!-- gremlin-core conflicts -->
<exclusion>
<groupId>com.esotericsoftware.kryo</groupId>
<artifactId>kryo</artifactId>
</exclusion>
<exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
</exclusion>
+ <!-- gremlin-groovy conflicts -->
<exclusion>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
+ <!-- hadoop conflicts -->
<exclusion>
- <groupId>commons</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
</exclusion>
+ <!-- giraph conflicts -->
<exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
</exclusion>
</exclusions>
</dependency>
- <!-- consistent dependencies chosen for hadoop-core -->
- <!--<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
-
- </dependency>-->
+ <!-- consistent dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.10.3</version>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.5.13.Final</version>
+ </dependency>
+ <!-- TEST -->
+ <dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-test</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/0f0b60ba/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 0586a14..fc6ad88 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -61,8 +61,8 @@ public class SparkGraphComputer implements GraphComputer {
configuration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX);
configuration.setMaster("local");
final JavaSparkContext sc = new JavaSparkContext(configuration);
- JavaRDD<String> rdd = sc.textFile("religious-traversals.txt");
- System.out.println(rdd.count());
+ JavaRDD<String> rdd = sc.textFile("README.asciidoc");
+ System.out.println("You made it: " + rdd.count());
}