You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/09/09 22:20:36 UTC
[06/18] incubator-tinkerpop git commit: more work trying to get
hadoop_split dependencies working. this is a nest to say the least.
more work trying to get hadoop_split dependencies working. this is a nest to say the least.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b2132b1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b2132b1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b2132b1f
Branch: refs/heads/master
Commit: b2132b1fae9bae418db06fff3037f44a8e8b8574
Parents: 718dd63
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Sep 3 14:57:11 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Sep 3 14:57:11 2015 -0600
----------------------------------------------------------------------
spark-gremlin/pom.xml | 65 ++++++++------------
.../spark/process/HadoopGraphProvider.java | 11 ----
2 files changed, 26 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2132b1f/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 277ce11..645c97f 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -44,34 +44,39 @@
<groupId>org.apache.tinkerpop</groupId>
<artifactId>hadoop-gremlin</artifactId>
<version>${project.version}</version>
- </dependency>
- <!-- SPARK GRAPH COMPUTER -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
- <version>1.2.1</version>
<exclusions>
- <!-- self conflicts -->
<exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <groupId>org.apache.giraph</groupId>
+ <artifactId>giraph-core</artifactId>
</exclusion>
<exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
</exclusion>
+ </exclusions>
+ </dependency>
+ <!-- SPARK GRAPH COMPUTER -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>1.2.1</version>
+ <exclusions>
<!-- gremlin-core conflicts -->
<exclusion>
<groupId>org.slf4j</groupId>
@@ -90,35 +95,17 @@
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
- <!-- hadoop conflicts -->
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- <!-- giraph conflicts -->
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
<!-- lgpl conflicts -->
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>findbugs</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
</exclusions>
</dependency>
- <!-- consistent dependencies -->
-<!-- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.10.3</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.5.13.Final</version>
- </dependency>
--->
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2132b1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
index f085678..333636e 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
@@ -20,7 +20,6 @@
package org.apache.tinkerpop.gremlin.spark.process;
import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.TestHelper;
@@ -110,16 +109,6 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- /// giraph configuration
- put(GiraphConstants.MIN_WORKERS, 1);
- put(GiraphConstants.MAX_WORKERS, 1);
- put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
- put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
- put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
- put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
put("mapred.reduce.tasks", 4);
//put("giraph.vertexOutputFormatThreadSafe", false);
//put("giraph.numOutputThreads", 3);