You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2013/12/17 07:42:30 UTC
[1/7] git commit: Removing extra code in new yarn
Updated Branches:
refs/heads/master 964a3b697 -> 7a8169be9
Removing extra code in new yarn
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/24f8220d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/24f8220d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/24f8220d
Branch: refs/heads/master
Commit: 24f8220dc8a0671df7ee32b3eb193120ca316878
Parents: ceb013f
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Dec 16 13:04:07 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 21:53:51 2013 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/24f8220d/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
----------------------------------------------------------------------
diff --git a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
index 11da1c4..ac519fc 100644
--- a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
+++ b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
@@ -59,7 +59,6 @@ class WorkerLauncher(args: ApplicationMasterArguments, conf: Configuration) exte
override def preStart() {
logInfo("Listen to driver: " + driverUrl)
driver = context.actorFor(driverUrl)
- driver ! "hello"
context.system.eventStream.subscribe(self, classOf[RemotingLifecycleEvent])
}
[3/7] git commit: Attempt with extra repositories
Posted by pw...@apache.org.
Attempt with extra repositories
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/c6f95e60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/c6f95e60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/c6f95e60
Branch: refs/heads/master
Commit: c6f95e603e2c58b2fab6255c824a839807f03026
Parents: 964a3b6
Author: Patrick Wendell <pw...@gmail.com>
Authored: Sun Dec 15 20:30:21 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 21:53:51 2013 -0800
----------------------------------------------------------------------
README.md | 6 +-
core/pom.xml | 9 +--
new-yarn/pom.xml | 6 +-
.../spark/deploy/yarn/WorkerLauncher.scala | 10 +--
.../deploy/yarn/YarnAllocationHandler.scala | 2 +-
pom.xml | 76 +++++++++++---------
project/SparkBuild.scala | 32 +++------
7 files changed, 65 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 80bbe31..1550a8b 100644
--- a/README.md
+++ b/README.md
@@ -54,7 +54,7 @@ versions without YARN, use:
# Cloudera CDH 4.2.0 with MapReduce v1
$ SPARK_HADOOP_VERSION=2.0.0-mr1-cdh4.2.0 sbt/sbt assembly
-For Apache Hadoop 2.0.X, 2.1.X, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions
+For Apache Hadoop 2.2.X, 2.1.X, 2.0.X, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions
with YARN, also set `SPARK_YARN=true`:
# Apache Hadoop 2.0.5-alpha
@@ -63,10 +63,8 @@ with YARN, also set `SPARK_YARN=true`:
# Cloudera CDH 4.2.0 with MapReduce v2
$ SPARK_HADOOP_VERSION=2.0.0-cdh4.2.0 SPARK_YARN=true sbt/sbt assembly
-When building for Hadoop 2.2.X and newer, you'll need to include the additional `new-yarn` profile:
-
# Apache Hadoop 2.2.X and newer
- $ mvn -Dyarn.version=2.2.0 -Dhadoop.version=2.2.0 -Pnew-yarn
+ $ SPARK_HADOOP_VERSION=2.2.0 SPARK_YARN=true sbt/sbt assembly
When developing a Spark application, specify the Hadoop version by adding the
"hadoop-client" artifact to your project's dependencies. For example, if you're
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index cdbaa52..b83a2a8 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -81,10 +81,6 @@
<artifactId>asm</artifactId>
</dependency>
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
<version>0.3.1</version>
@@ -96,15 +92,12 @@
</dependency>
<dependency>
<groupId>${akka.group}</groupId>
- <artifactId>akka-actor_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>${akka.group}</groupId>
<artifactId>akka-remote_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>${akka.group}</groupId>
<artifactId>akka-slf4j_${scala.binary.version}</artifactId>
+ <groupId>org.spark-project.akka</groupId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/new-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/new-yarn/pom.xml b/new-yarn/pom.xml
index 8a065c6..4cd28f3 100644
--- a/new-yarn/pom.xml
+++ b/new-yarn/pom.xml
@@ -25,7 +25,7 @@
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_2.9.3</artifactId>
+ <artifactId>spark-yarn_2.10</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN Support</name>
<url>http://spark.incubator.apache.org/</url>
@@ -33,7 +33,7 @@
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.9.3</artifactId>
+ <artifactId>spark-core_2.10</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -63,7 +63,7 @@
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
- <artifactId>scalatest_2.9.3</artifactId>
+ <artifactId>scalatest_2.10</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
----------------------------------------------------------------------
diff --git a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
index c38f33e..11da1c4 100644
--- a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
+++ b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.api.protocolrecords._
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import akka.actor._
-import akka.remote.{RemoteClientShutdown, RemoteClientDisconnected, RemoteClientLifeCycleEvent}
+import akka.remote._
import akka.actor.Terminated
import org.apache.spark.{SparkContext, Logging}
import org.apache.spark.util.{Utils, AkkaUtils}
@@ -59,12 +59,12 @@ class WorkerLauncher(args: ApplicationMasterArguments, conf: Configuration) exte
override def preStart() {
logInfo("Listen to driver: " + driverUrl)
driver = context.actorFor(driverUrl)
- context.system.eventStream.subscribe(self, classOf[RemoteClientLifeCycleEvent])
- context.watch(driver) // Doesn't work with remote actors, but useful for testing
+ driver ! "hello"
+ context.system.eventStream.subscribe(self, classOf[RemotingLifecycleEvent])
}
override def receive = {
- case Terminated(_) | RemoteClientDisconnected(_, _) | RemoteClientShutdown(_, _) =>
+ case x: DisassociatedEvent =>
logInfo("Driver terminated or disconnected! Shutting down.")
driverClosed = true
}
@@ -140,7 +140,7 @@ class WorkerLauncher(args: ApplicationMasterArguments, conf: Configuration) exte
System.setProperty("spark.driver.host", driverHost)
System.setProperty("spark.driver.port", driverPort.toString)
- val driverUrl = "akka://spark@%s:%s/user/%s".format(
+ val driverUrl = "akka.tcp://spark@%s:%s/user/%s".format(
driverHost, driverPort.toString, CoarseGrainedSchedulerBackend.ACTOR_NAME)
actor = actorSystem.actorOf(Props(new MonitorActor(driverUrl)), name = "YarnAM")
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
----------------------------------------------------------------------
diff --git a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index dba0f76..c27257c 100644
--- a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++ b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -253,7 +253,7 @@ private[yarn] class YarnAllocationHandler(
numWorkersRunning.decrementAndGet()
} else {
val workerId = workerIdCounter.incrementAndGet().toString
- val driverUrl = "akka://spark@%s:%s/user/%s".format(
+ val driverUrl = "akka.tcp://spark@%s:%s/user/%s".format(
System.getProperty("spark.driver.host"),
System.getProperty("spark.driver.port"),
CoarseGrainedSchedulerBackend.ACTOR_NAME)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fd99fab..39c8a8c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -104,12 +104,11 @@
<scala.version>2.10.3</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<mesos.version>0.13.0</mesos.version>
- <akka.version>2.2.3</akka.version>
- <akka.group>com.typesafe.akka</akka.group>
- <protobuf.version>2.4.1</protobuf.version>
+ <akka.version>2.2.3-shaded-protobuf</akka.version>
<slf4j.version>1.7.2</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<hadoop.version>1.0.4</hadoop.version>
+ <protobuf.version>2.4.1</protobuf.version>
<yarn.version>0.23.7</yarn.version>
<hbase.version>0.94.6</hbase.version>
@@ -200,6 +199,11 @@
<artifactId>asm</artifactId>
<version>4.0</version>
</dependency>
+ <!-- In theory we need not directly depend on protobuf since Spark does not directly
+ use it. However, when building with Hadoop/YARN 2.2 Maven doesn't correctly bump
+ the protobuf version up from the one Mesos gives. For now we include this variable
+ to explicitly bump the version when building with YARN. It would be nice to figure
+ out why Maven can't resolve this correctly (like SBT does). -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -216,6 +220,7 @@
<version>0.3.1</version>
</dependency>
<dependency>
+<<<<<<< HEAD
<groupId>${akka.group}</groupId>
<artifactId>akka-actor_${scala.binary.version}</artifactId>
<version>${akka.version}</version>
@@ -249,8 +254,13 @@
</exclusions>
</dependency>
<dependency>
+<<<<<<< HEAD
<groupId>${akka.group}</groupId>
<artifactId>akka-zeromq_${scala.binary.version}</artifactId>
+=======
+ <groupId>org.spark-project.akka</groupId>
+ <artifactId>akka-zeromq_2.10</artifactId>
+>>>>>>> Attempt with extra repositories
<version>${akka.version}</version>
<exclusions>
<exclusion>
@@ -461,6 +471,7 @@
</exclusion>
</exclusions>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
@@ -716,6 +727,7 @@
<hadoop.major.version>2</hadoop.major.version>
<!-- 0.23.* is same as 2.0.* - except hardened to run production jobs -->
<hadoop.version>0.23.7</hadoop.version>
+ <protobuf.version>2.5.0</protobuf.version>
<!--<hadoop.version>2.0.5-alpha</hadoop.version> -->
</properties>
@@ -743,39 +755,37 @@
</dependencyManagement>
</profile>
- <!-- <profile> -->
- <!-- <id>new-yarn</id> -->
- <!-- <properties> -->
- <!-- <akka.group>org.spark-project</akka.group> -->
- <!-- <akka.version>2.0.5-protobuf-2.5-java-1.5</akka.version> -->
- <!-- <hadoop.major.version>2</hadoop.major.version> -->
- <!-- <hadoop.version>2.2.0</hadoop.version> -->
- <!-- <protobuf.version>2.5.0</protobuf.version> -->
- <!-- </properties> -->
+ <profile>
+ <id>new-yarn</id>
+ <properties>
+ <hadoop.major.version>2</hadoop.major.version>
+ <hadoop.version>2.2.0</hadoop.version>
+ <protobuf.version>2.5.0</protobuf.version>
+ </properties>
- <!-- <modules> -->
- <!-- <module>new-yarn</module> -->
- <!-- </modules> -->
+ <modules>
+ <module>new-yarn</module>
+ </modules>
- <!-- <repositories> -->
- <!-- <repository> -->
- <!-- <id>maven-root</id> -->
- <!-- <name>Maven root repository</name> -->
- <!-- <url>http://repo1.maven.org/maven2/</url> -->
- <!-- <releases> -->
- <!-- <enabled>true</enabled> -->
- <!-- </releases> -->
- <!-- <snapshots> -->
- <!-- <enabled>false</enabled> -->
- <!-- </snapshots> -->
- <!-- </repository> -->
- <!-- </repositories> -->
+ <repositories>
+ <repository>
+ <id>maven-root</id>
+ <name>Maven root repository</name>
+ <url>http://repo1.maven.org/maven2/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
- <!-- <dependencyManagement> -->
- <!-- <dependencies> -->
- <!-- </dependencies> -->
- <!-- </dependencyManagement> -->
- <!-- </profile> -->
+ <dependencyManagement>
+ <dependencies>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
<profile>
<id>repl-bin</id>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c6f95e60/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 441dcc1..29f4a4b 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -84,21 +84,10 @@ object SparkBuild extends Build {
case Some(v) => v.toBoolean
}
- if (isNewHadoop && isYarnEnabled) {
- println( """Yarn with Hadoop version 2.2.x is not yet expected to work.
- Please set env SPARK_HADOOP_VERSION to appropriate version or set SPARK_YARN to false.""")
- throw new Exception("Yarn with Hadoop version 2.2.x is not yet expected to work.")
- }
-
- // Build against a protobuf-2.5 compatible Akka if Hadoop 2 is used.
- // lazy val protobufVersion = if (isNewHadoop) "2.5.0" else "2.4.1"
- // lazy val akkaVersion = if (isNewHadoop) "2.0.5-protobuf-2.5-java-1.5" else "2.0.5"
- // lazy val akkaGroup = if (isNewHadoop) "org.spark-project" else "com.typesafe.akka"
-
// Conditionally include the yarn sub-project
- //lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core)
+ lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core)
- lazy val yarn = Project("yarn", file("yarn"), settings = yarnSettings) dependsOn(core)
+ //lazy val yarn = Project("yarn", file("yarn"), settings = yarnSettings) dependsOn(core)
lazy val maybeYarn = if (isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]()
lazy val maybeYarnRef = if (isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]()
@@ -235,9 +224,8 @@ object SparkBuild extends Build {
"com.ning" % "compress-lzf" % "0.8.4",
"org.xerial.snappy" % "snappy-java" % "1.0.5",
"org.ow2.asm" % "asm" % "4.0",
- "com.google.protobuf" % "protobuf-java" % "2.4.1",
- "com.typesafe.akka" %% "akka-remote" % "2.2.3" excludeAll(excludeNetty),
- "com.typesafe.akka" %% "akka-slf4j" % "2.2.3" excludeAll(excludeNetty),
+ "org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
+ "org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
"net.liftweb" %% "lift-json" % "2.5.1" excludeAll(excludeNetty),
"it.unimi.dsi" % "fastutil" % "6.4.4",
"colt" % "colt" % "1.2.0",
@@ -312,16 +300,16 @@ object SparkBuild extends Build {
),
libraryDependencies ++= Seq(
- "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty, excludeSnappy),
- "com.sksamuel.kafka" %% "kafka" % "0.8.0-beta1"
+ "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty, excludeSnappy),
+ "com.sksamuel.kafka" %% "kafka" % "0.8.0-beta1"
exclude("com.sun.jdmk", "jmxtools")
exclude("com.sun.jmx", "jmxri")
exclude("net.sf.jopt-simple", "jopt-simple")
excludeAll(excludeNetty),
- "org.eclipse.paho" % "mqtt-client" % "0.4.0",
- "com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty),
- "org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty),
- "com.typesafe.akka" %% "akka-zeromq" % "2.2.3" excludeAll(excludeNetty)
+ "org.eclipse.paho" % "mqtt-client" % "0.4.0",
+ "com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty),
+ "org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty),
+ "org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
)
)
[4/7] git commit: Cleanup
Posted by pw...@apache.org.
Cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/c1fec898
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/c1fec898
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/c1fec898
Branch: refs/heads/master
Commit: c1fec89895f03dbdbb6f445ea3cdcd2d050555c4
Parents: 24f8220
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Dec 16 21:56:21 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 21:56:21 2013 -0800
----------------------------------------------------------------------
core/pom.xml | 1 -
pom.xml | 6 ------
2 files changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c1fec898/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index b83a2a8..043f6cf 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -97,7 +97,6 @@
<dependency>
<groupId>${akka.group}</groupId>
<artifactId>akka-slf4j_${scala.binary.version}</artifactId>
- <groupId>org.spark-project.akka</groupId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c1fec898/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30ef928..21d9da8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -220,7 +220,6 @@
<version>0.3.1</version>
</dependency>
<dependency>
-<<<<<<< HEAD
<groupId>${akka.group}</groupId>
<artifactId>akka-actor_${scala.binary.version}</artifactId>
<version>${akka.version}</version>
@@ -254,13 +253,8 @@
</exclusions>
</dependency>
<dependency>
-<<<<<<< HEAD
<groupId>${akka.group}</groupId>
<artifactId>akka-zeromq_${scala.binary.version}</artifactId>
-=======
- <groupId>org.spark-project.akka</groupId>
- <artifactId>akka-zeromq_2.10</artifactId>
->>>>>>> Attempt with extra repositories
<version>${akka.version}</version>
<exclusions>
<exclusion>
[7/7] git commit: Merge pull request #268 from
pwendell/shaded-protobuf
Posted by pw...@apache.org.
Merge pull request #268 from pwendell/shaded-protobuf
Add support for 2.2. to master (via shaded jars)
This patch does a few related things. NOTE: This may not compile correctly for ~24 hours until artifacts fully propagate to Maven Central.
1. Uses shaded versions of akka/protobuf. For more information on how these versions were prepared, see [1].
2. Brings the `new-yarn` project up-to-date with the changes for Akka 2.2.3.
3. Some clean-up of the build now that we don't have to switch akka groups for different YARN versions.
[1]
https://github.com/pwendell/spark-utils/tree/933a309ef85c22643e8e4b5e365652101c4e95de/shaded-protobuf
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/7a8169be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/7a8169be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/7a8169be
Branch: refs/heads/master
Commit: 7a8169be9a0b6b3d0d53a98aa38940d47b201296
Parents: 964a3b6 10c0ffa
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Dec 16 22:42:21 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 22:42:21 2013 -0800
----------------------------------------------------------------------
README.md | 6 +-
core/pom.xml | 8 --
new-yarn/pom.xml | 6 +-
.../spark/deploy/yarn/WorkerLauncher.scala | 13 ++--
.../deploy/yarn/YarnAllocationHandler.scala | 2 +-
pom.xml | 79 +++++++++++---------
project/SparkBuild.scala | 32 +++-----
7 files changed, 64 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
[6/7] git commit: One other fix
Posted by pw...@apache.org.
One other fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/10c0ffa1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/10c0ffa1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/10c0ffa1
Branch: refs/heads/master
Commit: 10c0ffa1eb23c6190f460be224ea555573458980
Parents: c1c0f80
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Dec 16 22:10:55 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 22:10:55 2013 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/10c0ffa1/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
----------------------------------------------------------------------
diff --git a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
index dca389d..bc31bb2 100644
--- a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
+++ b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
@@ -54,7 +54,7 @@ class WorkerLauncher(args: ApplicationMasterArguments, conf: Configuration) exte
// This actor just working as a monitor to watch on Driver Actor.
class MonitorActor(driverUrl: String) extends Actor {
- var driver: ActorRef = null
+ var driver: ActorSelection = null
override def preStart() {
logInfo("Listen to driver: " + driverUrl)
[5/7] git commit: Clean-up
Posted by pw...@apache.org.
Clean-up
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/c1c0f809
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/c1c0f809
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/c1c0f809
Branch: refs/heads/master
Commit: c1c0f8099f4559ff3ff9982878da6cd36d198ba0
Parents: c1fec89
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Dec 16 22:01:27 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 22:01:27 2013 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala | 2 +-
pom.xml | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c1c0f809/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
----------------------------------------------------------------------
diff --git a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
index ac519fc..dca389d 100644
--- a/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
+++ b/new-yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerLauncher.scala
@@ -58,7 +58,7 @@ class WorkerLauncher(args: ApplicationMasterArguments, conf: Configuration) exte
override def preStart() {
logInfo("Listen to driver: " + driverUrl)
- driver = context.actorFor(driverUrl)
+ driver = context.actorSelection(driverUrl)
context.system.eventStream.subscribe(self, classOf[RemotingLifecycleEvent])
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c1c0f809/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 21d9da8..57e8435 100644
--- a/pom.xml
+++ b/pom.xml
@@ -104,6 +104,7 @@
<scala.version>2.10.3</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<mesos.version>0.13.0</mesos.version>
+ <akka.group>org.spark-project.akka</akka.group>
<akka.version>2.2.3-shaded-protobuf</akka.version>
<slf4j.version>1.7.2</slf4j.version>
<log4j.version>1.2.17</log4j.version>
[2/7] git commit: Remove trailing slashes from repository
specifications.
Posted by pw...@apache.org.
Remove trailing slashes from repository specifications.
The correct format is to not have a trailing slash.
For me this caused non-deterministic failures due to issues fetching
certain artifacts. The issue was that some of the maven caches would
fail to fetch the artifact (due to the way that the artifact
path was concatenated with the repository) and this short-circuited
the download process in a silent way. Here is what the log output
looked like:
Downloading: http://repo.maven.apache.org/maven2/org/spark-project/akka/akka-remote_2.10/2.2.3-shaded-protobuf/akka-remote_2.10-2.2.3-shaded-protobuf.pom
[WARNING] The POM for org.spark-project.akka:akka-remote_2.10:jar:2.2.3-shaded-protobuf is missing, no dependency information available
This was pretty brutal to debug since there was no error message
anywhere and the path *looks* correct as reported by the Maven log.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ceb013f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ceb013f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ceb013f8
Branch: refs/heads/master
Commit: ceb013f8b97051ee96c65a8da7489a2b251ef799
Parents: c6f95e6
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Dec 16 12:38:01 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Dec 16 21:53:51 2013 -0800
----------------------------------------------------------------------
pom.xml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ceb013f8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 39c8a8c..30ef928 100644
--- a/pom.xml
+++ b/pom.xml
@@ -120,7 +120,7 @@
<repository>
<id>maven-repo</id> <!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
<name>Maven Repository</name>
- <url>http://repo.maven.apache.org/maven2/</url>
+ <url>http://repo.maven.apache.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
@@ -131,7 +131,7 @@
<repository>
<id>jboss-repo</id>
<name>JBoss Repository</name>
- <url>http://repository.jboss.org/nexus/content/repositories/releases/</url>
+ <url>http://repository.jboss.org/nexus/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
@@ -142,7 +142,7 @@
<repository>
<id>mqtt-repo</id>
<name>MQTT Repository</name>
- <url>https://repo.eclipse.org/content/repositories/paho-releases/</url>
+ <url>https://repo.eclipse.org/content/repositories/paho-releases</url>
<releases>
<enabled>true</enabled>
</releases>
@@ -739,7 +739,7 @@
<repository>
<id>maven-root</id>
<name>Maven root repository</name>
- <url>http://repo1.maven.org/maven2/</url>
+ <url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
@@ -771,7 +771,7 @@
<repository>
<id>maven-root</id>
<name>Maven root repository</name>
- <url>http://repo1.maven.org/maven2/</url>
+ <url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>