You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2013/12/13 06:14:57 UTC
[1/2] git commit: Disabled yarn 2.2 and added a message in the sbt
build
Updated Branches:
refs/heads/scala-2.10 2e89398e4 -> 0aeb182b0
Disabled yarn 2.2 and added a message in the sbt build
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/589b83a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/589b83a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/589b83a1
Branch: refs/heads/scala-2.10
Commit: 589b83a18f60c2862780bdba655662b73bb575c5
Parents: 2e89398
Author: Prashant Sharma <pr...@imaginea.com>
Authored: Thu Dec 12 16:20:10 2013 +0530
Committer: Prashant Sharma <pr...@imaginea.com>
Committed: Thu Dec 12 16:25:30 2013 +0530
----------------------------------------------------------------------
pom.xml | 60 +++++++++++++++++++++----------------------
project/SparkBuild.scala | 24 ++++++++++++-----
2 files changed, 47 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/589b83a1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6906ad2..aee07c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -738,39 +738,39 @@
</dependencyManagement>
</profile>
- <profile>
- <id>new-yarn</id>
- <properties>
- <akka.group>org.spark-project</akka.group>
- <akka.version>2.0.5-protobuf-2.5-java-1.5</akka.version>
- <hadoop.major.version>2</hadoop.major.version>
- <hadoop.version>2.2.0</hadoop.version>
- <protobuf.version>2.5.0</protobuf.version>
- </properties>
+ <!-- <profile> -->
+ <!-- <id>new-yarn</id> -->
+ <!-- <properties> -->
+ <!-- <akka.group>org.spark-project</akka.group> -->
+ <!-- <akka.version>2.0.5-protobuf-2.5-java-1.5</akka.version> -->
+ <!-- <hadoop.major.version>2</hadoop.major.version> -->
+ <!-- <hadoop.version>2.2.0</hadoop.version> -->
+ <!-- <protobuf.version>2.5.0</protobuf.version> -->
+ <!-- </properties> -->
- <modules>
- <module>new-yarn</module>
- </modules>
+ <!-- <modules> -->
+ <!-- <module>new-yarn</module> -->
+ <!-- </modules> -->
- <repositories>
- <repository>
- <id>maven-root</id>
- <name>Maven root repository</name>
- <url>http://repo1.maven.org/maven2/</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- </repositories>
+ <!-- <repositories> -->
+ <!-- <repository> -->
+ <!-- <id>maven-root</id> -->
+ <!-- <name>Maven root repository</name> -->
+ <!-- <url>http://repo1.maven.org/maven2/</url> -->
+ <!-- <releases> -->
+ <!-- <enabled>true</enabled> -->
+ <!-- </releases> -->
+ <!-- <snapshots> -->
+ <!-- <enabled>false</enabled> -->
+ <!-- </snapshots> -->
+ <!-- </repository> -->
+ <!-- </repositories> -->
- <dependencyManagement>
- <dependencies>
- </dependencies>
- </dependencyManagement>
- </profile>
+ <!-- <dependencyManagement> -->
+ <!-- <dependencies> -->
+ <!-- </dependencies> -->
+ <!-- </dependencyManagement> -->
+ <!-- </profile> -->
<profile>
<id>repl-bin</id>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/589b83a1/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 05d3f18..c322cbd 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -78,18 +78,28 @@ object SparkBuild extends Build {
}
case Some(v) => v.toBoolean
}
+
lazy val isYarnEnabled = scala.util.Properties.envOrNone("SPARK_YARN") match {
case None => DEFAULT_YARN
case Some(v) => v.toBoolean
}
+ if(isNewHadoop && isYarnEnabled) {
+ println("""Yarn with Hadoop version 2.2.x is not yet expected to work.
+ Please set env SPARK_HADOOP_VERSION to appropriate version or set SPARK_YARN to false.""")
+ throw new Exception("Yarn with Hadoop version 2.2.x is not yet expected to work.")
+ }
+
// Build against a protobuf-2.5 compatible Akka if Hadoop 2 is used.
- lazy val protobufVersion = if (isNewHadoop) "2.5.0" else "2.4.1"
- lazy val akkaVersion = if (isNewHadoop) "2.0.5-protobuf-2.5-java-1.5" else "2.0.5"
- lazy val akkaGroup = if (isNewHadoop) "org.spark-project" else "com.typesafe.akka"
+ // lazy val protobufVersion = if (isNewHadoop) "2.5.0" else "2.4.1"
+ // lazy val akkaVersion = if (isNewHadoop) "2.0.5-protobuf-2.5-java-1.5" else "2.0.5"
+ // lazy val akkaGroup = if (isNewHadoop) "org.spark-project" else "com.typesafe.akka"
// Conditionally include the yarn sub-project
- lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core)
+ //lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core)
+
+ lazy val yarn = Project("yarn", file("yarn"), settings = yarnSettings) dependsOn(core)
+
lazy val maybeYarn = if (isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]()
lazy val maybeYarnRef = if (isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]()
@@ -226,8 +236,8 @@ object SparkBuild extends Build {
"org.xerial.snappy" % "snappy-java" % "1.0.5",
"org.ow2.asm" % "asm" % "4.0",
"com.google.protobuf" % "protobuf-java" % "2.4.1",
- akkaGroup %% "akka-remote" % "2.2.3" excludeAll(excludeNetty),
- akkaGroup %% "akka-slf4j" % "2.2.3" excludeAll(excludeNetty),
+ "com.typesafe.akka" %% "akka-remote" % "2.2.3" excludeAll(excludeNetty),
+ "com.typesafe.akka" %% "akka-slf4j" % "2.2.3" excludeAll(excludeNetty),
"net.liftweb" %% "lift-json" % "2.5.1" excludeAll(excludeNetty),
"it.unimi.dsi" % "fastutil" % "6.4.4",
"colt" % "colt" % "1.2.0",
@@ -311,7 +321,7 @@ object SparkBuild extends Build {
"org.eclipse.paho" % "mqtt-client" % "0.4.0",
"com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty),
"org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty),
- akkaGroup %% "akka-zeromq" % "2.2.3" excludeAll(excludeNetty)
+ "com.typesafe.akka" %% "akka-zeromq" % "2.2.3" excludeAll(excludeNetty)
)
)
[2/2] git commit: Merge pull request #255 from ScrapCodes/scala-2.10
Posted by pw...@apache.org.
Merge pull request #255 from ScrapCodes/scala-2.10
Disabled yarn 2.2 in sbt and mvn build and added a message in the sbt build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0aeb182b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0aeb182b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0aeb182b
Branch: refs/heads/scala-2.10
Commit: 0aeb182b0f2881e3c9fd7f5953d7b1ebe2b890bd
Parents: 2e89398 589b83a
Author: Patrick Wendell <pw...@gmail.com>
Authored: Thu Dec 12 21:14:42 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Thu Dec 12 21:14:42 2013 -0800
----------------------------------------------------------------------
pom.xml | 60 +++++++++++++++++++++----------------------
project/SparkBuild.scala | 24 ++++++++++++-----
2 files changed, 47 insertions(+), 37 deletions(-)
----------------------------------------------------------------------