You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/12/27 07:56:20 UTC
[1/2] spark git commit: [SPARK-3787][BUILD] Assembly jar name is
wrong when we build with sbt omitting -Dhadoop.version
Repository: spark
Updated Branches:
refs/heads/branch-1.2 391080b68 -> 3c4acac43
[SPARK-3787][BUILD] Assembly jar name is wrong when we build with sbt omitting -Dhadoop.version
This PR is another solution for When we build with sbt with profile for hadoop and without property for hadoop version like:
sbt/sbt -Phadoop-2.2 assembly
jar name is always used default version (1.0.4).
When we build with maven with same condition for sbt, default version for each profile is used.
For instance, if we build like:
mvn -Phadoop-2.2 package
jar name is used hadoop2.2.0 as a default version of hadoop-2.2.
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Closes #3046 from sarutak/fix-assembly-jarname-2 and squashes the following commits:
41ef90e [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into fix-assembly-jarname-2
50c8676 [Kousuke Saruta] Merge branch 'fix-assembly-jarname-2' of github.com:sarutak/spark into fix-assembly-jarname-2
52a1cd2 [Kousuke Saruta] Fixed comflicts
dd30768 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into fix-assembly-jarname2
f1c90bb [Kousuke Saruta] Fixed SparkBuild.scala in order to read `hadoop.version` property from pom.xml
af6b100 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into fix-assembly-jarname
c81806b [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into fix-assembly-jarname
ad1f96e [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into fix-assembly-jarname
b2318eb [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into fix-assembly-jarname
5fc1259 [Kousuke Saruta] Fixed typo.
eebbb7d [Kousuke Saruta] Fixed wrong jar name
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2e0af872
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2e0af872
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2e0af872
Branch: refs/heads/branch-1.2
Commit: 2e0af87277093907207d0dd0896930f21a763f41
Parents: 391080b
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Authored: Fri Dec 26 22:52:04 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Fri Dec 26 22:56:08 2014 -0800
----------------------------------------------------------------------
project/SparkBuild.scala | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/2e0af872/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index ffc7a93..9d316d6 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import java.io.File
+
import scala.util.Properties
import scala.collection.JavaConversions._
@@ -23,7 +25,7 @@ import sbt.Classpaths.publishTask
import sbt.Keys._
import sbtunidoc.Plugin.genjavadocSettings
import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
-import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
+import com.typesafe.sbt.pom.{loadEffectivePom, PomBuild, SbtPomKeys}
import net.virtualvoid.sbt.graph.Plugin.graphSettings
object BuildCommons {
@@ -118,6 +120,15 @@ object SparkBuild extends PomBuild {
override val userPropertiesMap = System.getProperties.toMap
+ val pom = loadEffectivePom(new File("pom.xml"),
+ profiles = profiles,
+ userProps = userPropertiesMap)
+
+ if (System.getProperty("hadoop.version") == null) {
+ System.setProperty("hadoop.version",
+ pom.getProperties.get("hadoop.version").asInstanceOf[String])
+ }
+
lazy val MavenCompile = config("m2r") extend(Compile)
lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
@@ -303,8 +314,7 @@ object Assembly {
// This must match the same name used in maven (see network/yarn/pom.xml)
"spark-" + v + "-yarn-shuffle.jar"
} else {
- mName + "-" + v + "-hadoop" +
- Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar"
+ mName + "-" + v + "-hadoop" + System.getProperty("hadoop.version") + ".jar"
}
},
mergeStrategy in assembly := {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/2] spark git commit: HOTFIX: Slight tweak on previous commit.
Posted by pw...@apache.org.
HOTFIX: Slight tweak on previous commit.
Meant to merge this in when committing SPARK-3787.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3c4acac4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3c4acac4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3c4acac4
Branch: refs/heads/branch-1.2
Commit: 3c4acac43f8910e9ff46392491e64803f7b1ae5f
Parents: 2e0af87
Author: Patrick Wendell <pw...@gmail.com>
Authored: Fri Dec 26 22:55:04 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Fri Dec 26 22:56:10 2014 -0800
----------------------------------------------------------------------
project/SparkBuild.scala | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3c4acac4/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 9d316d6..49628b1 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -120,10 +120,12 @@ object SparkBuild extends PomBuild {
override val userPropertiesMap = System.getProperties.toMap
+ // Handle case where hadoop.version is set via profile.
+ // Needed only because we read back this property in sbt
+ // when we create the assembly jar.
val pom = loadEffectivePom(new File("pom.xml"),
profiles = profiles,
userProps = userPropertiesMap)
-
if (System.getProperty("hadoop.version") == null) {
System.setProperty("hadoop.version",
pom.getProperties.get("hadoop.version").asInstanceOf[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org