You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/11/14 23:22:08 UTC
spark git commit: SPARK-4375. no longer require -Pscala-2.10
Repository: spark
Updated Branches:
refs/heads/master bbd8f5bee -> f5f757e4e
SPARK-4375. no longer require -Pscala-2.10
It seems like the winds might have moved away from this approach, but wanted to post the PR anyway because I got it working and to show what it would look like.
Author: Sandy Ryza <sa...@cloudera.com>
Closes #3239 from sryza/sandy-spark-4375 and squashes the following commits:
0ffbe95 [Sandy Ryza] Enable -Dscala-2.11 in sbt
cd42d94 [Sandy Ryza] Update doc
f6644c3 [Sandy Ryza] SPARK-4375 take 2
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f5f757e4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f5f757e4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f5f757e4
Branch: refs/heads/master
Commit: f5f757e4ed80759dc5668c63d5663651689f8da8
Parents: bbd8f5b
Author: Sandy Ryza <sa...@cloudera.com>
Authored: Fri Nov 14 14:21:57 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Fri Nov 14 14:21:57 2014 -0800
----------------------------------------------------------------------
docs/building-spark.md | 4 +-
examples/pom.xml | 65 ++---------------------
pom.xml | 22 +-------
project/SparkBuild.scala | 3 ++
repl/pom.xml | 117 +++++++++++++++---------------------------
sql/catalyst/pom.xml | 11 +---
sql/hive/pom.xml | 3 --
7 files changed, 54 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/docs/building-spark.md
----------------------------------------------------------------------
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 20ba7da..bb18414 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -113,9 +113,9 @@ mvn -Pyarn -Phive -Phive-thriftserver-0.12.0 -Phadoop-2.4 -Dhadoop.version=2.4.0
{% endhighlight %}
# Building for Scala 2.11
-To produce a Spark package compiled with Scala 2.11, use the `-Pscala-2.11` profile:
+To produce a Spark package compiled with Scala 2.11, use the `-Dscala-2.11` property:
- mvn -Pyarn -Phadoop-2.4 -Pscala-2.11 -DskipTests clean package
+ mvn -Pyarn -Phadoop-2.4 -Dscala-2.11 -DskipTests clean package
Scala 2.11 support in Spark is experimental and does not support a few features.
Specifically, Spark's external Kafka library and JDBC component are not yet
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 2ec5728..2752ce3 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -389,11 +389,11 @@
</properties>
</profile>
<profile>
- <!-- We add source directories specific to Scala 2.10 and 2.11 since some examples
- work only in one and not the other -->
+ <!-- We add a source directory specific to Scala 2.10 since Kafka and Algebird
+ only work with it -->
<id>scala-2.10</id>
<activation>
- <activeByDefault>true</activeByDefault>
+ <property><name>!scala-2.11</name></property>
</activation>
<dependencies>
<dependency>
@@ -427,65 +427,6 @@
</sources>
</configuration>
</execution>
- <execution>
- <id>add-scala-test-sources</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- <source>scala-2.10/src/test/scala</source>
- <source>scala-2.10/src/test/java</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>scala-2.11</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <!-- Streaming Kafka and zeromq modules are disabled for now. -->
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-scala-sources</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/scala</source>
- <source>scala-2.11/src/main/scala</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-scala-test-sources</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- <source>scala-2.11/src/test/scala</source>
- </sources>
- </configuration>
- </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7bbde31..90d3bff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1296,9 +1296,6 @@
<profile>
<id>mapr3</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<properties>
<hadoop.version>1.0.3-mapr-3.0.3</hadoop.version>
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
@@ -1309,9 +1306,6 @@
<profile>
<id>mapr4</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<properties>
<hadoop.version>2.3.0-mapr-4.0.0-FCS</hadoop.version>
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
@@ -1341,9 +1335,6 @@
<!-- Build without Hadoop dependencies that are included in some runtime environments. -->
<profile>
<id>hadoop-provided</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -1390,18 +1381,12 @@
</profile>
<profile>
<id>hive-thriftserver</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<modules>
<module>sql/hive-thriftserver</module>
</modules>
</profile>
<profile>
<id>hive-0.12.0</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<properties>
<hive.version>0.12.0-protobuf-2.5</hive.version>
<hive.version.short>0.12.0</hive.version.short>
@@ -1410,9 +1395,6 @@
</profile>
<profile>
<id>hive-0.13.1</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<properties>
<hive.version>0.13.1a</hive.version>
<hive.version.short>0.13.1</hive.version.short>
@@ -1423,7 +1405,7 @@
<profile>
<id>scala-2.10</id>
<activation>
- <activeByDefault>true</activeByDefault>
+ <property><name>!scala-2.11</name></property>
</activation>
<properties>
<scala.version>2.10.4</scala.version>
@@ -1439,7 +1421,7 @@
<profile>
<id>scala-2.11</id>
<activation>
- <activeByDefault>false</activeByDefault>
+ <property><name>scala-2.11</name></property>
</activation>
<properties>
<scala.version>2.11.2</scala.version>
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index d95d50a..c96a6c4 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -100,8 +100,11 @@ object SparkBuild extends PomBuild {
"conjunction with environment variable.")
v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
}
+
if (profiles.exists(_.contains("scala-"))) {
profiles
+ } else if (System.getProperty("scala-2.11") != null) {
+ profiles ++ Seq("scala-2.11")
} else {
println("Enabled default scala profile")
profiles ++ Seq("scala-2.10")
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/repl/pom.xml
----------------------------------------------------------------------
diff --git a/repl/pom.xml b/repl/pom.xml
index bd688c8..c2bf9fd 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -35,6 +35,8 @@
<sbt.project.name>repl</sbt.project.name>
<deb.install.path>/usr/share/spark</deb.install.path>
<deb.user>root</deb.user>
+ <extra.source.dir>scala-2.10/src/main/scala</extra.source.dir>
+ <extra.testsource.dir>scala-2.10/src/test/scala</extra.testsource.dir>
</properties>
<dependencies>
@@ -122,86 +124,51 @@
</environmentVariables>
</configuration>
</plugin>
+ <!-- Include a source dir depending on the Scala version -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-scala-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ <source>${extra.source.dir}</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-scala-test-sources</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ <source>${extra.testsource.dir}</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<profiles>
<profile>
- <id>scala-2.10</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-scala-sources</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/scala</source>
- <source>scala-2.10/src/main/scala</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-scala-test-sources</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- <source>scala-2.10/src/test/scala</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<id>scala-2.11</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-scala-sources</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/main/scala</source>
- <source>scala-2.11/src/main/scala</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-scala-test-sources</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>src/test/scala</source>
- <source>scala-2.11/src/test/scala</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <activation>
+ <property><name>scala-2.11</name></property>
+ </activation>
+ <properties>
+ <extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
+ <extra.testsource.dir>scala-2.11/src/test/scala</extra.testsource.dir>
+ </properties>
</profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/sql/catalyst/pom.xml
----------------------------------------------------------------------
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 0cc3175..686d189 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -100,10 +100,11 @@
</plugins>
</build>
<profiles>
+ <!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. -->
<profile>
<id>scala-2.10</id>
<activation>
- <activeByDefault>true</activeByDefault>
+ <property><name>!scala-2.11</name></property>
</activation>
<dependencies>
<dependency>
@@ -113,13 +114,5 @@
</dependency>
</dependencies>
</profile>
- <profile>
- <id>scala-2.11</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. -->
- </profile>
-
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/f5f757e4/sql/hive/pom.xml
----------------------------------------------------------------------
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 67e36a9..fa9a1e6 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -144,9 +144,6 @@
</profile>
<profile>
<id>hive-0.12.0</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>com.twitter</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org