You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by ah...@apache.org on 2017/04/10 13:56:15 UTC
zeppelin git commit: [ZEPPELIN-2373] Remove -Pyarn build profile
Repository: zeppelin
Updated Branches:
refs/heads/master f9830a7d6 -> 540ceb87d
[ZEPPELIN-2373] Remove -Pyarn build profile
### What is this PR for?
Currently users who build Zeppelin from source need to include `-Pyarn` build profile to enable Yarn for embedded local Spark. This PR is to remove `-Pyarn` and make Yarn related libraries can be automatically downloaded without the profile during build time.
### What type of PR is it?
just removed Yarn build profile in `spark-dependencies/pom.xml`
### What is the Jira issue?
[ZEPPELIN-2373](https://issues.apache.org/jira/browse/ZEPPELIN-2373)
### How should this be tested?
1. apply this patch and build Zeppelin with below command
```
mvn clean package -DskipTests -pl 'zeppelin-server, spark-dependencies, spark' --am
```
2. check `spark-yarn_${scala.binary.version}.jar` is downloaded(or included) during build time
```
[INFO] Including org.apache.spark:spark-yarn_2.10:jar:2.1.0 in the shaded jar.
```
In current master, the above line won't be shown if you don't use `-Pyarn` build profile. But with this patch, it will be shown without the profile.
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: AhyoungRyu <fb...@hanmail.net>
Closes #2232 from AhyoungRyu/ZEPPELIN-2373/includeYarnByDefault and squashes the following commits:
59eb2b5 [AhyoungRyu] Remove -Pyarn build profile
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/540ceb87
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/540ceb87
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/540ceb87
Branch: refs/heads/master
Commit: 540ceb87d86a5751a5a93d3c050ecec36382e91f
Parents: f9830a7
Author: AhyoungRyu <fb...@hanmail.net>
Authored: Fri Apr 7 19:44:26 2017 +0900
Committer: ahyoungryu <ah...@apache.org>
Committed: Mon Apr 10 22:56:11 2017 +0900
----------------------------------------------------------------------
dev/create_release.sh | 4 ++--
dev/publish_release.sh | 2 +-
docs/install/build.md | 17 ++++++-----------
spark-dependencies/pom.xml | 30 +++++++++++++-----------------
4 files changed, 22 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/540ceb87/dev/create_release.sh
----------------------------------------------------------------------
diff --git a/dev/create_release.sh b/dev/create_release.sh
index 33ee84f..9cb61e0 100755
--- a/dev/create_release.sh
+++ b/dev/create_release.sh
@@ -106,8 +106,8 @@ function make_binary_release() {
git_clone
make_source_package
-make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Pscala-${SCALA_VERSION}"
-make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
+make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION}"
+make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
# remove non release files and dirs
rm -rf "${WORKING_DIR}/zeppelin"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/540ceb87/dev/publish_release.sh
----------------------------------------------------------------------
diff --git a/dev/publish_release.sh b/dev/publish_release.sh
index 76eac4b..b569ec4 100755
--- a/dev/publish_release.sh
+++ b/dev/publish_release.sh
@@ -46,7 +46,7 @@ if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
DO_SNAPSHOT="yes"
fi
-PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Pr"
+PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pr"
PROJECT_OPTIONS="-pl !zeppelin-distribution"
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
NEXUS_PROFILE="153446d1ac37c4"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/540ceb87/docs/install/build.md
----------------------------------------------------------------------
diff --git a/docs/install/build.md b/docs/install/build.md
index 5543f0b..c0974d9 100644
--- a/docs/install/build.md
+++ b/docs/install/build.md
@@ -69,7 +69,7 @@ If you're unsure about the options, use the same commands that creates official
# update all pom.xml to use scala 2.11
./dev/change_scala_version.sh 2.11
# build zeppelin with all interpreters and include latest version of Apache spark support for local mode.
-mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pyarn -Pr -Pscala-2.11
+mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pr -Pscala-2.11
```
####3. Done
@@ -140,11 +140,6 @@ Available profiles are
-Pscala-2.11
```
-##### `-Pyarn` (optional)
-
-enable YARN support for local mode
-> YARN for local mode is not supported for Spark v1.5.0 or higher. Set `SPARK_HOME` instead.
-
##### `-Pr` (optional)
enable [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration.
@@ -180,14 +175,14 @@ Here are some examples with several options:
```bash
# build with spark-2.1, scala-2.11
./dev/change_scala_version.sh 2.11
-mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Pscala-2.11 -DskipTests
+mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pscala-2.11 -DskipTests
# build with spark-2.0, scala-2.11
./dev/change_scala_version.sh 2.11
-mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Pscala-2.11 -DskipTests
+mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pscala-2.11 -DskipTests
# build with spark-1.6, scala-2.10
-mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -DskipTests
+mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
# spark-cassandra integration
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests -DskipTests
@@ -320,10 +315,10 @@ mvn clean package -Pbuild-distr
To build a distribution with specific profiles, run:
```sh
-mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn
+mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4
```
-The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`.
+The profiles `-Pspark-1.5 -Phadoop-2.4` can be adjusted if you wish to build to a specific spark versions.
The archive is generated under _`zeppelin-distribution/target`_ directory
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/540ceb87/spark-dependencies/pom.xml
----------------------------------------------------------------------
diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml
index b6a74dd..50d8ed5 100644
--- a/spark-dependencies/pom.xml
+++ b/spark-dependencies/pom.xml
@@ -344,6 +344,19 @@
</exclusions>
</dependency>
+ <!-- yarn (not supported for Spark v1.5.0 or higher) -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${yarn.version}</version>
+ </dependency>
+
</dependencies>
<profiles>
@@ -810,23 +823,6 @@
</repositories>
</profile>
- <profile>
- <id>yarn</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${yarn.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
</profiles>
<build>