You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by al...@apache.org on 2020/04/22 06:55:15 UTC
[zeppelin] branch master updated: [ZEPPELIN-4758] Additional
cleanup for Java 8 support
This is an automated email from the ASF dual-hosted git repository.
alexott pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new b68248b [ZEPPELIN-4758] Additional cleanup for Java 8 support
b68248b is described below
commit b68248b542cab019609ff3a91ec82879ceb8c45a
Author: Alex Ott <al...@gmail.com>
AuthorDate: Sun Apr 19 13:04:26 2020 +0200
[ZEPPELIN-4758] Additional cleanup for Java 8 support
### What is this PR for?
Improve Java 8 support:
- Remove obsolete Java options
- add the check for minimum Java version (Java 8u151) to `zeppelin.sh` & `interpreter.sh`
### What type of PR is it?
Refactoring
### What is the Jira issue?
* ZEPPELIN-4758
### How should this be tested?
* https://travis-ci.org/github/alexott/zeppelin/builds/677566776
Author: Alex Ott <al...@gmail.com>
Closes #3738 from alexott/ZEPPELIN-4758 and squashes the following commits:
e607ee61a [Alex Ott] [ZEPPELIN-4758] Additional cleanup for Java 8 support
---
.travis.yml | 4 ++--
bin/common.sh | 14 ++++++++++++++
bin/interpreter.sh | 7 +++++--
bin/zeppelin.sh | 6 ++++--
conf/zeppelin-env.cmd.template | 4 ++--
conf/zeppelin-env.sh.template | 4 ++--
dev/publish_release.sh | 2 +-
docs/quickstart/install.md | 4 ++--
docs/setup/basics/how_to_build.md | 2 +-
docs/setup/operation/configuration.md | 2 +-
docs/setup/operation/upgrading.md | 2 +-
flink/pom.xml | 3 +--
pom.xml | 3 +--
r/pom.xml | 2 +-
rlang/pom.xml | 2 +-
scio/pom.xml | 2 +-
spark/interpreter/pom.xml | 2 +-
spark/pom.xml | 3 +--
spark/spark-dependencies/pom.xml | 2 +-
spark/spark-scala-parent/pom.xml | 3 +--
20 files changed, 44 insertions(+), 29 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index d607a44..cdf1f93 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -170,7 +170,7 @@ before_install:
- gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog=""
- clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true)
- if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache verify; else echo "Using cached bower_components."; fi
- - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
+ - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
- bash -x ./testing/install_external_dependencies.sh
- ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
@@ -197,7 +197,7 @@ before_script:
# https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
- if [[ -n $TEST_MODULES ]]; then export DISPLAY=:99.0; sh -e /etc/init.d/xvfb start; sleep 3; fi
# display info log for debugging
- - if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi
+ - if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi
script:
- if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
diff --git a/bin/common.sh b/bin/common.sh
index c9b01dd..eb959e1 100644
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -62,6 +62,20 @@ fi
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}"
+function check_java_version() {
+ java_ver_output=$("${JAVA:-java}" -version 2>&1)
+ jvmver=$(echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1)
+ JVM_VERSION=$(echo "$jvmver"|sed -e 's|^\([0-9]\+\)\..*$|\1|')
+ if [ "$JVM_VERSION" = "1" ]; then
+ JVM_VERSION=$(echo "$jvmver"|sed -e 's|^1\.\([0-9]\+\)\..*$|\1|')
+ fi
+
+ if [ "$JVM_VERSION" -lt 8 ] || ([ "$JVM_VERSION" -eq 8 ] && [ "${jvmver#*_}" -lt 151 ]) ; then
+ echo "Apache Zeppelin requires either Java 8 update 151 or newer"
+ exit 1;
+ fi
+}
+
function addEachJarInDir(){
if [[ -d "${1}" ]]; then
for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index 672f307..e165fa4 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -17,8 +17,8 @@
#
-bin=$(dirname "${BASH_SOURCE-$0}")
-bin=$(cd "${bin}">/dev/null; pwd)
+bin="$(dirname "${BASH_SOURCE-$0}")"
+bin="$(cd "${bin}">/dev/null; pwd)"
function usage() {
echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
@@ -92,6 +92,9 @@ fi
. "${bin}/common.sh"
+check_java_version
+
+
ZEPPELIN_INTERPRETER_API_JAR=$(find "${ZEPPELIN_HOME}/interpreter" -name 'zeppelin-interpreter-shaded-*.jar')
ZEPPELIN_INTP_CLASSPATH="${CLASSPATH}:${ZEPPELIN_INTERPRETER_API_JAR}"
diff --git a/bin/zeppelin.sh b/bin/zeppelin.sh
index d559618..d8cf00b 100755
--- a/bin/zeppelin.sh
+++ b/bin/zeppelin.sh
@@ -72,11 +72,13 @@ do
done
set -- "${POSITIONAL[@]}" # restore positional parameters
-bin=$(dirname "${BASH_SOURCE-$0}")
-bin=$(cd "${bin}">/dev/null; pwd)
+bin="$(dirname "${BASH_SOURCE-$0}")"
+bin="$(cd "${bin}">/dev/null; pwd)"
. "${bin}/common.sh"
+check_java_version
+
if [ "$1" == "--version" ] || [ "$1" == "-v" ]; then
getZeppelinVersion
fi
diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template
index 83b5ee7..ee3c798 100644
--- a/conf/zeppelin-env.cmd.template
+++ b/conf/zeppelin-env.cmd.template
@@ -19,8 +19,8 @@ REM
REM set JAVA_HOME=
REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
-REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
-REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default -Xmx1024m -Xms1024m -XX:MaxPermSize=512m
+REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxMetaspaceSize=512m
+REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default -Xmx1024m -Xms1024m -XX:MaxMetaspaceSize=512m
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
REM set ZEPPELIN_JMX_ENABLE REM Enable JMX feature by defining it like "true"
REM set ZEPPELIN_JMX_PORT REM Port number which JMX uses. If not set, JMX won't be enabled
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index c31dd67..ee03635 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -22,8 +22,8 @@
# export ZEPPELIN_PORT # port number to listen (default 8080)
# export ZEPPELIN_LOCAL_IP # Zeppelin's thrift server ip address, if not specified, one random IP address will be choosen.
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
-# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
-# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
+# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxMetaspaceSize=512m
+# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxMetaspaceSize=512m
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
# export ZEPPELIN_SSL_PORT # ssl port (used when ssl environment variable is set to true)
# export ZEPPELIN_JMX_ENABLE # Enable JMX feature by defining "true"
diff --git a/dev/publish_release.sh b/dev/publish_release.sh
index 3c8ecf6..b389fd7 100755
--- a/dev/publish_release.sh
+++ b/dev/publish_release.sh
@@ -36,7 +36,7 @@ for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do
fi
done
-export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m"
+export MAVEN_OPTS="-Xmx2g -XX:MaxMetaspaceSize=512m"
RED='\033[0;31m'
NC='\033[0m' # No Color
diff --git a/docs/quickstart/install.md b/docs/quickstart/install.md
index 4ed778d..f0c3985 100644
--- a/docs/quickstart/install.md
+++ b/docs/quickstart/install.md
@@ -35,8 +35,8 @@ Apache Zeppelin officially supports and is tested on the following environments:
<th>Value</th>
</tr>
<tr>
- <td>Oracle JDK</td>
- <td>1.8 (171) <br /> (set <code>JAVA_HOME</code>)</td>
+ <td>OpenJDK or Oracle JDK</td>
+ <td>1.8 (151+) <br /> (set <code>JAVA_HOME</code>)</td>
</tr>
<tr>
<td>OS</td>
diff --git a/docs/setup/basics/how_to_build.md b/docs/setup/basics/how_to_build.md
index 209f169..af0e530 100644
--- a/docs/setup/basics/how_to_build.md
+++ b/docs/setup/basics/how_to_build.md
@@ -237,7 +237,7 @@ sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/local/bin/mvn
_Notes:_
- Ensure node is installed by running `node --version`
- Ensure maven is running version 3.1.x or higher with `mvn -version`
- - Configure maven to use more memory than usual by `export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=1024m"`
+ - Configure maven to use more memory than usual by `export MAVEN_OPTS="-Xmx2g -XX:MaxMetaspaceSize=512m"`
diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md
index cadbc21..584b8c2 100644
--- a/docs/setup/operation/configuration.md
+++ b/docs/setup/operation/configuration.md
@@ -74,7 +74,7 @@ If both are defined, then the **environment variables** will take priority.
<tr>
<td><h6 class="properties">ZEPPELIN_MEM</h6></td>
<td>N/A</td>
- <td>-Xmx1024m -XX:MaxPermSize=512m</td>
+ <td>-Xmx1024m -XX:MaxMetaspaceSize=512m</td>
<td>JVM mem options</td>
</tr>
<tr>
diff --git a/docs/setup/operation/upgrading.md b/docs/setup/operation/upgrading.md
index b998a6b..9c196e4 100644
--- a/docs/setup/operation/upgrading.md
+++ b/docs/setup/operation/upgrading.md
@@ -53,7 +53,7 @@ So, copying `notebook` and `conf` directory should be enough.
### Upgrading from Zeppelin 0.6 to 0.7
- - From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly. If you don't set `ZEPPELIN_INTP_MEM`, Zeppelin will set it to `-Xms1024m -Xmx1024m -XX:MaxPermSize=512m` by default.
+ - From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly. If you don't set `ZEPPELIN_INTP_MEM`, Zeppelin will set it to `-Xms1024m -Xmx1024m -XX:MaxMetaspaceSize=512m` by default.
- Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.
- Usage of `ZEPPELIN_PORT` is not supported in ssl mode. Instead use `ZEPPELIN_SSL_PORT` to configure the ssl port. Value from `ZEPPELIN_PORT` is used only when `ZEPPELIN_SSL` is set to `false`.
- The support on Spark 1.1.x to 1.3.x is deprecated.
diff --git a/flink/pom.xml b/flink/pom.xml
index 69f0b32..69eb026 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -595,8 +595,7 @@
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
- <jvmArg>-XX:PermSize=${PermGen}</jvmArg>
- <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
+ <jvmArg>-XX:MaxMetaspaceSize=${MaxMetaspace}</jvmArg>
</jvmArgs>
<javacArgs>
<javacArg>-source</javacArg>
diff --git a/pom.xml b/pom.xml
index c9bc824..fd38091 100644
--- a/pom.xml
+++ b/pom.xml
@@ -183,8 +183,7 @@
<plugin.gitcommitid.useNativeGit>false</plugin.gitcommitid.useNativeGit>
- <PermGen>64m</PermGen>
- <MaxPermGen>512m</MaxPermGen>
+ <MaxMetaspace>512m</MaxMetaspace>
<!-- to be able to exclude some tests using command line -->
<tests.to.exclude/>
diff --git a/r/pom.xml b/r/pom.xml
index d0502d2..9867e3a 100644
--- a/r/pom.xml
+++ b/r/pom.xml
@@ -178,7 +178,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xmx1024m -XX:MaxMetaspaceSize=512m</argLine>
<skipTests>true</skipTests>
</configuration>
</plugin>
diff --git a/rlang/pom.xml b/rlang/pom.xml
index 7dcaade..072ed1a 100644
--- a/rlang/pom.xml
+++ b/rlang/pom.xml
@@ -217,7 +217,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx2048m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xmx2048m -XX:MaxMetaspaceSize=512m</argLine>
<environmentVariables>
<ZEPPELIN_HOME>${basedir}/../</ZEPPELIN_HOME>
</environmentVariables>
diff --git a/scio/pom.xml b/scio/pom.xml
index 06a3d27..56f233a 100644
--- a/scio/pom.xml
+++ b/scio/pom.xml
@@ -127,7 +127,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xmx1024m -XX:MaxMetaspaceSize=512m</argLine>
</configuration>
</plugin>
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index c125f4e..f98d53e 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -467,7 +467,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx3072m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xmx3072m -XX:MaxMetaspaceSize=256m</argLine>
<excludes>
<exclude>${pyspark.test.exclude}</exclude>
<exclude>${tests.to.exclude}</exclude>
diff --git a/spark/pom.xml b/spark/pom.xml
index 8a635ab..97f0b06 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -145,8 +145,7 @@
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
- <jvmArg>-XX:PermSize=${PermGen}</jvmArg>
- <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
+ <jvmArg>-XX:MaxMetaspaceSize=${MaxMetaspace}</jvmArg>
</jvmArgs>
<javacArgs>
<javacArg>-source</javacArg>
diff --git a/spark/spark-dependencies/pom.xml b/spark/spark-dependencies/pom.xml
index 4f4fcaa..79cd871 100644
--- a/spark/spark-dependencies/pom.xml
+++ b/spark/spark-dependencies/pom.xml
@@ -167,7 +167,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xmx1024m -XX:MaxMetaspaceSize=256m</argLine>
</configuration>
</plugin>
diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml
index e989230..10749c1 100644
--- a/spark/spark-scala-parent/pom.xml
+++ b/spark/spark-scala-parent/pom.xml
@@ -250,8 +250,7 @@
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
- <jvmArg>-XX:PermSize=${PermGen}</jvmArg>
- <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
+ <jvmArg>-XX:MaxMetaspaceSize=${MaxMetaspace}</jvmArg>
</jvmArgs>
<javacArgs>
<javacArg>-source</javacArg>