You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/06/01 08:44:49 UTC
[zeppelin] branch branch-0.9 updated: [ZEPPELIN-4802]. pyspark
warnings with spark-3.0.0-preview - Sync of properties between JVM and PVM
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new f8528ad [ZEPPELIN-4802]. pyspark warnings with spark-3.0.0-preview - Sync of properties between JVM and PVM
f8528ad is described below
commit f8528ad912f2f9808beb4607e61c84c916107f00
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Wed May 27 17:37:52 2020 +0800
[ZEPPELIN-4802]. pyspark warnings with spark-3.0.0-preview - Sync of properties between JVM and PVM
### What is this PR for?
This PR will first set env `PYSPARK_PIN_THREAD` to be `true` and then suppress the warning message in `pyspark.util`.
### What type of PR is it?
[Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4802
### How should this be tested?
* CI pass
* Manually tested
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? No
* Does this needs documentation?
Author: Jeff Zhang <zj...@apache.org>
Closes #3778 from zjffdu/ZEPPELIN-4802 and squashes the following commits:
d6a65cbde [Jeff Zhang] [ZEPPELIN-4802]. pyspark warnings with spark-3.0.0-preview - Sync of properties between JVM and PVM
(cherry picked from commit df86e50218eaede551b1a3b84d6142646a17a42c)
Signed-off-by: Jeff Zhang <zj...@apache.org>
---
.../src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java | 4 ++++
.../src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java | 4 ++++
spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py | 4 ++++
spark/interpreter/src/main/resources/python/zeppelin_pyspark.py | 5 +++++
.../zeppelin/interpreter/launcher/SparkInterpreterLauncher.java | 2 ++
5 files changed, 19 insertions(+)
diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
index b1e1baf..c7d449f 100644
--- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
+++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
@@ -148,6 +148,10 @@ public class IPySparkInterpreter extends IPythonInterpreter {
return sparkInterpreter.getSparkVersion().getMajorVersion() == 1;
}
+ public boolean isSpark3() {
+ return sparkInterpreter.getSparkVersion().getMajorVersion() == 3;
+ }
+
public JavaSparkContext getJavaSparkContext() {
return sparkInterpreter.getJavaSparkContext();
}
diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
index f180799..945aa40 100644
--- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
+++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
@@ -221,4 +221,8 @@ public class PySparkInterpreter extends PythonInterpreter {
public boolean isSpark1() {
return sparkInterpreter.getSparkVersion().getMajorVersion() == 1;
}
+
+ public boolean isSpark3() {
+ return sparkInterpreter.getSparkVersion().getMajorVersion() == 3;
+ }
}
diff --git a/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
index 5d199bc..4b9f67f 100644
--- a/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
+++ b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
@@ -39,6 +39,10 @@ java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
intp = gateway.entry_point
+
+if intp.isSpark3():
+ warnings.filterwarnings(action='ignore', module='pyspark.util')
+
jsc = intp.getJavaSparkContext()
java_import(gateway.jvm, "org.apache.spark.sql.*")
diff --git a/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py b/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
index 7132b8a..9a02cd2 100644
--- a/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
+++ b/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
@@ -15,6 +15,8 @@
# limitations under the License.
#
+import warnings
+
from py4j.java_gateway import java_import
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
@@ -24,6 +26,9 @@ from pyspark.sql import SQLContext, Row
intp = gateway.entry_point
+if intp.isSpark3():
+ warnings.filterwarnings(action='ignore', module='pyspark.util')
+
jsc = intp.getJavaSparkContext()
java_import(gateway.jvm, "org.apache.spark.SparkEnv")
java_import(gateway.jvm, "org.apache.spark.SparkConf")
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 7dc888f..617587a 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -187,6 +187,8 @@ public class SparkInterpreterLauncher extends StandardInterpreterLauncher {
} else {
LOGGER.info("Run Spark under non-secure mode as no keytab and principal is specified");
}
+
+ env.put("PYSPARK_PIN_THREAD", "true");
LOGGER.debug("buildEnvFromProperties: " + env);
return env;