You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/10/10 05:48:07 UTC
[zeppelin] branch branch-0.9 updated: [ZEPPELIN-4409]. Set
spark.app.name to know which user is running which notebook
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 4494fec [ZEPPELIN-4409]. Set spark.app.name to know which user is running which notebook
4494fec is described below
commit 4494fec0cf62a307f5a7645ac094eeb123528e16
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Fri Oct 9 17:21:48 2020 +0800
[ZEPPELIN-4409]. Set spark.app.name to know which user is running which notebook
### What is this PR for?
Use the interpreter group id as the spark app name if it is not set. So that user can easily figure out which yarn app is what he is looking for no matter it is note isolated or user isolated.
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4409
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3927 from zjffdu/ZEPPELIN-4409 and squashes the following commits:
d4b72a729 [Jeff Zhang] remove unnecessary code
28051f8af [Jeff Zhang] [ZEPPELIN-4409]. Set spark.app.name to know which user is running which notebook
(cherry picked from commit e4248082223dc132a0afa38ba254009f7043ee46)
Signed-off-by: Jeff Zhang <zj...@apache.org>
---
.../src/main/resources/interpreter-setting.json | 2 +-
.../launcher/SparkInterpreterLauncher.java | 6 ++++++
.../launcher/SparkInterpreterLauncherTest.java | 20 +++++++++++++-------
3 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/spark/interpreter/src/main/resources/interpreter-setting.json b/spark/interpreter/src/main/resources/interpreter-setting.json
index 0c832fb..9d70f9c 100644
--- a/spark/interpreter/src/main/resources/interpreter-setting.json
+++ b/spark/interpreter/src/main/resources/interpreter-setting.json
@@ -29,7 +29,7 @@
"spark.app.name": {
"envName": "",
"propertyName": "spark.app.name",
- "defaultValue": "Zeppelin",
+ "defaultValue": "",
"description": "The name of spark application.",
"type": "string"
},
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 66340ca..7de01db 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -78,6 +78,12 @@ public class SparkInterpreterLauncher extends StandardInterpreterLauncher {
}
}
+ // set spark.app.name if it is not set or empty
+ if (!sparkProperties.containsKey("spark.app.name") ||
+ StringUtils.isBlank(sparkProperties.getProperty("spark.app.name"))) {
+ sparkProperties.setProperty("spark.app.name", context.getInterpreterGroupId());
+ }
+
setupPropertiesForPySpark(sparkProperties);
setupPropertiesForSparkR(sparkProperties);
if (isYarnMode() && getDeployMode().equals("cluster")) {
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index be16d87..4a7189b 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -108,7 +108,7 @@ public class SparkInterpreterLauncherTest {
assertEquals(sparkHome, interpreterProcess.getEnv().get("SPARK_HOME"));
assertFalse(interpreterProcess.getEnv().containsKey("ENV_1"));
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --conf spark.files=file_1" +
- " --conf spark.jars=jar_1 --conf spark.master=local[*]"),
+ " --conf spark.jars=jar_1 --conf spark.app.name=intpGroupId --conf spark.master=local[*]"),
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}
@@ -140,7 +140,7 @@ public class SparkInterpreterLauncherTest {
String sparkFiles = "file_1";
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --conf spark.yarn.dist.archives=" + sparkrZip +
" --conf spark.files=" + sparkFiles + " --conf spark.jars=" + sparkJars +
- " --conf spark.yarn.isPython=true --conf spark.master=yarn-client"),
+ " --conf spark.yarn.isPython=true --conf spark.app.name=intpGroupId --conf spark.master=yarn-client"),
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}
@@ -174,7 +174,7 @@ public class SparkInterpreterLauncherTest {
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --conf spark.yarn.dist.archives=" + sparkrZip +
" --conf spark.files=" + sparkFiles + " --conf spark.jars=" + sparkJars +
" --conf spark.submit.deployMode=client" +
- " --conf spark.yarn.isPython=true --conf spark.master=yarn"),
+ " --conf spark.yarn.isPython=true --conf spark.app.name=intpGroupId --conf spark.master=yarn"),
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}
@@ -212,6 +212,7 @@ public class SparkInterpreterLauncherTest {
" --conf spark.files=" + sparkFiles + " --conf spark.jars=" + sparkJars +
" --conf spark.yarn.isPython=true" +
" --conf spark.yarn.submit.waitAppCompletion=false" +
+ " --conf spark.app.name=intpGroupId" +
" --conf spark.master=yarn-cluster"),
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}
@@ -253,10 +254,12 @@ public class SparkInterpreterLauncherTest {
String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
String sparkFiles = "file_1," + zeppelinHome + "/conf/log4j_yarn_cluster.properties";
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --conf spark.yarn.dist.archives=" + sparkrZip +
+ " --conf spark.yarn.isPython=true --conf spark.app.name=intpGroupId" +
" --conf spark.yarn.maxAppAttempts=1" +
+ " --conf spark.master=yarn" +
" --conf spark.files=" + sparkFiles + " --conf spark.jars=" + sparkJars +
- " --conf spark.submit.deployMode=cluster --conf spark.yarn.isPython=true" +
- " --conf spark.yarn.submit.waitAppCompletion=false --conf spark.master=yarn" +
+ " --conf spark.submit.deployMode=cluster" +
+ " --conf spark.yarn.submit.waitAppCompletion=false" +
" --proxy-user user1"),
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
Files.deleteIfExists(Paths.get(localRepoPath.toAbsolutePath().toString(), "test.jar"));
@@ -300,11 +303,14 @@ public class SparkInterpreterLauncherTest {
// escape special characters
String sparkFiles = "{}," + zeppelinHome + "/conf/log4j_yarn_cluster.properties";
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --conf spark.yarn.dist.archives=" + sparkrZip +
+ " --conf spark.yarn.isPython=true" +
+ " --conf spark.app.name=intpGroupId" +
" --conf spark.yarn.maxAppAttempts=1" +
+ " --conf spark.master=yarn" +
" --conf spark.files=" + sparkFiles + " --conf spark.jars=" + sparkJars +
- " --conf spark.submit.deployMode=cluster --conf spark.yarn.isPython=true" +
+ " --conf spark.submit.deployMode=cluster" +
" --conf spark.yarn.submit.waitAppCompletion=false" +
- " --conf spark.master=yarn --proxy-user user1"),
+ " --proxy-user user1"),
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
FileUtils.deleteDirectory(localRepoPath.toFile());
}