You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2018/08/01 00:33:41 UTC
zeppelin git commit: ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI)
button
Repository: zeppelin
Updated Branches:
refs/heads/master e71fac375 -> 343fd178e
ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button
### What is this PR for?
This is to fix the bug of unable to hide spark jobs.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3668
### How should this be tested?
* Unit test added
* Verify it manually as well.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3104 from zjffdu/ZEPPELIN-3668 and squashes the following commits:
c7ddecc6e [Jeff Zhang] ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/343fd178
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/343fd178
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/343fd178
Branch: refs/heads/master
Commit: 343fd178edf85bb7880ebd4fcecf0b11a7f38561
Parents: e71fac3
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Jul 31 09:04:57 2018 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Wed Aug 1 08:33:36 2018 +0800
----------------------------------------------------------------------
.../zeppelin/spark/NewSparkInterpreterTest.java | 34 +++++++++++++++++---
.../apache/zeppelin/spark/SparkShimsTest.java | 5 ++-
.../org/apache/zeppelin/spark/SparkShims.java | 22 +++++--------
.../org/apache/zeppelin/spark/Spark1Shims.java | 4 ++-
.../org/apache/zeppelin/spark/Spark2Shims.java | 4 ++-
5 files changed, 45 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
index 48be45b..59b39dd 100644
--- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
+++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
@@ -44,7 +44,6 @@ import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -53,6 +52,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -163,7 +163,7 @@ public class NewSparkInterpreterTest {
result = interpreter.interpret(
"case class Bank(age:Integer, job:String, marital : String, education : String, balance : Integer)\n" +
- "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n" +
+ "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n" +
" s => Bank(s(0).toInt, \n" +
" s(1).replaceAll(\"\\\"\", \"\"),\n" +
" s(2).replaceAll(\"\\\"\", \"\"),\n" +
@@ -188,7 +188,7 @@ public class NewSparkInterpreterTest {
"df.show()", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(output.contains(
- "+---+----+\n" +
+ "+---+----+\n" +
"| _1| _2|\n" +
"+---+----+\n" +
"| 1| a|\n" +
@@ -203,7 +203,7 @@ public class NewSparkInterpreterTest {
"df.show()", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(output.contains(
- "+---+----+\n" +
+ "+---+----+\n" +
"| _1| _2|\n" +
"+---+----+\n" +
"| 1| a|\n" +
@@ -318,7 +318,7 @@ public class NewSparkInterpreterTest {
interpretThread.start();
boolean nonZeroProgress = false;
int progress = 0;
- while(interpretThread.isAlive()) {
+ while (interpretThread.isAlive()) {
progress = interpreter.getProgress(context2);
assertTrue(progress >= 0);
if (progress != 0 && progress != 100) {
@@ -463,6 +463,30 @@ public class NewSparkInterpreterTest {
assertEquals(null, interpreter.getSparkContext().getLocalProperty("spark.scheduler.pool"));
}
+ @Test
+ public void testDisableSparkUI() throws InterpreterException {
+ Properties properties = new Properties();
+ properties.setProperty("spark.master", "local");
+ properties.setProperty("spark.app.name", "test");
+ properties.setProperty("zeppelin.spark.maxResult", "100");
+ properties.setProperty("zeppelin.spark.test", "true");
+ properties.setProperty("zeppelin.spark.useNew", "true");
+ properties.setProperty("spark.ui.enabled", "false");
+
+ interpreter = new SparkInterpreter(properties);
+ assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
+ interpreter.setInterpreterGroup(mock(InterpreterGroup.class));
+ InterpreterContext.set(getInterpreterContext());
+ interpreter.open();
+
+ InterpreterContext context = getInterpreterContext();
+ InterpreterResult result = interpreter.interpret("sc.range(1, 10).sum", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+
+ // spark job url is not sent
+ verify(mockRemoteEventClient, never()).onParaInfosReceived(any(Map.class));
+ }
+
@After
public void tearDown() throws InterpreterException {
if (this.interpreter != null) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
index ccebac3..fd47ce2 100644
--- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
+++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java
@@ -108,7 +108,6 @@ public class SparkShimsTest {
@PrepareForTest({BaseZeppelinContext.class, VersionInfo.class})
@PowerMockIgnore({"javax.net.*", "javax.security.*"})
public static class SingleTests {
- @Mock Properties mockProperties;
@Captor ArgumentCaptor<Map<String, String>> argumentCaptor;
SparkShims sparkShims;
@@ -130,7 +129,7 @@ public class SparkShimsTest {
@Test
public void runUnderLocalTest() {
- sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockProperties, mockContext);
+ sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockContext);
Map<String, String> mapValue = argumentCaptor.getValue();
assertTrue(mapValue.keySet().contains("jobUrl"));
@@ -140,7 +139,7 @@ public class SparkShimsTest {
@Test
public void runUnderYarnTest() {
- sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockProperties, mockContext);
+ sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockContext);
Map<String, String> mapValue = argumentCaptor.getValue();
assertTrue(mapValue.keySet().contains("jobUrl"));
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
----------------------------------------------------------------------
diff --git a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
index d308762..1287c57 100644
--- a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
+++ b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java
@@ -104,26 +104,20 @@ public abstract class SparkShims {
protected void buildSparkJobUrl(String master,
String sparkWebUrl,
int jobId,
- Properties jobProperties,
InterpreterContext context) {
- String uiEnabled = jobProperties.getProperty("spark.ui.enabled");
String jobUrl = sparkWebUrl + "/jobs/job?id=" + jobId;
- // Button visible if Spark UI property not set, set as invalid boolean or true
- boolean showSparkUI =
- uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false");
String version = VersionInfo.getVersion();
if (master.toLowerCase().contains("yarn") && !supportYarn6615(version)) {
jobUrl = sparkWebUrl + "/jobs";
}
- if (showSparkUI && jobUrl != null) {
- Map<String, String> infos = new java.util.HashMap<String, String>();
- infos.put("jobUrl", jobUrl);
- infos.put("label", "SPARK JOB");
- infos.put("tooltip", "View in Spark web UI");
- infos.put("noteId", context.getNoteId());
- infos.put("paraId", context.getParagraphId());
- context.getIntpEventClient().onParaInfosReceived(infos);
- }
+
+ Map<String, String> infos = new java.util.HashMap<String, String>();
+ infos.put("jobUrl", jobUrl);
+ infos.put("label", "SPARK JOB");
+ infos.put("tooltip", "View in Spark web UI");
+ infos.put("noteId", context.getNoteId());
+ infos.put("paraId", context.getParagraphId());
+ context.getIntpEventClient().onParaInfosReceived(infos);
}
/**
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
----------------------------------------------------------------------
diff --git a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
index db0727c..3981204 100644
--- a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
+++ b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java
@@ -38,7 +38,9 @@ public class Spark1Shims extends SparkShims {
sc.addSparkListener(new JobProgressListener(sc.getConf()) {
@Override
public void onJobStart(SparkListenerJobStart jobStart) {
- buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context);
+ if (sc.getConf().getBoolean("spark.ui.enabled", true)) {
+ buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), context);
+ }
}
});
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
----------------------------------------------------------------------
diff --git a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
index 177b0ac..5f0cf87 100644
--- a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
+++ b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java
@@ -39,7 +39,9 @@ public class Spark2Shims extends SparkShims {
sc.addSparkListener(new SparkListener() {
@Override
public void onJobStart(SparkListenerJobStart jobStart) {
- buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context);
+ if (sc.getConf().getBoolean("spark.ui.enabled", true)) {
+ buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), context);
+ }
}
});
}