You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2018/06/12 10:20:53 UTC
zeppelin git commit: ZEPPELIN-3422 Add JMX support
Repository: zeppelin
Updated Branches:
refs/heads/master b7d98b3f1 -> 6137c2a81
ZEPPELIN-3422 Add JMX support
### What is this PR for?
Providing JMX support optionally and add some MBeans for checking a basic status
### What type of PR is it?
[Improvement]
### Todos
* [x] - Add JMX options to `JAVA_OPTS`
* [x] - Add Mbeans
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3422
### How should this be tested?
* You can connect JMX with the port # of 9996 by defining `ZEPPELIN_JMX_ENABLE`
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jongyoul Lee <jo...@gmail.com>
Closes #3014 from jongyoul/ZEPPELIN-3422 and squashes the following commits:
86d9a003e [Jongyoul Lee] Remove `ZEPPELIN_JMX_OVERRIDE` option to avoid security issue Change the logic to check if `ZEPPELIN_ENABLE_JMX` is `true` or not
44c493c63 [Jongyoul Lee] Align spaces
b3b82dcfa [Jongyoul Lee] Align spaces Add logic to check if JMX enables or not
c177ecdd6 [Jongyoul Lee] Fix Rat
5ed993bca [Jongyoul Lee] Revert unwanted file
f6de068ce [Jongyoul Lee] Add commend line options and environment variables
8f4f1c8df [Jongyoul Lee] Add MBeans Add NotebookServerMBean to track connected users and to send message to them Add InterpreterSettingManagerMBean to get running interpreters
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/6137c2a8
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/6137c2a8
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/6137c2a8
Branch: refs/heads/master
Commit: 6137c2a8136b31ae934662c2909ad1b54070a053
Parents: b7d98b3
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Tue Jun 12 11:11:09 2018 +0900
Committer: Jongyoul Lee <jo...@apache.org>
Committed: Tue Jun 12 19:20:48 2018 +0900
----------------------------------------------------------------------
bin/common.cmd | 8 ++++++
bin/common.sh | 12 ++++++++
conf/zeppelin-env.cmd.template | 2 ++
conf/zeppelin-env.sh.template | 2 ++
.../apache/zeppelin/server/ZeppelinServer.java | 26 ++++++++++++++++++
.../apache/zeppelin/socket/NotebookServer.java | 29 ++++++++++++++++----
.../zeppelin/socket/NotebookServerMBean.java | 26 ++++++++++++++++++
.../websocket/websocket-event.factory.js | 2 ++
.../interpreter/InterpreterSettingManager.java | 15 +++++++++-
.../InterpreterSettingManagerMBean.java | 24 ++++++++++++++++
.../zeppelin/notebook/socket/Message.java | 3 +-
11 files changed, 141 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/bin/common.cmd
----------------------------------------------------------------------
diff --git a/bin/common.cmd b/bin/common.cmd
index 13f33e5..21657c1 100644
--- a/bin/common.cmd
+++ b/bin/common.cmd
@@ -71,6 +71,14 @@ if not defined ZEPPELIN_JAVA_OPTS (
set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
)
+if defined ZEPPELIN_JMX_ENABLE (
+ if not defined ZEPPELIN_JMX_PORT (
+ set ZEPPELIN_JMX_PORT="9996"
+ }
+ set JMX_JAVA_OPTS=" -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+ set ZEPPELIN_JAVA_OPTS=%JMX_JAVA_OPTS% %ZEPPELIN_JAVA_OPTS
+)
+
if not defined JAVA_OPTS (
set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
) else (
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/bin/common.sh
----------------------------------------------------------------------
diff --git a/bin/common.sh b/bin/common.sh
index 6447ec8..c56fbd4 100644
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -121,6 +121,18 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS
+if [[ x"${ZEPPELIN_JMX_ENABLE}" == x"true" ]]; then
+ if [[ -z "${ZEPPELIN_JMX_PORT}" ]]; then
+ ZEPPELIN_JMX_PORT="9996"
+ fi
+ JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote"
+ JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT}"
+ JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.authenticate=false"
+ JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.ssl=false"
+ JAVA_OPTS="${JMX_JAVA_OPTS} ${JAVA_OPTS}"
+fi
+export JAVA_OPTS
+
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/conf/zeppelin-env.cmd.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template
index e40429a..e69f23b 100644
--- a/conf/zeppelin-env.cmd.template
+++ b/conf/zeppelin-env.cmd.template
@@ -22,6 +22,8 @@ REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set Z
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default -Xmx1024m -Xms1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
+REM set ZEPPELIN_JMX_ENABLE REM Enable JMX feature by defining it like "true"
+REM set ZEPPELIN_JMX_PORT REM Port number which JMX uses. Default: "9996"
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index c7204bd..5aab0e0 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -23,6 +23,8 @@
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
# export ZEPPELIN_SSL_PORT # ssl port (used when ssl environment variable is set to true)
+# export ZEPPELIN_JMX_ENABLE # Enable JMX feature by defining "true"
+# export ZEPPELIN_JMX_PORT # Port number which JMX uses. Default: "9996"
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 3e92a9a..b699d2f 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -16,7 +16,14 @@
*/
package org.apache.zeppelin.server;
+import java.lang.management.ManagementFactory;
import java.util.Collection;
+import javax.management.InstanceAlreadyExistsException;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
import org.apache.commons.lang.StringUtils;
import org.apache.shiro.UnavailableSecurityManagerException;
import org.apache.shiro.realm.Realm;
@@ -193,6 +200,25 @@ public class ZeppelinServer extends Application {
notebook.addNotebookEventListener(heliumApplicationFactory);
notebook.addNotebookEventListener(notebookWsServer.getNotebookInformationListener());
this.interpreterService = new InterpreterService(conf, interpreterSettingManager);
+
+ // Register MBean
+ if ("true".equals(System.getenv("ZEPPELIN_ENABLE_JMX"))) {
+ MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
+ try {
+ mBeanServer.registerMBean(
+ notebookWsServer,
+ new ObjectName("org.apache.zeppelin:type=" + NotebookServer.class.getSimpleName()));
+ mBeanServer.registerMBean(
+ interpreterSettingManager,
+ new ObjectName(
+ "org.apache.zeppelin:type=" + InterpreterSettingManager.class.getSimpleName()));
+ } catch (InstanceAlreadyExistsException
+ | MBeanRegistrationException
+ | MalformedObjectNameException
+ | NotCompliantMBeanException e) {
+ LOG.error("Failed to register MBeans", e);
+ }
+ }
}
public static void main(String[] args) throws InterruptedException {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
index ca0b0ab..4a823d9 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
@@ -100,13 +100,14 @@ import org.apache.zeppelin.utils.InterpreterBindingUtils;
import org.apache.zeppelin.utils.SecurityUtils;
import org.bitbucket.cowwoc.diffmatchpatch.DiffMatchPatch;
-
-/**
- * Zeppelin websocket service.
- */
+/** Zeppelin websocket service. */
public class NotebookServer extends WebSocketServlet
- implements NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener,
- RemoteInterpreterProcessListener, ApplicationEventListener {
+ implements NotebookSocketListener,
+ JobListenerFactory,
+ AngularObjectRegistryListener,
+ RemoteInterpreterProcessListener,
+ ApplicationEventListener,
+ NotebookServerMBean {
/**
* Job manager service type.
@@ -2706,4 +2707,20 @@ public class NotebookServer extends WebSocketServlet
broadcastNoteForms(note);
}
}
+
+ @Override
+ public Set<String> getConnectedUsers() {
+ Set<String> connectionList = Sets.newHashSet();
+ for (NotebookSocket notebookSocket : connectedSockets) {
+ connectionList.add(notebookSocket.getUser());
+ }
+ return connectionList;
+ }
+
+ @Override
+ public void sendMessage(String message) {
+ Message m = new Message(OP.NOTICE);
+ m.data.put("notice", message);
+ broadcast(m);
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServerMBean.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServerMBean.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServerMBean.java
new file mode 100644
index 0000000..f94af89
--- /dev/null
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServerMBean.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.socket;
+
+import java.util.Set;
+
+public interface NotebookServerMBean {
+ Set<String> getConnectedUsers();
+
+ void sendMessage(String message);
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-web/src/components/websocket/websocket-event.factory.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/components/websocket/websocket-event.factory.js b/zeppelin-web/src/components/websocket/websocket-event.factory.js
index 91d7076..25a3336 100644
--- a/zeppelin-web/src/components/websocket/websocket-event.factory.js
+++ b/zeppelin-web/src/components/websocket/websocket-event.factory.js
@@ -185,6 +185,8 @@ function WebsocketEventFactory($rootScope, $websocket, $location, baseUrlSrv, ng
ngToast.info(data.message);
} else if (op === 'INTERPRETER_INSTALL_RESULT') {
ngToast.info(data.message);
+ } else if (op === 'NOTICE') {
+ ngToast.info(data.notice);
} else {
console.error(`unknown websocket op: ${op}`);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
index 125bc18..d595a5d 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -81,7 +81,7 @@ import java.util.Map;
* Besides that InterpreterSettingManager also manage the interpreter setting binding.
* TODO(zjffdu) We could move it into another separated component.
*/
-public class InterpreterSettingManager {
+public class InterpreterSettingManager implements InterpreterSettingManagerMBean {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSettingManager.class);
private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
@@ -958,4 +958,17 @@ public class InterpreterSettingManager {
}
}
}
+
+ @Override
+ public Set<String> getRunningInterpreters() {
+ Set<String> runningInterpreters = Sets.newHashSet();
+ for (Map.Entry<String, InterpreterSetting> entry : interpreterSettings.entrySet()) {
+ for (ManagedInterpreterGroup mig : entry.getValue().getAllInterpreterGroups()) {
+ if (null != mig.getRemoteInterpreterProcess()) {
+ runningInterpreters.add(entry.getKey());
+ }
+ }
+ }
+ return runningInterpreters;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerMBean.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerMBean.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerMBean.java
new file mode 100644
index 0000000..3cc3b08
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerMBean.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import java.util.Set;
+
+public interface InterpreterSettingManagerMBean {
+ Set<String> getRunningInterpreters();
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6137c2a8/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
index 1daa008..7a042ea 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
@@ -190,7 +190,8 @@ public class Message implements JsonSerializable {
INTERPRETER_INSTALL_STARTED, // [s-c] start to download an interpreter
INTERPRETER_INSTALL_RESULT, // [s-c] Status of an interpreter installation
COLLABORATIVE_MODE_STATUS, // [s-c] collaborative mode status
- PATCH_PARAGRAPH // [c-s][s-c] patch editor text
+ PATCH_PARAGRAPH, // [c-s][s-c] patch editor text
+ NOTICE // [s-c] Notice
}
private static final Gson gson = new Gson();