You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/11/04 18:18:47 UTC
[04/10] storm git commit: Make worker profiling optional and pluggable
Make worker profiling optional and pluggable
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d2f9305a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d2f9305a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d2f9305a
Branch: refs/heads/master
Commit: d2f9305abb16ee3573abaab56de7b1e673168cd5
Parents: 0c2021e
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Mon Nov 2 23:51:33 2015 -0600
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Nov 2 23:51:33 2015 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 2 +
docs/DYNAMIC_WORKER_PROFILING.md | 5 +++
.../clj/backtype/storm/daemon/supervisor.clj | 39 ++++++++++----------
storm-core/src/clj/backtype/storm/ui/core.clj | 31 ++++++++++++----
storm-core/src/jvm/backtype/storm/Config.java | 15 ++++++++
storm-core/src/ui/public/component.html | 12 +++---
6 files changed, 72 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 84babc3..322d386 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -145,6 +145,8 @@ worker.heap.memory.mb: 768
worker.childopts: "-Xmx%HEAP-MEM%m -XX:+PrintGCDetails -Xloggc:artifacts/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump"
worker.gc.childopts: ""
worker.profiler.childopts: "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder"
+worker.profiler.enabled: true
+worker.profiler.command: "flight.bash"
worker.heartbeat.frequency.secs: 1
# check whether dynamic log levels can be reset from DEBUG to INFO in workers
http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/docs/DYNAMIC_WORKER_PROFILING.md
----------------------------------------------------------------------
diff --git a/docs/DYNAMIC_WORKER_PROFILING.md b/docs/DYNAMIC_WORKER_PROFILING.md
index 727322f..4b55a80 100644
--- a/docs/DYNAMIC_WORKER_PROFILING.md
+++ b/docs/DYNAMIC_WORKER_PROFILING.md
@@ -22,3 +22,8 @@ Click on "My Dump Files" to go the logviewer UI for list of worker specific dump
+Configuration
+-------------
+
+The "worker.profiler.command" can be configured to point to specific pluggable profiler, heapdump commands. The "worker.profiler.enabled" can be disabled if plugin is not available.
+
http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 8fe6eed..e5740cb 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -530,25 +530,23 @@
(defn java-cmd []
(jvm-cmd "java"))
-(def PROFILE-CMD "flight.bash")
+(defn jmap-dump-cmd [profile-cmd pid target-dir]
+ [profile-cmd pid "jmap" target-dir])
-(defn jmap-dump-cmd [pid target-dir]
- [PROFILE-CMD pid "jmap" target-dir])
+(defn jstack-dump-cmd [profile-cmd pid target-dir]
+ [profile-cmd pid "jstack" target-dir])
-(defn jstack-dump-cmd [pid target-dir]
- [PROFILE-CMD pid "jstack" target-dir])
+(defn jprofile-start [profile-cmd pid]
+ [profile-cmd pid "start"])
-(defn jprofile-start [pid]
- [PROFILE-CMD pid "start" ])
+(defn jprofile-stop [profile-cmd pid target-dir]
+ [profile-cmd pid "stop" target-dir])
-(defn jprofile-stop [pid target-dir]
- [PROFILE-CMD pid "stop" target-dir])
+(defn jprofile-dump [profile-cmd pid workers-artifacts-directory]
+ [profile-cmd pid "dump" workers-artifacts-directory])
-(defn jprofile-dump [pid workers-artifacts-directory]
- [PROFILE-CMD pid "dump" workers-artifacts-directory])
-
-(defn jprofile-jvm-restart [pid]
- [PROFILE-CMD pid "kill" ])
+(defn jprofile-jvm-restart [profile-cmd pid]
+ [profile-cmd pid "kill"])
(defn- delete-topology-profiler-action [storm-cluster-state storm-id profile-action]
(log-message "Deleting profiler action.." profile-action)
@@ -587,6 +585,7 @@
stormid->profiler-actions @(:stormid->profiler-actions supervisor)
storm-cluster-state (:storm-cluster-state supervisor)
hostname (:my-hostname supervisor)
+ profile-cmd (conf WORKER-PROFILER-COMMAND)
new-assignment @(:curr-assignment supervisor)
assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)]
(doseq [[storm-id profiler-actions] stormid->profiler-actions]
@@ -605,14 +604,14 @@
;; Until PROFILER_STOP action is invalid, keep launching profiler start in case worker restarted
;; The profiler plugin script validates if JVM is recording before starting another recording.
command (cond
- (= action ProfileAction/JMAP_DUMP) (jmap-dump-cmd worker-pid target-dir)
- (= action ProfileAction/JSTACK_DUMP) (jstack-dump-cmd worker-pid target-dir)
- (= action ProfileAction/JPROFILE_DUMP) (jprofile-dump worker-pid target-dir)
- (= action ProfileAction/JVM_RESTART) (jprofile-jvm-restart worker-pid)
+ (= action ProfileAction/JMAP_DUMP) (jmap-dump-cmd profile-cmd worker-pid target-dir)
+ (= action ProfileAction/JSTACK_DUMP) (jstack-dump-cmd profile-cmd worker-pid target-dir)
+ (= action ProfileAction/JPROFILE_DUMP) (jprofile-dump profile-cmd worker-pid target-dir)
+ (= action ProfileAction/JVM_RESTART) (jprofile-jvm-restart profile-cmd worker-pid)
(and (not stop?)
(= action ProfileAction/JPROFILE_STOP))
- (jprofile-start worker-pid) ;; Ensure the profiler is still running
- (and stop? (= action ProfileAction/JPROFILE_STOP)) (jprofile-stop worker-pid target-dir))
+ (jprofile-start profile-cmd worker-pid) ;; Ensure the profiler is still running
+ (and stop? (= action ProfileAction/JPROFILE_STOP)) (jprofile-stop profile-cmd worker-pid target-dir))
action-on-exit (fn [exit-code]
(log-message log-prefix " profile-action exited for code: " exit-code)
(if (and (= exit-code 0) stop?)
http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index c6ffe22..60404f0 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -83,6 +83,14 @@
(throw (AuthorizationException.
(str "UI request '" op "' for '" user "' user is not authorized")))))))))
+
+(defn assert-authorized-profiler-action
+ [op]
+ (if-not (*STORM-CONF* WORKER-PROFILER-ENABLED)
+ (throw (AuthorizationException.
+ (str "UI request for profiler action '" op "' is disabled.")))))
+
+
(defn executor-summary-type
[topology ^ExecutorSummary s]
(component-type topology (.get_component_id s)))
@@ -799,7 +807,10 @@
(.get_eventlog_host comp-page-info)
(.get_eventlog_port comp-page-info)
secure?)
- "profilerActive" (get-active-profile-actions nimbus topology-id component)))))
+ "profileActionEnabled" (*STORM-CONF* WORKER-PROFILER-ENABLED)
+ "profilerActive" (if (*STORM-CONF* WORKER-PROFILER-ENABLED)
+ (get-active-profile-actions nimbus topology-id component)
+ [])))))
(defn- level-to-dict [level]
(if level
@@ -1017,7 +1028,8 @@
(let [user (.getUserName http-creds-handler servlet-request)
topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (assert-authorized-profiler-action "start"))
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
@@ -1041,7 +1053,8 @@
(let [user (.getUserName http-creds-handler servlet-request)
topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (assert-authorized-profiler-action "stop"))
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp 0
@@ -1059,7 +1072,8 @@
(let [user (.getUserName http-creds-handler servlet-request)
topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (assert-authorized-profiler-action "dumpprofile"))
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
@@ -1077,7 +1091,8 @@
(let [user (.getUserName http-creds-handler servlet-request)
topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (assert-authorized-profiler-action "dumpjstack"))
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
@@ -1095,7 +1110,8 @@
(let [user (.getUserName http-creds-handler servlet-request)
topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (assert-authorized-profiler-action "restartworker"))
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
@@ -1113,7 +1129,8 @@
(let [user (.getUserName http-creds-handler servlet-request)
topology-conf (from-json
(.getTopologyConf ^Nimbus$Client nimbus id))]
- (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (assert-authorized-profiler-action "dumpheap"))
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 303a3a0..ee6de1b 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -1093,6 +1093,21 @@ public class Config extends HashMap<String, Object> {
public static final String WORKER_PROFILER_CHILDOPTS = "worker.profiler.childopts";
/**
+ * This configuration would enable or disable component page profiing and debugging for workers.
+ */
+ @isBoolean
+ public static final String WORKER_PROFILER_ENABLED = "worker.profiler.enabled";
+
+ /**
+ * The command launched supervisor with worker arguments
+ * pid, action and [target_directory]
+ * Where action is - start profile, stop profile, jstack, heapdump and kill against pid
+ *
+ */
+ @isString
+ public static final String WORKER_PROFILER_COMMAND = "worker.profiler.command";
+
+ /**
* The jvm opts provided to workers launched by this supervisor for GC. All "%ID%" substrings are replaced
* with an identifier for this worker. Because the JVM complains about multiple GC opts the topology
* can override this default value by setting topology.worker.gc.childopts.
http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html
index 60a85cf..ae2463c 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -166,11 +166,13 @@ $(document).ready(function() {
componentActions.append(Mustache.render($(template).filter("#component-actions-template").html(),buttonJsonData));
});
- jsError(function() {
- var part = $(template).filter('#profiler-active-partial').html();
- var partials = {"profilerActive": part};
- profilerControl.append(Mustache.render($(template).filter("#profiling-template").html(), response, partials));
- });
+ if(response["profileActionEnabled"] == true) {
+ jsError(function () {
+ var part = $(template).filter('#profiler-active-partial').html();
+ var partials = {"profilerActive": part};
+ profilerControl.append(Mustache.render($(template).filter("#profiling-template").html(), response, partials));
+ });
+ }
if(response["componentType"] == "spout") {
componentStatsDetail.append(Mustache.render($(template).filter("#spout-stats-detail-template").html(),response));