You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2016/01/20 23:16:24 UTC
[1/4] storm git commit: adds comments about licensing the profiler
feature
Repository: storm
Updated Branches:
refs/heads/1.x-branch 1bc086442 -> bb503d287
adds comments about licensing the profiler feature
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9b5a1bf0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9b5a1bf0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9b5a1bf0
Branch: refs/heads/1.x-branch
Commit: 9b5a1bf055a237cfa339c9ab0a7a8af56d6efb32
Parents: 67d9181
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Jan 14 16:44:14 2016 -0600
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 20 15:19:35 2016 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 6 +++++-
storm-core/src/jvm/org/apache/storm/Config.java | 4 +++-
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9b5a1bf0/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 8722a5f..8873d12 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -166,8 +166,12 @@ supervisor.cpu.capacity: 400.0
worker.heap.memory.mb: 768
worker.childopts: "-Xmx%HEAP-MEM%m -XX:+PrintGCDetails -Xloggc:artifacts/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump"
worker.gc.childopts: ""
-worker.profiler.childopts: "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder"
+
+# Unlocking commercial features requires a special license from Oracle.
+# See http://www.oracle.com/technetwork/java/javase/terms/products/index.html
+# For this reason, profiler features are disabled by default.
worker.profiler.enabled: false
+worker.profiler.childopts: "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder"
worker.profiler.command: "flight.bash"
worker.heartbeat.frequency.secs: 1
http://git-wip-us.apache.org/repos/asf/storm/blob/9b5a1bf0/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index e1f2281..66c330f 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -1374,7 +1374,9 @@ public class Config extends HashMap<String, Object> {
public static final String WORKER_PROFILER_CHILDOPTS = "worker.profiler.childopts";
/**
- * This configuration would enable or disable component page profiing and debugging for workers.
+ * Enable profiling of worker JVMs using Oracle's Java Flight Recorder.
+ * Unlocking commercial features requires a special license from Oracle.
+ * See http://www.oracle.com/technetwork/java/javase/terms/products/index.html
*/
@isBoolean
public static final String WORKER_PROFILER_ENABLED = "worker.profiler.enabled";
[2/4] storm git commit: Fixes profiling/debugging out of the box
Posted by pt...@apache.org.
Fixes profiling/debugging out of the box
Ships the flight.bash script
Disables UI display of Profiling and Debugging if on windows
Changes worker.profiler.enabled to control only profiling, not other debugging actions
Changes default of worker.profiler.enabled to false
Adds missing UI REST API data.
Populates user context on profiler UI routes
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/67d9181d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/67d9181d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/67d9181d
Branch: refs/heads/1.x-branch
Commit: 67d9181d2a9c825f33a2cf7095cda003fd5b84bc
Parents: 6912aab
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Wed Jan 13 16:34:14 2016 -0600
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Wed Jan 20 15:19:35 2016 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 2 +-
.../clj/org/apache/storm/daemon/supervisor.clj | 5 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 123 ++++++++++---------
storm-core/src/ui/public/component.html | 2 +-
.../templates/component-page-template.html | 2 +
storm-dist/binary/src/main/assembly/binary.xml | 1 +
6 files changed, 76 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/67d9181d/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index c15f0b3..8722a5f 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -167,7 +167,7 @@ worker.heap.memory.mb: 768
worker.childopts: "-Xmx%HEAP-MEM%m -XX:+PrintGCDetails -Xloggc:artifacts/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump"
worker.gc.childopts: ""
worker.profiler.childopts: "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder"
-worker.profiler.enabled: true
+worker.profiler.enabled: false
worker.profiler.command: "flight.bash"
worker.heartbeat.frequency.secs: 1
http://git-wip-us.apache.org/repos/asf/storm/blob/67d9181d/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index e4b44b0..6e6ecc5 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -709,7 +709,10 @@
stormid->profiler-actions @(:stormid->profiler-actions supervisor)
storm-cluster-state (:storm-cluster-state supervisor)
hostname (:my-hostname supervisor)
- profile-cmd (conf WORKER-PROFILER-COMMAND)
+ storm-home (System/getProperty "storm.home")
+ profile-cmd (str (clojure.java.io/file storm-home
+ "bin"
+ (conf WORKER-PROFILER-COMMAND)))
new-assignment @(:curr-assignment supervisor)
assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)]
(doseq [[storm-id profiler-actions] stormid->profiler-actions]
http://git-wip-us.apache.org/repos/asf/storm/blob/67d9181d/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj
index 9bdc0e9..1139eb2 100644
--- a/storm-core/src/clj/org/apache/storm/ui/core.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/core.clj
@@ -105,14 +105,6 @@
(throw (AuthorizationException.
(str "UI request '" op "' for '" user "' user is not authorized")))))))))
-
-(defn assert-authorized-profiler-action
- [op]
- (if-not (*STORM-CONF* WORKER-PROFILER-ENABLED)
- (throw (AuthorizationException.
- (str "UI request for profiler action '" op "' is disabled.")))))
-
-
(defn executor-summary-type
[topology ^ExecutorSummary s]
(component-type topology (.get_component_id s)))
@@ -863,6 +855,7 @@
(.get_eventlog_host comp-page-info)
(.get_eventlog_port comp-page-info)
secure?)
+ "profilingAndDebuggingCapable" (not on-windows?)
"profileActionEnabled" (*STORM-CONF* WORKER-PROFILER-ENABLED)
"profilerActive" (if (*STORM-CONF* WORKER-PROFILER-ENABLED)
(get-active-profile-actions nimbus topology-id component)
@@ -921,6 +914,15 @@
[servlet-request]
(.getUserName http-creds-handler servlet-request))
+(defn json-profiling-disabled
+ "Return a JSON response communicating that profiling is disabled and
+ therefore unavailable."
+ [callback]
+ (json-response {"status" "disabled",
+ "message" "Profiling is not enabled on this server"}
+ callback
+ :status 501))
+
(defroutes main-routes
(GET "/api/v1/cluster/configuration" [& m]
(mark! ui:num-cluster-configuration-http-requests)
@@ -1107,62 +1109,71 @@
(GET "/api/v1/topology/:id/profiling/start/:host-port/:timeout"
[:as {:keys [servlet-request]} id host-port timeout & m]
- (thrift/with-configured-nimbus-connection nimbus
- (assert-authorized-user "setWorkerProfiler" (topology-config id))
- (assert-authorized-profiler-action "start")
- (let [[host, port] (split host-port #":")
- nodeinfo (NodeInfo. host (set [(Long. port)]))
- timestamp (+ (System/currentTimeMillis) (* 60000 (Long. timeout)))
- request (ProfileRequest. nodeinfo
- ProfileAction/JPROFILE_STOP)]
- (.set_time_stamp request timestamp)
- (.setWorkerProfiler nimbus id request)
- (json-response {"status" "ok"
- "id" host-port
- "timeout" timeout
- "dumplink" (worker-dump-link
- host
- port
- id)}
- (m "callback")))))
+ (if (get *STORM-CONF* WORKER-PROFILER-ENABLED)
+ (do
+ (populate-context! servlet-request)
+ (thrift/with-configured-nimbus-connection nimbus
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (let [[host, port] (split host-port #":")
+ nodeinfo (NodeInfo. host (set [(Long. port)]))
+ timestamp (+ (System/currentTimeMillis) (* 60000 (Long. timeout)))
+ request (ProfileRequest. nodeinfo
+ ProfileAction/JPROFILE_STOP)]
+ (.set_time_stamp request timestamp)
+ (.setWorkerProfiler nimbus id request)
+ (json-response {"status" "ok"
+ "id" host-port
+ "timeout" timeout
+ "dumplink" (worker-dump-link
+ host
+ port
+ id)}
+ (m "callback")))))
+ (json-profiling-disabled (m "callback"))))
(GET "/api/v1/topology/:id/profiling/stop/:host-port"
[:as {:keys [servlet-request]} id host-port & m]
- (thrift/with-configured-nimbus-connection nimbus
- (assert-authorized-user "setWorkerProfiler" (topology-config id))
- (assert-authorized-profiler-action "stop")
- (let [[host, port] (split host-port #":")
- nodeinfo (NodeInfo. host (set [(Long. port)]))
- timestamp 0
- request (ProfileRequest. nodeinfo
- ProfileAction/JPROFILE_STOP)]
- (.set_time_stamp request timestamp)
- (.setWorkerProfiler nimbus id request)
- (json-response {"status" "ok"
- "id" host-port}
- (m "callback")))))
-
+ (if (get *STORM-CONF* WORKER-PROFILER-ENABLED)
+ (do
+ (populate-context! servlet-request)
+ (thrift/with-configured-nimbus-connection nimbus
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (let [[host, port] (split host-port #":")
+ nodeinfo (NodeInfo. host (set [(Long. port)]))
+ timestamp 0
+ request (ProfileRequest. nodeinfo
+ ProfileAction/JPROFILE_STOP)]
+ (.set_time_stamp request timestamp)
+ (.setWorkerProfiler nimbus id request)
+ (json-response {"status" "ok"
+ "id" host-port}
+ (m "callback")))))
+ (json-profiling-disabled (m "callback"))))
+
(GET "/api/v1/topology/:id/profiling/dumpprofile/:host-port"
[:as {:keys [servlet-request]} id host-port & m]
- (thrift/with-configured-nimbus-connection nimbus
- (assert-authorized-user "setWorkerProfiler" (topology-config id))
- (assert-authorized-profiler-action "dumpprofile")
- (let [[host, port] (split host-port #":")
- nodeinfo (NodeInfo. host (set [(Long. port)]))
- timestamp (System/currentTimeMillis)
- request (ProfileRequest. nodeinfo
- ProfileAction/JPROFILE_DUMP)]
- (.set_time_stamp request timestamp)
- (.setWorkerProfiler nimbus id request)
- (json-response {"status" "ok"
- "id" host-port}
- (m "callback")))))
+ (if (get *STORM-CONF* WORKER-PROFILER-ENABLED)
+ (do
+ (populate-context! servlet-request)
+ (thrift/with-configured-nimbus-connection nimbus
+ (assert-authorized-user "setWorkerProfiler" (topology-config id))
+ (let [[host, port] (split host-port #":")
+ nodeinfo (NodeInfo. host (set [(Long. port)]))
+ timestamp (System/currentTimeMillis)
+ request (ProfileRequest. nodeinfo
+ ProfileAction/JPROFILE_DUMP)]
+ (.set_time_stamp request timestamp)
+ (.setWorkerProfiler nimbus id request)
+ (json-response {"status" "ok"
+ "id" host-port}
+ (m "callback")))))
+ (json-profiling-disabled (m "callback"))))
(GET "/api/v1/topology/:id/profiling/dumpjstack/:host-port"
[:as {:keys [servlet-request]} id host-port & m]
+ (populate-context! servlet-request)
(thrift/with-configured-nimbus-connection nimbus
(assert-authorized-user "setWorkerProfiler" (topology-config id))
- (assert-authorized-profiler-action "dumpjstack")
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
@@ -1176,9 +1187,9 @@
(GET "/api/v1/topology/:id/profiling/restartworker/:host-port"
[:as {:keys [servlet-request]} id host-port & m]
+ (populate-context! servlet-request)
(thrift/with-configured-nimbus-connection nimbus
(assert-authorized-user "setWorkerProfiler" (topology-config id))
- (assert-authorized-profiler-action "restartworker")
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
@@ -1192,9 +1203,9 @@
(GET "/api/v1/topology/:id/profiling/dumpheap/:host-port"
[:as {:keys [servlet-request]} id host-port & m]
+ (populate-context! servlet-request)
(thrift/with-configured-nimbus-connection nimbus
(assert-authorized-user "setWorkerProfiler" (topology-config id))
- (assert-authorized-profiler-action "dumpheap")
(let [[host, port] (split host-port #":")
nodeinfo (NodeInfo. host (set [(Long. port)]))
timestamp (System/currentTimeMillis)
http://git-wip-us.apache.org/repos/asf/storm/blob/67d9181d/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html
index 06c485c..88187b6 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -219,7 +219,7 @@ $(document).ready(function() {
componentActions.append(Mustache.render($(template).filter("#component-actions-template").html(),buttonJsonData));
});
- if(response["profileActionEnabled"] == true) {
+ if (response["profilingAndDebuggingCapable"] == true) {
jsError(function () {
var part = $(template).filter('#profiler-active-partial').html();
var partials = {"profilerActive": part};
http://git-wip-us.apache.org/repos/asf/storm/blob/67d9181d/storm-core/src/ui/public/templates/component-page-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/component-page-template.html b/storm-core/src/ui/public/templates/component-page-template.html
index 79a21cd..3626291 100644
--- a/storm-core/src/ui/public/templates/component-page-template.html
+++ b/storm-core/src/ui/public/templates/component-page-template.html
@@ -254,7 +254,9 @@
</td>
<td>
<span id="workerActionButtons">
+ {{#profileActionEnabled}}
<input type="button" value="Start" name="start" onClick="start_profiling()" class="btn btn-secondary" disabled/>
+ {{/profileActionEnabled}}
<input type="button" value="JStack" name="jstack" onClick="dump_jstacks()" class="btn btn-secondary" disabled/>
<input type="button" value="Restart Worker" name="jvmrestart" onClick="restart_worker_jvms()" class="btn btn-secondary" disabled/>
<input type="button" value="Heap" name="heap" onClick="dump_heaps()" class="btn btn-secondary" disabled/>
http://git-wip-us.apache.org/repos/asf/storm/blob/67d9181d/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index b2a6104..6667474 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -42,6 +42,7 @@
<outputDirectory>bin</outputDirectory>
<includes>
<include>storm*</include>
+ <include>flight.bash</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
[3/4] storm git commit: Merge branch
'storm-1452-profiler-broken-by-default' of github.com:d2r/storm into
1.x-branch
Posted by pt...@apache.org.
Merge branch 'storm-1452-profiler-broken-by-default' of github.com:d2r/storm into 1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/76462f2a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/76462f2a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/76462f2a
Branch: refs/heads/1.x-branch
Commit: 76462f2ae39c7701b3dc5d8508f3ab40a6dec8fd
Parents: 1bc0864 9b5a1bf
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Jan 20 17:15:26 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Jan 20 17:15:26 2016 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 6 +-
.../clj/org/apache/storm/daemon/supervisor.clj | 5 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 123 ++++++++++---------
storm-core/src/jvm/org/apache/storm/Config.java | 4 +-
storm-core/src/ui/public/component.html | 2 +-
.../templates/component-page-template.html | 2 +
storm-dist/binary/src/main/assembly/binary.xml | 1 +
7 files changed, 83 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/76462f2a/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
[4/4] storm git commit: add STORM-1452 to changelog
Posted by pt...@apache.org.
add STORM-1452 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bb503d28
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bb503d28
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bb503d28
Branch: refs/heads/1.x-branch
Commit: bb503d287175b4d536426392d05ff26984768cfb
Parents: 76462f2
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Jan 20 17:16:14 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Jan 20 17:16:14 2016 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bb503d28/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdad3da..75086d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.0.0
+ * STORM-1452: Fixes profiling/debugging out of the box
* STORM-1406: Add MQTT Support
* STORM-1481: avoid Math.abs(Integer) get a negative value
* STORM-1473: enable log search for daemon logs