You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/11/20 23:26:26 UTC
[01/15] storm git commit: Hide resources columns for non-RAS
Repository: storm
Updated Branches:
refs/heads/master ab5b2f92c -> 7fb80a1d1
Hide resources columns for non-RAS
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6b12a0b0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6b12a0b0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6b12a0b0
Branch: refs/heads/master
Commit: 6b12a0b0290b6f3df9c28eb1185d8b294141e60d
Parents: dda1c32
Author: zhuol <zh...@yahoo-inc.com>
Authored: Thu Nov 12 16:10:34 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:35:22 2015 -0600
----------------------------------------------------------------------
storm-core/src/ui/public/index.html | 16 ++++++++++++++++
storm-core/src/ui/public/topology.html | 9 ++++++++-
2 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6b12a0b0/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/index.html b/storm-core/src/ui/public/index.html
index eae423c..6e52923 100644
--- a/storm-core/src/ui/public/index.html
+++ b/storm-core/src/ui/public/index.html
@@ -143,6 +143,13 @@ $(document).ready(function() {
]
});
$('#topology-summary [data-toggle="tooltip"]').tooltip();
+ $.getJSON("/api/v1/cluster/configuration", function(json){
+ var scheduler = json["storm.scheduler"];
+ if (scheduler != "backtype.storm.scheduler.resource.ResourceAwareScheduler"){
+ $('#topology-summary td:nth-child(9),#topology-summary th:nth-child(9)').hide();
+ $('#topology-summary td:nth-child(10),#topology-summary th:nth-child(10)').hide();
+ }
+ });
});
});
$.getJSON("/api/v1/supervisor/summary",function(response,status,jqXHR) {
@@ -156,6 +163,15 @@ $(document).ready(function() {
]
});
$('#supervisor-summary [data-toggle="tooltip"]').tooltip();
+ $.getJSON("/api/v1/cluster/configuration", function(json){
+ var scheduler = json["storm.scheduler"];
+ if (scheduler != "backtype.storm.scheduler.resource.ResourceAwareScheduler"){
+ $('#supervisor-summary td:nth-child(6),#supervisor-summary th:nth-child(6)').hide();
+ $('#supervisor-summary td:nth-child(7),#supervisor-summary th:nth-child(7)').hide();
+ $('#supervisor-summary td:nth-child(8),#supervisor-summary th:nth-child(8)').hide();
+ $('#supervisor-summary td:nth-child(9),#supervisor-summary th:nth-child(9)').hide();
+ }
+ });
});
});
$.getJSON("/api/v1/cluster/configuration",function(response,status,jqXHR) {
http://git-wip-us.apache.org/repos/asf/storm/blob/6b12a0b0/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index e2abab1..b8dd1f9 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -59,7 +59,7 @@
</div>
<div class="row">
<div class="col-md-12">
- <h2>Topology resources</h2>
+ <h2 id="topology-resources-header">Topology resources</h2>
<div id="topology-resources"></div>
</div>
</div>
@@ -293,6 +293,13 @@ $(document).ready(function() {
searchForm.append(Mustache.render($(template).filter("#search-form-template").html(),{id: topologyId}));
topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
topologyResources.append(Mustache.render($(template).filter("#topology-resources-template").html(),response));
+ $.getJSON("/api/v1/cluster/configuration", function(json){
+ var scheduler = json["storm.scheduler"];
+ if (scheduler != "backtype.storm.scheduler.resource.ResourceAwareScheduler"){
+ $('#topology-resources-header').hide();
+ $('#topology-resources').hide();
+ }
+ });
topologyActions.append(Mustache.render($(template).filter("#topology-actions-template").html(),buttonJsonData));
topologyStats.append(Mustache.render($(template).filter("#topology-stats-template").html(),response));
//window, emitted, transferred, complete latency, acked, failed
[05/15] storm git commit: Using config to decide whether to show
resource rather than hard-coded scheduler
Posted by bo...@apache.org.
Using config to decide whether to show resource rather than hard-coded scheduler
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2d993c51
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2d993c51
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2d993c51
Branch: refs/heads/master
Commit: 2d993c515ecde059ad986bf2c3a77cf463d75b74
Parents: 6b12a0b
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Nov 13 13:29:50 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:39:25 2015 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 2 ++
storm-core/src/jvm/backtype/storm/Config.java | 9 +++++++++
storm-core/src/ui/public/index.html | 8 ++++----
storm-core/src/ui/public/topology.html | 4 ++--
4 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2d993c51/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index b498571..af5daa1 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -74,6 +74,8 @@ topology.max.replication.wait.time.sec: 60
nimbus.credential.renewers.freq.secs: 600
nimbus.impersonation.authorizer: "backtype.storm.security.auth.authorizer.ImpersonationAuthorizer"
+scheduler.resource.display: false
+
### ui.* configs are for the master
ui.host: 0.0.0.0
ui.port: 8080
http://git-wip-us.apache.org/repos/asf/storm/blob/2d993c51/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 2830f2f..852757d 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -174,6 +174,15 @@ public class Config extends HashMap<String, Object> {
public static final String STORM_SCHEDULER = "storm.scheduler";
/**
+ * Whether we want to display all the resource capacity and scheduled usage on the UI page.
+ * We suggest to have this variable set if you are using any kind of resource-related scheduler.
+ *
+ * If this is not set, we will not display resource capacity and usage on the UI.
+ */
+ @isBoolean
+ public static final String SCHEDULER_RESOURCE_DISPLAY = "scheduler.resource.display";
+
+ /**
* The mode this Storm cluster is running in. Either "distributed" or "local".
*/
@isString
http://git-wip-us.apache.org/repos/asf/storm/blob/2d993c51/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/index.html b/storm-core/src/ui/public/index.html
index 6e52923..d113e30 100644
--- a/storm-core/src/ui/public/index.html
+++ b/storm-core/src/ui/public/index.html
@@ -144,8 +144,8 @@ $(document).ready(function() {
});
$('#topology-summary [data-toggle="tooltip"]').tooltip();
$.getJSON("/api/v1/cluster/configuration", function(json){
- var scheduler = json["storm.scheduler"];
- if (scheduler != "backtype.storm.scheduler.resource.ResourceAwareScheduler"){
+ var displayResource = json["scheduler.display.resource"];
+ if (!displayResource){
$('#topology-summary td:nth-child(9),#topology-summary th:nth-child(9)').hide();
$('#topology-summary td:nth-child(10),#topology-summary th:nth-child(10)').hide();
}
@@ -164,8 +164,8 @@ $(document).ready(function() {
});
$('#supervisor-summary [data-toggle="tooltip"]').tooltip();
$.getJSON("/api/v1/cluster/configuration", function(json){
- var scheduler = json["storm.scheduler"];
- if (scheduler != "backtype.storm.scheduler.resource.ResourceAwareScheduler"){
+ var displayResource = json["scheduler.display.resource"];
+ if (!displayResource){
$('#supervisor-summary td:nth-child(6),#supervisor-summary th:nth-child(6)').hide();
$('#supervisor-summary td:nth-child(7),#supervisor-summary th:nth-child(7)').hide();
$('#supervisor-summary td:nth-child(8),#supervisor-summary th:nth-child(8)').hide();
http://git-wip-us.apache.org/repos/asf/storm/blob/2d993c51/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index b8dd1f9..b062196 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -294,8 +294,8 @@ $(document).ready(function() {
topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
topologyResources.append(Mustache.render($(template).filter("#topology-resources-template").html(),response));
$.getJSON("/api/v1/cluster/configuration", function(json){
- var scheduler = json["storm.scheduler"];
- if (scheduler != "backtype.storm.scheduler.resource.ResourceAwareScheduler"){
+ var displayResource = json["scheduler.resource.display"];
+ if (!displayResource){
$('#topology-resources-header').hide();
$('#topology-resources').hide();
}
[09/15] storm git commit: Use camel case to be consistent
Posted by bo...@apache.org.
Use camel case to be consistent
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bac4a053
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bac4a053
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bac4a053
Branch: refs/heads/master
Commit: bac4a053687f1ba8c806f6fe20b9d47f3fba2407
Parents: 71a5b1a
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Nov 20 13:05:35 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Nov 20 13:05:35 2015 -0600
----------------------------------------------------------------------
.../jvm/backtype/storm/scheduler/Cluster.java | 24 ++++++++++----------
1 file changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bac4a053/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
index ff08ad1..571ee1b 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
@@ -425,7 +425,7 @@ public class Cluster {
for (String topologyId : this.assignments.keySet()) {
ret.put(topologyId, this.assignments.get(topologyId));
}
-
+
return ret;
}
@@ -472,22 +472,22 @@ public class Cluster {
private Double getAssignedMemoryForSlot(Map topConf) {
Double totalWorkerMemory = 0.0;
- String topology_worker_childopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), null);
- String worker_childopts = Utils.getString(topConf.get(Config.WORKER_CHILDOPTS), null);
- Double mem_topology_worker_childopts = Utils.parseWorkerChildOpts(topology_worker_childopts, null);
- Double mem_worker_childopts = Utils.parseWorkerChildOpts(worker_childopts, null);
+ String topologyWorkerChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), null);
+ String workerChildopts = Utils.getString(topConf.get(Config.WORKER_CHILDOPTS), null);
+ Double memTopologyWorkerChildopts = Utils.parseWorkerChildOpts(topologyWorkerChildopts, null);
+ Double memWorkerChildopts = Utils.parseWorkerChildOpts(workerChildopts, null);
- if (mem_topology_worker_childopts != null) {
- totalWorkerMemory += mem_topology_worker_childopts;
- } else if (mem_worker_childopts != null) {
- totalWorkerMemory += mem_worker_childopts;
+ if (memTopologyWorkerChildopts != null) {
+ totalWorkerMemory += memTopologyWorkerChildopts;
+ } else if (memWorkerChildopts != null) {
+ totalWorkerMemory += memWorkerChildopts;
} else {
totalWorkerMemory += Utils.getInt(topConf.get(Config.WORKER_HEAP_MEMORY_MB));
}
- String topology_worker_lw_childiopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
- if (topology_worker_lw_childiopts != null) {
- totalWorkerMemory += Utils.parseWorkerChildOpts(topology_worker_lw_childiopts, 0.0);
+ String topoWorkerLwChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
+ if (topoWorkerLwChildopts != null) {
+ totalWorkerMemory += Utils.parseWorkerChildOpts(topoWorkerLwChildopts, 0.0);
}
return totalWorkerMemory;
}
[07/15] storm git commit: Minor
Posted by bo...@apache.org.
Minor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/89fff440
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/89fff440
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/89fff440
Branch: refs/heads/master
Commit: 89fff440819836c78762b1c27fae8b005a752746
Parents: ce7085b
Author: zhuol <zh...@yahoo-inc.com>
Authored: Tue Nov 17 16:03:28 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:39:27 2015 -0600
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 2 +-
storm-core/src/jvm/backtype/storm/Config.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/89fff440/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index 176ce6b..71d4654 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -746,7 +746,7 @@
;; the new assignments for all the topologies are in the cluster object.
_ (.schedule (:scheduler nimbus) topologies cluster)
_ (.setResourcesMap cluster @(:id->resources nimbus))
- _ (if-not (conf SCHEDULER-RESOURCE-DISPLAY) (.updateAssignedMemoryForTopologyAndSupervisor cluster topologies))
+ _ (if-not (conf SCHEDULER-DISPLAY-RESOURCE) (.updateAssignedMemoryForTopologyAndSupervisor cluster topologies))
_ (reset! (:id->sched-status nimbus) (.getStatusMap cluster))
_ (reset! (:node-id->resources nimbus) (.getSupervisorsResourcesMap cluster))
_ (reset! (:id->resources nimbus) (.getResourcesMap cluster))]
http://git-wip-us.apache.org/repos/asf/storm/blob/89fff440/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index afa200b..64476a0 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -180,7 +180,7 @@ public class Config extends HashMap<String, Object> {
* If this is not set, we will not display resource capacity and usage on the UI.
*/
@isBoolean
- public static final String SCHEDULER_RESOURCE_DISPLAY = "scheduler.display.resource";
+ public static final String SCHEDULER_DISPLAY_RESOURCE = "scheduler.display.resource";
/**
* The mode this Storm cluster is running in. Either "distributed" or "local".
[13/15] storm git commit: Merge branch '1198' of
https://github.com/zhuoliu/storm into STORM-1198
Posted by bo...@apache.org.
Merge branch '1198' of https://github.com/zhuoliu/storm into STORM-1198
STORM-1198: Web UI to show resource usages and Total Resources on all supervisors
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/06462862
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/06462862
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/06462862
Branch: refs/heads/master
Commit: 064628626f5268b1dc8537ac7cce408702672c0f
Parents: ab5b2f9 e58d254
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Nov 20 16:06:06 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Nov 20 16:06:06 2015 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 2 +
docs/documentation/ui-rest-api.md | 14 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 11 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 17 +-
storm-core/src/jvm/backtype/storm/Config.java | 9 +
.../storm/generated/SupervisorSummary.java | 206 ++++++++++++++++++-
.../jvm/backtype/storm/scheduler/Cluster.java | 115 ++++++++++-
.../resource/ResourceAwareScheduler.java | 18 +-
.../src/jvm/backtype/storm/utils/Utils.java | 47 +++++
storm-core/src/py/storm/ttypes.py | 28 ++-
storm-core/src/storm.thrift | 2 +
.../public/templates/index-page-template.html | 36 ++++
.../templates/topology-page-template.html | 6 +
storm-core/src/ui/public/topology.html | 7 +-
14 files changed, 502 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/06462862/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[12/15] storm git commit: Minor
Posted by bo...@apache.org.
Minor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e58d2541
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e58d2541
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e58d2541
Branch: refs/heads/master
Commit: e58d254102daff659250ee8adc0df68dabac6931
Parents: ca21dc7
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Nov 20 15:33:13 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Nov 20 15:33:13 2015 -0600
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/scheduler/Cluster.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e58d2541/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
index f4410ad..ff2b233 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
@@ -507,7 +507,7 @@ public class Cluster {
String topoWorkerLwChildopts = null;
if (topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS) instanceof List) {
- topoWorkerLwChildopts = getStringFromStringList(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS);
+ topoWorkerLwChildopts = getStringFromStringList(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS));
} else {
topoWorkerLwChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
}
[10/15] storm git commit: Able to solve mkgMKG and list of strings
Posted by bo...@apache.org.
Able to solve mkgMKG and list of strings
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2e56ede5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2e56ede5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2e56ede5
Branch: refs/heads/master
Commit: 2e56ede5d00af774796344be6581af99076a9590
Parents: bac4a05
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Nov 20 14:45:40 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Nov 20 14:45:40 2015 -0600
----------------------------------------------------------------------
.../jvm/backtype/storm/scheduler/Cluster.java | 6 ++--
.../src/jvm/backtype/storm/utils/Utils.java | 29 +++++++++++++++-----
2 files changed, 25 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2e56ede5/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
index 571ee1b..a8ff2ab 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
@@ -474,8 +474,8 @@ public class Cluster {
String topologyWorkerChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), null);
String workerChildopts = Utils.getString(topConf.get(Config.WORKER_CHILDOPTS), null);
- Double memTopologyWorkerChildopts = Utils.parseWorkerChildOpts(topologyWorkerChildopts, null);
- Double memWorkerChildopts = Utils.parseWorkerChildOpts(workerChildopts, null);
+ Double memTopologyWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(topologyWorkerChildopts, null);
+ Double memWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(workerChildopts, null);
if (memTopologyWorkerChildopts != null) {
totalWorkerMemory += memTopologyWorkerChildopts;
@@ -487,7 +487,7 @@ public class Cluster {
String topoWorkerLwChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
if (topoWorkerLwChildopts != null) {
- totalWorkerMemory += Utils.parseWorkerChildOpts(topoWorkerLwChildopts, 0.0);
+ totalWorkerMemory += Utils.parseJvmHeapMemByChildOpts(topoWorkerLwChildopts, 0.0);
}
return totalWorkerMemory;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/2e56ede5/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 9400c44..766b10f 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -509,9 +509,15 @@ public class Utils {
if (null == o) {
return defaultValue;
}
-
if (o instanceof String) {
return (String) o;
+ } else if (o instanceof List) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : (List<String>) o) {
+ sb.append(s);
+ sb.append(" ");
+ }
+ return sb.toString();
} else {
throw new IllegalArgumentException("Don't know how to convert " + o + " + to String");
}
@@ -780,22 +786,31 @@ public class Utils {
}
/**
- * parses the arguments to extract jvm heap memory size.
+ * parses the arguments to extract jvm heap memory size in MB.
* @param input
* @param defaultValue
- * @return the value of the JVM heap memory setting in a java command.
+ * @return the value of the JVM heap memory setting (in MB) in a java command.
*/
- public static Double parseWorkerChildOpts(String input, Double defaultValue) {
+ public static Double parseJvmHeapMemByChildOpts(String input, Double defaultValue) {
if (input != null) {
- Pattern optsPattern = Pattern.compile("Xmx[0-9]+m");
+ Pattern optsPattern = Pattern.compile("Xmx[0-9]+[mkgMKG]");
Matcher m = optsPattern.matcher(input);
String memoryOpts = null;
while (m.find()) {
memoryOpts = m.group();
}
- if(memoryOpts != null) {
+ if (memoryOpts != null) {
+ int unit = 1;
+ if (memoryOpts.toLowerCase().endsWith("k")) {
+ unit = 1024;
+ } else if (memoryOpts.toLowerCase().endsWith("m")) {
+ unit = 1024 * 1024;
+ } else if (memoryOpts.toLowerCase().endsWith("g")) {
+ unit = 1024 * 1024 * 1024;
+ }
memoryOpts = memoryOpts.replaceAll("[a-zA-Z]", "");
- return Double.parseDouble(memoryOpts);
+ Double result = Double.parseDouble(memoryOpts) * unit / 1024.0 / 1024.0;
+ return (result < 1.0) ? 1.0 : result;
} else {
return defaultValue;
}
[03/15] storm git commit: [STORM-1198] Web UI to show resource usages
and Total Resources on all supervisors
Posted by bo...@apache.org.
[STORM-1198] Web UI to show resource usages and Total Resources on all supervisors
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/400f2154
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/400f2154
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/400f2154
Branch: refs/heads/master
Commit: 400f2154e458b2033299b5be5bde165dfbb57cf8
Parents: 1c07f1d
Author: zhuol <zh...@yahoo-inc.com>
Authored: Tue Nov 10 14:57:14 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:35:22 2015 -0600
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/nimbus.clj | 7 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 5 +
.../storm/generated/SupervisorSummary.java | 206 ++++++++++++++++++-
.../jvm/backtype/storm/scheduler/Cluster.java | 15 +-
.../resource/ResourceAwareScheduler.java | 16 ++
storm-core/src/py/storm/ttypes.py | 28 ++-
storm-core/src/storm.thrift | 2 +
.../public/templates/index-page-template.html | 24 +++
8 files changed, 296 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index d23da03..87b5a8b 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -163,7 +163,8 @@
:leader-elector (zk-leader-elector conf)
:code-distributor (mk-code-distributor conf)
:id->sched-status (atom {})
- :id->resources (atom {})
+ :node-id->resources (atom {}) ;;resources of supervisors
+ :id->resources (atom {}) ;;resources of topologies
:cred-renewers (AuthUtils/GetCredentialRenewers conf)
:topology-history-lock (Object.)
:topo-history-state (nimbus-topo-history-state conf)
@@ -745,6 +746,7 @@
;; the new assignments for all the topologies are in the cluster object.
_ (.schedule (:scheduler nimbus) topologies cluster)
_ (reset! (:id->sched-status nimbus) (.getStatusMap cluster))
+ _ (reset! (:node-id->resources nimbus) (.getSupervisorsResourcesMap cluster))
_ (reset! (:id->resources nimbus) (merge @(:id->resources nimbus) (.getResourcesMap cluster)))]
(.getAssignments cluster)))
@@ -1646,6 +1648,9 @@
(count (:used-ports info))
id) ]
(.set_total_resources sup-sum (map-val double (:resources-map info)))
+ (when-let [[total-mem total-cpu used-mem used-cpu] (.get @(:node-id->resources nimbus) id)]
+ (.set_used_mem sup-sum used-mem)
+ (.set_used_cpu sup-sum used-cpu))
(when-let [version (:version info)] (.set_version sup-sum version))
sup-sum))
nimbus-uptime ((:uptime nimbus))
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 99f75e4..64dc1b9 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -42,6 +42,7 @@
(:import [backtype.storm.generated AuthorizationException ProfileRequest ProfileAction NodeInfo])
(:import [backtype.storm.security.auth AuthUtils])
(:import [backtype.storm.utils VersionInfo])
+ (:import [backtype.storm Config])
(:import [java.io File])
(:require [compojure.route :as route]
[compojure.handler :as handler]
@@ -433,6 +434,10 @@
"uptimeSeconds" (.get_uptime_secs s)
"slotsTotal" (.get_num_workers s)
"slotsUsed" (.get_num_used_workers s)
+ "totalMem" (get (.get_total_resources s) Config/SUPERVISOR_MEMORY_CAPACITY_MB)
+ "totalCpu" (get (.get_total_resources s) Config/SUPERVISOR_CPU_CAPACITY)
+ "usedMem" (.get_used_mem s)
+ "usedCpu" (.get_used_cpu s)
"version" (.get_version s)})}))
(defn all-topologies-summary
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java b/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
index 2c3f330..88202d8 100644
--- a/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
+++ b/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
@@ -62,6 +62,8 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
private static final org.apache.thrift.protocol.TField SUPERVISOR_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("supervisor_id", org.apache.thrift.protocol.TType.STRING, (short)5);
private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.protocol.TField TOTAL_RESOURCES_FIELD_DESC = new org.apache.thrift.protocol.TField("total_resources", org.apache.thrift.protocol.TType.MAP, (short)7);
+ private static final org.apache.thrift.protocol.TField USED_MEM_FIELD_DESC = new org.apache.thrift.protocol.TField("used_mem", org.apache.thrift.protocol.TType.DOUBLE, (short)8);
+ private static final org.apache.thrift.protocol.TField USED_CPU_FIELD_DESC = new org.apache.thrift.protocol.TField("used_cpu", org.apache.thrift.protocol.TType.DOUBLE, (short)9);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -76,6 +78,8 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
private String supervisor_id; // required
private String version; // optional
private Map<String,Double> total_resources; // optional
+ private double used_mem; // optional
+ private double used_cpu; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -85,7 +89,9 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
NUM_USED_WORKERS((short)4, "num_used_workers"),
SUPERVISOR_ID((short)5, "supervisor_id"),
VERSION((short)6, "version"),
- TOTAL_RESOURCES((short)7, "total_resources");
+ TOTAL_RESOURCES((short)7, "total_resources"),
+ USED_MEM((short)8, "used_mem"),
+ USED_CPU((short)9, "used_cpu");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -114,6 +120,10 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
return VERSION;
case 7: // TOTAL_RESOURCES
return TOTAL_RESOURCES;
+ case 8: // USED_MEM
+ return USED_MEM;
+ case 9: // USED_CPU
+ return USED_CPU;
default:
return null;
}
@@ -157,8 +167,10 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
private static final int __UPTIME_SECS_ISSET_ID = 0;
private static final int __NUM_WORKERS_ISSET_ID = 1;
private static final int __NUM_USED_WORKERS_ISSET_ID = 2;
+ private static final int __USED_MEM_ISSET_ID = 3;
+ private static final int __USED_CPU_ISSET_ID = 4;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.VERSION,_Fields.TOTAL_RESOURCES};
+ private static final _Fields optionals[] = {_Fields.VERSION,_Fields.TOTAL_RESOURCES,_Fields.USED_MEM,_Fields.USED_CPU};
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -178,6 +190,10 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))));
+ tmpMap.put(_Fields.USED_MEM, new org.apache.thrift.meta_data.FieldMetaData("used_mem", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
+ tmpMap.put(_Fields.USED_CPU, new org.apache.thrift.meta_data.FieldMetaData("used_cpu", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SupervisorSummary.class, metaDataMap);
}
@@ -226,6 +242,8 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
Map<String,Double> __this__total_resources = new HashMap<String,Double>(other.total_resources);
this.total_resources = __this__total_resources;
}
+ this.used_mem = other.used_mem;
+ this.used_cpu = other.used_cpu;
}
public SupervisorSummary deepCopy() {
@@ -245,6 +263,10 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
this.version = "VERSION_NOT_PROVIDED";
this.total_resources = null;
+ set_used_mem_isSet(false);
+ this.used_mem = 0.0;
+ set_used_cpu_isSet(false);
+ this.used_cpu = 0.0;
}
public String get_host() {
@@ -416,6 +438,50 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
}
}
+ public double get_used_mem() {
+ return this.used_mem;
+ }
+
+ public void set_used_mem(double used_mem) {
+ this.used_mem = used_mem;
+ set_used_mem_isSet(true);
+ }
+
+ public void unset_used_mem() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __USED_MEM_ISSET_ID);
+ }
+
+ /** Returns true if field used_mem is set (has been assigned a value) and false otherwise */
+ public boolean is_set_used_mem() {
+ return EncodingUtils.testBit(__isset_bitfield, __USED_MEM_ISSET_ID);
+ }
+
+ public void set_used_mem_isSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __USED_MEM_ISSET_ID, value);
+ }
+
+ public double get_used_cpu() {
+ return this.used_cpu;
+ }
+
+ public void set_used_cpu(double used_cpu) {
+ this.used_cpu = used_cpu;
+ set_used_cpu_isSet(true);
+ }
+
+ public void unset_used_cpu() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __USED_CPU_ISSET_ID);
+ }
+
+ /** Returns true if field used_cpu is set (has been assigned a value) and false otherwise */
+ public boolean is_set_used_cpu() {
+ return EncodingUtils.testBit(__isset_bitfield, __USED_CPU_ISSET_ID);
+ }
+
+ public void set_used_cpu_isSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __USED_CPU_ISSET_ID, value);
+ }
+
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case HOST:
@@ -474,6 +540,22 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
}
break;
+ case USED_MEM:
+ if (value == null) {
+ unset_used_mem();
+ } else {
+ set_used_mem((Double)value);
+ }
+ break;
+
+ case USED_CPU:
+ if (value == null) {
+ unset_used_cpu();
+ } else {
+ set_used_cpu((Double)value);
+ }
+ break;
+
}
}
@@ -500,6 +582,12 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
case TOTAL_RESOURCES:
return get_total_resources();
+ case USED_MEM:
+ return Double.valueOf(get_used_mem());
+
+ case USED_CPU:
+ return Double.valueOf(get_used_cpu());
+
}
throw new IllegalStateException();
}
@@ -525,6 +613,10 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
return is_set_version();
case TOTAL_RESOURCES:
return is_set_total_resources();
+ case USED_MEM:
+ return is_set_used_mem();
+ case USED_CPU:
+ return is_set_used_cpu();
}
throw new IllegalStateException();
}
@@ -605,6 +697,24 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
return false;
}
+ boolean this_present_used_mem = true && this.is_set_used_mem();
+ boolean that_present_used_mem = true && that.is_set_used_mem();
+ if (this_present_used_mem || that_present_used_mem) {
+ if (!(this_present_used_mem && that_present_used_mem))
+ return false;
+ if (this.used_mem != that.used_mem)
+ return false;
+ }
+
+ boolean this_present_used_cpu = true && this.is_set_used_cpu();
+ boolean that_present_used_cpu = true && that.is_set_used_cpu();
+ if (this_present_used_cpu || that_present_used_cpu) {
+ if (!(this_present_used_cpu && that_present_used_cpu))
+ return false;
+ if (this.used_cpu != that.used_cpu)
+ return false;
+ }
+
return true;
}
@@ -647,6 +757,16 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
if (present_total_resources)
list.add(total_resources);
+ boolean present_used_mem = true && (is_set_used_mem());
+ list.add(present_used_mem);
+ if (present_used_mem)
+ list.add(used_mem);
+
+ boolean present_used_cpu = true && (is_set_used_cpu());
+ list.add(present_used_cpu);
+ if (present_used_cpu)
+ list.add(used_cpu);
+
return list.hashCode();
}
@@ -728,6 +848,26 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
return lastComparison;
}
}
+ lastComparison = Boolean.valueOf(is_set_used_mem()).compareTo(other.is_set_used_mem());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (is_set_used_mem()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.used_mem, other.used_mem);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(is_set_used_cpu()).compareTo(other.is_set_used_cpu());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (is_set_used_cpu()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.used_cpu, other.used_cpu);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -795,6 +935,18 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
}
first = false;
}
+ if (is_set_used_mem()) {
+ if (!first) sb.append(", ");
+ sb.append("used_mem:");
+ sb.append(this.used_mem);
+ first = false;
+ }
+ if (is_set_used_cpu()) {
+ if (!first) sb.append(", ");
+ sb.append("used_cpu:");
+ sb.append(this.used_cpu);
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -928,6 +1080,22 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 8: // USED_MEM
+ if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
+ struct.used_mem = iprot.readDouble();
+ struct.set_used_mem_isSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 9: // USED_CPU
+ if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) {
+ struct.used_cpu = iprot.readDouble();
+ struct.set_used_cpu_isSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -982,6 +1150,16 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
oprot.writeFieldEnd();
}
}
+ if (struct.is_set_used_mem()) {
+ oprot.writeFieldBegin(USED_MEM_FIELD_DESC);
+ oprot.writeDouble(struct.used_mem);
+ oprot.writeFieldEnd();
+ }
+ if (struct.is_set_used_cpu()) {
+ oprot.writeFieldBegin(USED_CPU_FIELD_DESC);
+ oprot.writeDouble(struct.used_cpu);
+ oprot.writeFieldEnd();
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -1011,7 +1189,13 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
if (struct.is_set_total_resources()) {
optionals.set(1);
}
- oprot.writeBitSet(optionals, 2);
+ if (struct.is_set_used_mem()) {
+ optionals.set(2);
+ }
+ if (struct.is_set_used_cpu()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.is_set_version()) {
oprot.writeString(struct.version);
}
@@ -1025,6 +1209,12 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
}
}
}
+ if (struct.is_set_used_mem()) {
+ oprot.writeDouble(struct.used_mem);
+ }
+ if (struct.is_set_used_cpu()) {
+ oprot.writeDouble(struct.used_cpu);
+ }
}
@Override
@@ -1040,7 +1230,7 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
struct.set_num_used_workers_isSet(true);
struct.supervisor_id = iprot.readString();
struct.set_supervisor_id_isSet(true);
- BitSet incoming = iprot.readBitSet(2);
+ BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.version = iprot.readString();
struct.set_version_isSet(true);
@@ -1060,6 +1250,14 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
}
struct.set_total_resources_isSet(true);
}
+ if (incoming.get(2)) {
+ struct.used_mem = iprot.readDouble();
+ struct.set_used_mem_isSet(true);
+ }
+ if (incoming.get(3)) {
+ struct.used_cpu = iprot.readDouble();
+ struct.set_used_cpu_isSet(true);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
index b3028e9..c4a815f 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
@@ -34,7 +34,11 @@ public class Cluster {
/**
* key: supervisor id, value: supervisor details
*/
- private Map<String, SupervisorDetails> supervisors;
+ private Map<String, SupervisorDetails> supervisors;
+ /**
+ * key: supervisor id, value: supervisor's total and used resources
+ */
+ private Map<String, Double[]> supervisorsResources;
/**
* key: rack, value: nodes in that rack
@@ -73,6 +77,7 @@ public class Cluster {
this.assignments.putAll(assignments);
this.status = new HashMap<String, String>();
this.resources = new HashMap<String, Double[]>();
+ this.supervisorsResources = new HashMap<String, Double[]>();
this.hostToId = new HashMap<String, List<String>>();
for (Map.Entry<String, SupervisorDetails> entry : supervisors.entrySet()) {
String nodeId = entry.getKey();
@@ -476,4 +481,12 @@ public class Cluster {
public Map<String, Double[]> getResourcesMap() {
return this.resources;
}
+
+ public void setSupervisorsResources(Map<String, Double[]> supervisors_resources) {
+ this.supervisorsResources.putAll(supervisors_resources);
+ }
+
+ public Map<String, Double[]> getSupervisorsResourcesMap() {
+ return this.supervisorsResources;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java b/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
index 6e455f2..ad33b43 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
@@ -110,6 +110,22 @@ public class ResourceAwareScheduler implements IScheduler {
cluster.setStatus(td.getId(), "Fully Scheduled");
}
}
+ updateSupervisorsResources(cluster, topologies);
+ }
+
+ private void updateSupervisorsResources(Cluster cluster, Topologies topologies) {
+ Map<String, Double[]> supervisors_resources = new HashMap<String, Double[]>();
+ Map<String, RAS_Node> nodes = RAS_Node.getAllNodesFrom(cluster, topologies);
+ for (Map.Entry<String, RAS_Node> entry : nodes.entrySet()) {
+ RAS_Node node = entry.getValue();
+ Double totalMem = node.getTotalMemoryResources();
+ Double totalCpu = node.getTotalCpuResources();
+ Double usedMem = totalMem - node.getAvailableMemoryResources();
+ Double usedCpu = totalCpu - node.getAvailableCpuResources();
+ Double[] resources = {totalMem, totalCpu, usedMem, usedCpu};
+ supervisors_resources.put(entry.getKey(), resources);
+ }
+ cluster.setSupervisorsResources(supervisors_resources);
}
private Map<String, Double> getUserConf() {
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/py/storm/ttypes.py
----------------------------------------------------------------------
diff --git a/storm-core/src/py/storm/ttypes.py b/storm-core/src/py/storm/ttypes.py
index 23a82e1..ea3c768 100644
--- a/storm-core/src/py/storm/ttypes.py
+++ b/storm-core/src/py/storm/ttypes.py
@@ -2510,6 +2510,8 @@ class SupervisorSummary:
- supervisor_id
- version
- total_resources
+ - used_mem
+ - used_cpu
"""
thrift_spec = (
@@ -2521,9 +2523,11 @@ class SupervisorSummary:
(5, TType.STRING, 'supervisor_id', None, None, ), # 5
(6, TType.STRING, 'version', None, "VERSION_NOT_PROVIDED", ), # 6
(7, TType.MAP, 'total_resources', (TType.STRING,None,TType.DOUBLE,None), None, ), # 7
+ (8, TType.DOUBLE, 'used_mem', None, None, ), # 8
+ (9, TType.DOUBLE, 'used_cpu', None, None, ), # 9
)
- def __init__(self, host=None, uptime_secs=None, num_workers=None, num_used_workers=None, supervisor_id=None, version=thrift_spec[6][4], total_resources=None,):
+ def __init__(self, host=None, uptime_secs=None, num_workers=None, num_used_workers=None, supervisor_id=None, version=thrift_spec[6][4], total_resources=None, used_mem=None, used_cpu=None,):
self.host = host
self.uptime_secs = uptime_secs
self.num_workers = num_workers
@@ -2531,6 +2535,8 @@ class SupervisorSummary:
self.supervisor_id = supervisor_id
self.version = version
self.total_resources = total_resources
+ self.used_mem = used_mem
+ self.used_cpu = used_cpu
def read(self, iprot):
if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
@@ -2582,6 +2588,16 @@ class SupervisorSummary:
iprot.readMapEnd()
else:
iprot.skip(ftype)
+ elif fid == 8:
+ if ftype == TType.DOUBLE:
+ self.used_mem = iprot.readDouble();
+ else:
+ iprot.skip(ftype)
+ elif fid == 9:
+ if ftype == TType.DOUBLE:
+ self.used_cpu = iprot.readDouble();
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -2624,6 +2640,14 @@ class SupervisorSummary:
oprot.writeDouble(viter74)
oprot.writeMapEnd()
oprot.writeFieldEnd()
+ if self.used_mem is not None:
+ oprot.writeFieldBegin('used_mem', TType.DOUBLE, 8)
+ oprot.writeDouble(self.used_mem)
+ oprot.writeFieldEnd()
+ if self.used_cpu is not None:
+ oprot.writeFieldBegin('used_cpu', TType.DOUBLE, 9)
+ oprot.writeDouble(self.used_cpu)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -2650,6 +2674,8 @@ class SupervisorSummary:
value = (value * 31) ^ hash(self.supervisor_id)
value = (value * 31) ^ hash(self.version)
value = (value * 31) ^ hash(self.total_resources)
+ value = (value * 31) ^ hash(self.used_mem)
+ value = (value * 31) ^ hash(self.used_cpu)
return value
def __repr__(self):
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/storm.thrift
----------------------------------------------------------------------
diff --git a/storm-core/src/storm.thrift b/storm-core/src/storm.thrift
index 6d28d88..1818ea7 100644
--- a/storm-core/src/storm.thrift
+++ b/storm-core/src/storm.thrift
@@ -162,6 +162,8 @@ struct SupervisorSummary {
5: required string supervisor_id;
6: optional string version = "VERSION_NOT_PROVIDED";
7: optional map<string, double> total_resources;
+ 8: optional double used_mem;
+ 9: optional double used_cpu;
}
struct NimbusSummary {
http://git-wip-us.apache.org/repos/asf/storm/blob/400f2154/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/index-page-template.html b/storm-core/src/ui/public/templates/index-page-template.html
index 15d72a7..bf1f108 100644
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@ -215,6 +215,26 @@
</span>
</th>
<th>
+ <span data-toggle="tooltip" data-placement="above" title="Memory capacity of a supervisor.">
+ Total Mem (MB)
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" data-placement="left" title="Memory that has been allocated.">
+ Used Mem (MB)
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" data-placement="above" title="CPU capacity of a supervisor. Every 100 means one core.">
+ Total CPU (%)
+ </span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" data-placement="left" title="CPU that has been allocated. Every 100 means one core">
+ Used CPU (%)
+ </span>
+ </th>
+ <th>
<span data-toggle="tooltip" data-placement="left" title="Version">
Version
</span>
@@ -229,6 +249,10 @@
<td>{{uptime}}</td>
<td>{{slotsTotal}}</td>
<td>{{slotsUsed}}</td>
+ <td>{{totalMem}}</td>
+ <td>{{usedMem}}</td>
+ <td>{{totalCpu}}</td>
+ <td>{{usedCpu}}</td>
<td>{{version}}</td>
</tr>
{{/supervisors}}
[08/15] storm git commit: Minor fix
Posted by bo...@apache.org.
Minor fix
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/71a5b1a6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/71a5b1a6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/71a5b1a6
Branch: refs/heads/master
Commit: 71a5b1a656b099748ddfb87b6c78a26d25146904
Parents: 515722d
Author: zhuol <zh...@yahoo-inc.com>
Authored: Thu Nov 19 15:38:34 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 16:56:17 2015 -0600
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/utils/Utils.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/71a5b1a6/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 91fdb09..9400c44 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -777,6 +777,7 @@ public class Utils {
public static double zeroIfNaNOrInf(double x) {
return (Double.isNaN(x) || Double.isInfinite(x)) ? 0.0 : x;
+ }
/**
* parses the arguments to extract jvm heap memory size.
@@ -785,14 +786,14 @@ public class Utils {
* @return the value of the JVM heap memory setting in a java command.
*/
public static Double parseWorkerChildOpts(String input, Double defaultValue) {
- if(input != null) {
+ if (input != null) {
Pattern optsPattern = Pattern.compile("Xmx[0-9]+m");
Matcher m = optsPattern.matcher(input);
String memoryOpts = null;
while (m.find()) {
memoryOpts = m.group();
}
- if(memoryOpts!=null) {
+ if(memoryOpts != null) {
memoryOpts = memoryOpts.replaceAll("[a-zA-Z]", "");
return Double.parseDouble(memoryOpts);
} else {
[11/15] storm git commit: Move pass List to String away from Utils
Posted by bo...@apache.org.
Move pass List to String away from Utils
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ca21dc7d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ca21dc7d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ca21dc7d
Branch: refs/heads/master
Commit: ca21dc7d615e1dae1cd4c0f692c3b5f561688347
Parents: 2e56ede
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Nov 20 15:20:12 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Nov 20 15:20:12 2015 -0600
----------------------------------------------------------------------
.../jvm/backtype/storm/scheduler/Cluster.java | 31 ++++++++++++++++++--
.../src/jvm/backtype/storm/utils/Utils.java | 7 -----
2 files changed, 28 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ca21dc7d/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
index a8ff2ab..f4410ad 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
@@ -466,15 +466,35 @@ public class Cluster {
return networkTopography;
}
+ private String getStringFromStringList(Object o) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : (List<String>) o) {
+ sb.append(s);
+ sb.append(" ");
+ }
+ return sb.toString();
+ }
+
/*
* Get heap memory usage for a worker's main process and logwriter process
* */
private Double getAssignedMemoryForSlot(Map topConf) {
Double totalWorkerMemory = 0.0;
- String topologyWorkerChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), null);
- String workerChildopts = Utils.getString(topConf.get(Config.WORKER_CHILDOPTS), null);
+ String topologyWorkerChildopts = null;
+ if (topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS) instanceof List) {
+ topologyWorkerChildopts = getStringFromStringList(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS));
+ } else {
+ topologyWorkerChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), null);
+ }
Double memTopologyWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(topologyWorkerChildopts, null);
+
+ String workerChildopts = null;
+ if (topConf.get(Config.WORKER_CHILDOPTS) instanceof List) {
+ workerChildopts = getStringFromStringList(topConf.get(Config.WORKER_CHILDOPTS));
+ } else {
+ workerChildopts = Utils.getString(topConf.get(Config.WORKER_CHILDOPTS), null);
+ }
Double memWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(workerChildopts, null);
if (memTopologyWorkerChildopts != null) {
@@ -485,7 +505,12 @@ public class Cluster {
totalWorkerMemory += Utils.getInt(topConf.get(Config.WORKER_HEAP_MEMORY_MB));
}
- String topoWorkerLwChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
+ String topoWorkerLwChildopts = null;
+ if (topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS) instanceof List) {
+ topoWorkerLwChildopts = getStringFromStringList(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS);
+ } else {
+ topoWorkerLwChildopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
+ }
if (topoWorkerLwChildopts != null) {
totalWorkerMemory += Utils.parseJvmHeapMemByChildOpts(topoWorkerLwChildopts, 0.0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ca21dc7d/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 766b10f..b9a89f7 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -511,13 +511,6 @@ public class Utils {
}
if (o instanceof String) {
return (String) o;
- } else if (o instanceof List) {
- StringBuilder sb = new StringBuilder();
- for (String s : (List<String>) o) {
- sb.append(s);
- sb.append(" ");
- }
- return sb.toString();
} else {
throw new IllegalArgumentException("Don't know how to convert " + o + " + to String");
}
[04/15] storm git commit: Calculate memory usage of each topology and
supervisor for non-RAS schedulers
Posted by bo...@apache.org.
Calculate memory usage of each topology and supervisor for non-RAS schedulers
Minor change before thrift-9.3
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ce7085ba
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ce7085ba
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ce7085ba
Branch: refs/heads/master
Commit: ce7085bafe0f75a3183977414f2321fdd1b55c94
Parents: 2d993c5
Author: zhuol <zh...@yahoo-inc.com>
Authored: Mon Nov 16 14:11:55 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:39:25 2015 -0600
----------------------------------------------------------------------
conf/defaults.yaml | 2 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 4 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 3 +-
storm-core/src/jvm/backtype/storm/Config.java | 2 +-
.../storm/generated/SupervisorSummary.java | 4 +-
.../jvm/backtype/storm/scheduler/Cluster.java | 75 +++++++++++++++++++-
.../resource/ResourceAwareScheduler.java | 4 +-
.../src/jvm/backtype/storm/utils/Utils.java | 38 ++++++++++
storm-core/src/py/storm/ttypes.py | 4 +-
storm-core/src/ui/public/index.html | 2 -
.../templates/topology-page-template.html | 6 ++
storm-core/src/ui/public/topology.html | 2 +-
12 files changed, 132 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index af5daa1..b520745 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -74,7 +74,7 @@ topology.max.replication.wait.time.sec: 60
nimbus.credential.renewers.freq.secs: 600
nimbus.impersonation.authorizer: "backtype.storm.security.auth.authorizer.ImpersonationAuthorizer"
-scheduler.resource.display: false
+scheduler.display.resource: false
### ui.* configs are for the master
ui.host: 0.0.0.0
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index 87b5a8b..176ce6b 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -745,9 +745,11 @@
;; call scheduler.schedule to schedule all the topologies
;; the new assignments for all the topologies are in the cluster object.
_ (.schedule (:scheduler nimbus) topologies cluster)
+ _ (.setResourcesMap cluster @(:id->resources nimbus))
+ _ (if-not (conf SCHEDULER-RESOURCE-DISPLAY) (.updateAssignedMemoryForTopologyAndSupervisor cluster topologies))
_ (reset! (:id->sched-status nimbus) (.getStatusMap cluster))
_ (reset! (:node-id->resources nimbus) (.getSupervisorsResourcesMap cluster))
- _ (reset! (:id->resources nimbus) (merge @(:id->resources nimbus) (.getResourcesMap cluster)))]
+ _ (reset! (:id->resources nimbus) (.getResourcesMap cluster))]
(.getAssignments cluster)))
(defn changed-executors [executor->node+port new-executor->node+port]
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 64dc1b9..480dcd3 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -464,7 +464,7 @@
"schedulerInfo" (.get_sched_status t)
"requestedMemOnHeap" (.get_requested_memonheap t)
"requestedMemOffHeap" (.get_requested_memoffheap t)
- "requestedMem" (+ (.get_requested_memonheap t) (.get_requested_memoffheap t))
+ "requestedTotalMem" (+ (.get_requested_memonheap t) (.get_requested_memoffheap t))
"requestedCpu" (.get_requested_cpu t)
"assignedMemOnHeap" (.get_assigned_memonheap t)
"assignedMemOffHeap" (.get_assigned_memoffheap t)
@@ -592,6 +592,7 @@
"requestedCpu" (.get_requested_cpu topo-info)
"assignedMemOnHeap" (.get_assigned_memonheap topo-info)
"assignedMemOffHeap" (.get_assigned_memoffheap topo-info)
+ "assignedTotalMem" (+ (.get_assigned_memonheap topo-info) (.get_assigned_memoffheap topo-info))
"assignedCpu" (.get_assigned_cpu topo-info)
"topologyStats" topo-stats
"spouts" (map (partial comp-agg-stats-json id secure?)
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 852757d..afa200b 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -180,7 +180,7 @@ public class Config extends HashMap<String, Object> {
* If this is not set, we will not display resource capacity and usage on the UI.
*/
@isBoolean
- public static final String SCHEDULER_RESOURCE_DISPLAY = "scheduler.resource.display";
+ public static final String SCHEDULER_RESOURCE_DISPLAY = "scheduler.display.resource";
/**
* The mode this Storm cluster is running in. Either "distributed" or "local".
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java b/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
index 88202d8..edb2016 100644
--- a/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
+++ b/storm-core/src/jvm/backtype/storm/generated/SupervisorSummary.java
@@ -583,10 +583,10 @@ public class SupervisorSummary implements org.apache.thrift.TBase<SupervisorSumm
return get_total_resources();
case USED_MEM:
- return Double.valueOf(get_used_mem());
+ return get_used_mem();
case USED_CPU:
- return Double.valueOf(get_used_cpu());
+ return get_used_cpu();
}
throw new IllegalStateException();
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
index c4a815f..ff08ad1 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/Cluster.java
@@ -466,6 +466,75 @@ public class Cluster {
return networkTopography;
}
+ /*
+ * Get heap memory usage for a worker's main process and logwriter process
+ * */
+ private Double getAssignedMemoryForSlot(Map topConf) {
+ Double totalWorkerMemory = 0.0;
+
+ String topology_worker_childopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), null);
+ String worker_childopts = Utils.getString(topConf.get(Config.WORKER_CHILDOPTS), null);
+ Double mem_topology_worker_childopts = Utils.parseWorkerChildOpts(topology_worker_childopts, null);
+ Double mem_worker_childopts = Utils.parseWorkerChildOpts(worker_childopts, null);
+
+ if (mem_topology_worker_childopts != null) {
+ totalWorkerMemory += mem_topology_worker_childopts;
+ } else if (mem_worker_childopts != null) {
+ totalWorkerMemory += mem_worker_childopts;
+ } else {
+ totalWorkerMemory += Utils.getInt(topConf.get(Config.WORKER_HEAP_MEMORY_MB));
+ }
+
+ String topology_worker_lw_childiopts = Utils.getString(topConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS), null);
+ if (topology_worker_lw_childiopts != null) {
+ totalWorkerMemory += Utils.parseWorkerChildOpts(topology_worker_lw_childiopts, 0.0);
+ }
+ return totalWorkerMemory;
+ }
+
+ /*
+ * Update memory usage for each topology and each supervisor node after every round of scheduling
+ * */
+ public void updateAssignedMemoryForTopologyAndSupervisor(Topologies topologies) {
+ Map<String, Double> supervisorToAssignedMem = new HashMap<String, Double>();
+
+ for (Map.Entry<String, SchedulerAssignment> entry : this.getAssignments().entrySet()) {
+ String topId = entry.getValue().getTopologyId();
+ Map topConf = topologies.getById(topId).getConf();
+ Double assignedMemForTopology = 0.0;
+ Double assignedMemPerSlot = getAssignedMemoryForSlot(topConf);
+ for (WorkerSlot ws: entry.getValue().getSlots()) {
+ assignedMemForTopology += assignedMemPerSlot;
+ String nodeId = ws.getNodeId();
+ if (supervisorToAssignedMem.containsKey(nodeId)) {
+ supervisorToAssignedMem.put(nodeId, supervisorToAssignedMem.get(nodeId) + assignedMemPerSlot);
+ } else {
+ supervisorToAssignedMem.put(nodeId, assignedMemPerSlot);
+ }
+ }
+ if (this.getResourcesMap().containsKey(topId)) {
+ Double[] topo_resources = getResourcesMap().get(topId);
+ topo_resources[3] = assignedMemForTopology;
+ } else {
+ Double[] topo_resources = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
+ topo_resources[3] = assignedMemForTopology;
+ this.setResources(topId, topo_resources);
+ }
+ }
+
+ for (Map.Entry<String, Double> entry : supervisorToAssignedMem.entrySet()) {
+ String nodeId = entry.getKey();
+ if (this.supervisorsResources.containsKey(nodeId)) {
+ Double[] supervisor_resources = supervisorsResources.get(nodeId);
+ supervisor_resources[2] = entry.getValue();
+ } else {
+ Double[] supervisor_resources = {0.0, 0.0, 0.0, 0.0};
+ supervisor_resources[2] = entry.getValue();
+ this.supervisorsResources.put(nodeId, supervisor_resources);
+ }
+ }
+ }
+
public void setStatus(String topologyId, String status) {
this.status.put(topologyId, status);
}
@@ -478,11 +547,15 @@ public class Cluster {
this.resources.put(topologyId, resources);
}
+ public void setResourcesMap(Map<String, Double[]> topologies_resources) {
+ this.resources.putAll(topologies_resources);
+ }
+
public Map<String, Double[]> getResourcesMap() {
return this.resources;
}
- public void setSupervisorsResources(Map<String, Double[]> supervisors_resources) {
+ public void setSupervisorsResourcesMap(Map<String, Double[]> supervisors_resources) {
this.supervisorsResources.putAll(supervisors_resources);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java b/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
index ad33b43..ac34d44 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/resource/ResourceAwareScheduler.java
@@ -102,7 +102,7 @@ public class ResourceAwareScheduler implements IScheduler {
Double[] resources = {requestedMemOnHeap, requestedMemOffHeap, requestedCpu,
assignedMemOnHeap, assignedMemOffHeap, assignedCpu};
LOG.debug("setResources for {}: requested on-heap mem, off-heap mem, cpu: {} {} {} " +
- "assigned on-heap mem, off-heap mem, cpu: {} {} {}",
+ "assigned on-heap mem, off-heap mem, cpu: {} {} {}",
td.getId(), requestedMemOnHeap, requestedMemOffHeap, requestedCpu,
assignedMemOnHeap, assignedMemOffHeap, assignedCpu);
cluster.setResources(td.getId(), resources);
@@ -125,7 +125,7 @@ public class ResourceAwareScheduler implements IScheduler {
Double[] resources = {totalMem, totalCpu, usedMem, usedCpu};
supervisors_resources.put(entry.getKey(), resources);
}
- cluster.setSupervisorsResources(supervisors_resources);
+ cluster.setSupervisorsResourcesMap(supervisors_resources);
}
private Map<String, Double> getUserConf() {
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 00af367..91fdb09 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -69,6 +69,8 @@ import java.util.HashSet;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
@@ -503,6 +505,18 @@ public class Utils {
}
}
+ public static String getString(Object o, String defaultValue) {
+ if (null == o) {
+ return defaultValue;
+ }
+
+ if (o instanceof String) {
+ return (String) o;
+ } else {
+ throw new IllegalArgumentException("Don't know how to convert " + o + " + to String");
+ }
+ }
+
public static long secureRandomLong() {
return UUID.randomUUID().getLeastSignificantBits();
}
@@ -763,6 +777,30 @@ public class Utils {
public static double zeroIfNaNOrInf(double x) {
return (Double.isNaN(x) || Double.isInfinite(x)) ? 0.0 : x;
+
+ /**
+ * parses the arguments to extract jvm heap memory size.
+ * @param input
+ * @param defaultValue
+ * @return the value of the JVM heap memory setting in a java command.
+ */
+ public static Double parseWorkerChildOpts(String input, Double defaultValue) {
+ if(input != null) {
+ Pattern optsPattern = Pattern.compile("Xmx[0-9]+m");
+ Matcher m = optsPattern.matcher(input);
+ String memoryOpts = null;
+ while (m.find()) {
+ memoryOpts = m.group();
+ }
+ if(memoryOpts!=null) {
+ memoryOpts = memoryOpts.replaceAll("[a-zA-Z]", "");
+ return Double.parseDouble(memoryOpts);
+ } else {
+ return defaultValue;
+ }
+ } else {
+ return defaultValue;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/py/storm/ttypes.py
----------------------------------------------------------------------
diff --git a/storm-core/src/py/storm/ttypes.py b/storm-core/src/py/storm/ttypes.py
index ea3c768..4694ba2 100644
--- a/storm-core/src/py/storm/ttypes.py
+++ b/storm-core/src/py/storm/ttypes.py
@@ -2590,12 +2590,12 @@ class SupervisorSummary:
iprot.skip(ftype)
elif fid == 8:
if ftype == TType.DOUBLE:
- self.used_mem = iprot.readDouble();
+ self.used_mem = iprot.readDouble()
else:
iprot.skip(ftype)
elif fid == 9:
if ftype == TType.DOUBLE:
- self.used_cpu = iprot.readDouble();
+ self.used_cpu = iprot.readDouble()
else:
iprot.skip(ftype)
else:
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/index.html b/storm-core/src/ui/public/index.html
index d113e30..454fe64 100644
--- a/storm-core/src/ui/public/index.html
+++ b/storm-core/src/ui/public/index.html
@@ -146,7 +146,6 @@ $(document).ready(function() {
$.getJSON("/api/v1/cluster/configuration", function(json){
var displayResource = json["scheduler.display.resource"];
if (!displayResource){
- $('#topology-summary td:nth-child(9),#topology-summary th:nth-child(9)').hide();
$('#topology-summary td:nth-child(10),#topology-summary th:nth-child(10)').hide();
}
});
@@ -167,7 +166,6 @@ $(document).ready(function() {
var displayResource = json["scheduler.display.resource"];
if (!displayResource){
$('#supervisor-summary td:nth-child(6),#supervisor-summary th:nth-child(6)').hide();
- $('#supervisor-summary td:nth-child(7),#supervisor-summary th:nth-child(7)').hide();
$('#supervisor-summary td:nth-child(8),#supervisor-summary th:nth-child(8)').hide();
$('#supervisor-summary td:nth-child(9),#supervisor-summary th:nth-child(9)').hide();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/topology-page-template.html b/storm-core/src/ui/public/templates/topology-page-template.html
index 0297839..1f81f1b 100644
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@ -64,6 +64,11 @@
</span>
</th>
<th>
+ <span data-toggle="tooltip" data-placement="above" title="Assigned Total Memory by Scheduler.">
+ Assigned Mem (MB)
+ </span>
+ </th>
+ <th>
<span data-toggle="tooltip" data-placement="left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster.">
Scheduler Info
</span>
@@ -81,6 +86,7 @@
<td>{{executorsTotal}}</td>
<td>{{tasksTotal}}</td>
<td>{{replicationCount}}</td>
+ <td>{{assignedTotalMem}}</td>
<td>{{schedulerInfo}}</td>
</tr>
</tbody>
http://git-wip-us.apache.org/repos/asf/storm/blob/ce7085ba/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index b062196..c5b8684 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -294,7 +294,7 @@ $(document).ready(function() {
topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
topologyResources.append(Mustache.render($(template).filter("#topology-resources-template").html(),response));
$.getJSON("/api/v1/cluster/configuration", function(json){
- var displayResource = json["scheduler.resource.display"];
+ var displayResource = json["scheduler.display.resource"];
if (!displayResource){
$('#topology-resources-header').hide();
$('#topology-resources').hide();
[06/15] storm git commit: Add schedulerDisplayResource REST API,
which the resource hiding/display is based
Posted by bo...@apache.org.
Add schedulerDisplayResource REST API, which the resource hiding/display is based
Add one rest for topo/id, delete the last REST getJSON call
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/515722df
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/515722df
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/515722df
Branch: refs/heads/master
Commit: 515722dfdee7f5881b219f81263064b16d2f38a6
Parents: 89fff44
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Nov 18 13:50:57 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:39:27 2015 -0600
----------------------------------------------------------------------
docs/documentation/ui-rest-api.md | 4 ++++
storm-core/src/clj/backtype/storm/ui/core.clj | 9 ++++++---
storm-core/src/ui/public/index.html | 14 --------------
.../src/ui/public/templates/index-page-template.html | 12 ++++++++++++
storm-core/src/ui/public/topology.html | 12 +++++-------
5 files changed, 27 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/515722df/docs/documentation/ui-rest-api.md
----------------------------------------------------------------------
diff --git a/docs/documentation/ui-rest-api.md b/docs/documentation/ui-rest-api.md
index f2d36a9..1fabcf5 100644
--- a/docs/documentation/ui-rest-api.md
+++ b/docs/documentation/ui-rest-api.md
@@ -143,6 +143,7 @@ Sample response:
"usedCPU": 160
}
]
+ "schedulerDisplayResource": true
}
```
@@ -252,6 +253,7 @@ Sample response:
"assignedCpu": 80
}
]
+ "schedulerDisplayResource": true
}
```
@@ -315,6 +317,7 @@ Response fields:
|executorsTotal| Integer |Number of executors used for this topology|
|msgTimeout| Integer | Number of seconds a tuple has before the spout considers it failed |
|windowHint| String | window param value in "hh mm ss" format. Default value is "All Time"|
+|schedulerDisplayResource| Boolean | Whether to display scheduler resource information|
|topologyStats| Array | Array of all the topology related stats per time window|
|topologyStats.windowPretty| String |Duration passed in HH:MM:SS format|
|topologyStats.window| String |User requested time window for metrics|
@@ -373,6 +376,7 @@ Sample response:
"uptimeSeconds": 1759,
"msgTimeout": 30,
"windowHint": "10m 0s",
+ "schedulerDisplayResource": true,
"topologyStats": [
{
"windowPretty": "10m 0s",
http://git-wip-us.apache.org/repos/asf/storm/blob/515722df/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index 480dcd3..cc6632a 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -438,7 +438,8 @@
"totalCpu" (get (.get_total_resources s) Config/SUPERVISOR_CPU_CAPACITY)
"usedMem" (.get_used_mem s)
"usedCpu" (.get_used_cpu s)
- "version" (.get_version s)})}))
+ "version" (.get_version s)})
+ "schedulerDisplayResource" (*STORM-CONF* Config/SCHEDULER_DISPLAY_RESOURCE)}))
(defn all-topologies-summary
([]
@@ -469,7 +470,8 @@
"assignedMemOnHeap" (.get_assigned_memonheap t)
"assignedMemOffHeap" (.get_assigned_memoffheap t)
"assignedTotalMem" (+ (.get_assigned_memonheap t) (.get_assigned_memoffheap t))
- "assignedCpu" (.get_assigned_cpu t)})}))
+ "assignedCpu" (.get_assigned_cpu t)})
+ "schedulerDisplayResource" (*STORM-CONF* Config/SCHEDULER_DISPLAY_RESOURCE)}))
(defn topology-stats [window stats]
(let [times (stats-times (:emitted stats))
@@ -633,7 +635,8 @@
"windowHint" window-hint
"msgTimeout" msg-timeout
"configuration" topology-conf
- "visualizationTable" []}))))
+ "visualizationTable" []
+ "schedulerDisplayResource" (*STORM-CONF* Config/SCHEDULER_DISPLAY_RESOURCE)}))))
(defn component-errors
[errors-list topology-id secure?]
http://git-wip-us.apache.org/repos/asf/storm/blob/515722df/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/index.html b/storm-core/src/ui/public/index.html
index 454fe64..eae423c 100644
--- a/storm-core/src/ui/public/index.html
+++ b/storm-core/src/ui/public/index.html
@@ -143,12 +143,6 @@ $(document).ready(function() {
]
});
$('#topology-summary [data-toggle="tooltip"]').tooltip();
- $.getJSON("/api/v1/cluster/configuration", function(json){
- var displayResource = json["scheduler.display.resource"];
- if (!displayResource){
- $('#topology-summary td:nth-child(10),#topology-summary th:nth-child(10)').hide();
- }
- });
});
});
$.getJSON("/api/v1/supervisor/summary",function(response,status,jqXHR) {
@@ -162,14 +156,6 @@ $(document).ready(function() {
]
});
$('#supervisor-summary [data-toggle="tooltip"]').tooltip();
- $.getJSON("/api/v1/cluster/configuration", function(json){
- var displayResource = json["scheduler.display.resource"];
- if (!displayResource){
- $('#supervisor-summary td:nth-child(6),#supervisor-summary th:nth-child(6)').hide();
- $('#supervisor-summary td:nth-child(8),#supervisor-summary th:nth-child(8)').hide();
- $('#supervisor-summary td:nth-child(9),#supervisor-summary th:nth-child(9)').hide();
- }
- });
});
});
$.getJSON("/api/v1/cluster/configuration",function(response,status,jqXHR) {
http://git-wip-us.apache.org/repos/asf/storm/blob/515722df/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/templates/index-page-template.html b/storm-core/src/ui/public/templates/index-page-template.html
index bf1f108..5173356 100644
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@ -154,11 +154,13 @@
Assigned Mem (MB)
</span>
</th>
+ {{#schedulerDisplayResource}}
<th>
<span data-toggle="tooltip" data-placement="above" title="Assigned Total CPU by Scheduler. Every 100 means 1 core.">
Assigned CPU (%)
</span>
</th>
+ {{/schedulerDisplayResource}}
<th>
<span data-toggle="tooltip" data-placement="left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster.">
Scheduler Info
@@ -178,7 +180,9 @@
<td>{{tasksTotal}}</td>
<td>{{replicationCount}}</td>
<td>{{assignedTotalMem}}</td>
+ {{#schedulerDisplayResource}}
<td>{{assignedCpu}}</td>
+ {{/schedulerDisplayResource}}
<td>{{schedulerInfo}}</td>
</tr>
{{/topologies}}
@@ -214,16 +218,19 @@
Used slots
</span>
</th>
+ {{#schedulerDisplayResource}}
<th>
<span data-toggle="tooltip" data-placement="above" title="Memory capacity of a supervisor.">
Total Mem (MB)
</span>
</th>
+ {{/schedulerDisplayResource}}
<th>
<span data-toggle="tooltip" data-placement="left" title="Memory that has been allocated.">
Used Mem (MB)
</span>
</th>
+ {{#schedulerDisplayResource}}
<th>
<span data-toggle="tooltip" data-placement="above" title="CPU capacity of a supervisor. Every 100 means one core.">
Total CPU (%)
@@ -234,6 +241,7 @@
Used CPU (%)
</span>
</th>
+ {{/schedulerDisplayResource}}
<th>
<span data-toggle="tooltip" data-placement="left" title="Version">
Version
@@ -249,10 +257,14 @@
<td>{{uptime}}</td>
<td>{{slotsTotal}}</td>
<td>{{slotsUsed}}</td>
+ {{#schedulerDisplayResource}}
<td>{{totalMem}}</td>
+ {{/schedulerDisplayResource}}
<td>{{usedMem}}</td>
+ {{#schedulerDisplayResource}}
<td>{{totalCpu}}</td>
<td>{{usedCpu}}</td>
+ {{/schedulerDisplayResource}}
<td>{{version}}</td>
</tr>
{{/supervisors}}
http://git-wip-us.apache.org/repos/asf/storm/blob/515722df/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index c5b8684..5869d9a 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -293,13 +293,11 @@ $(document).ready(function() {
searchForm.append(Mustache.render($(template).filter("#search-form-template").html(),{id: topologyId}));
topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
topologyResources.append(Mustache.render($(template).filter("#topology-resources-template").html(),response));
- $.getJSON("/api/v1/cluster/configuration", function(json){
- var displayResource = json["scheduler.display.resource"];
- if (!displayResource){
- $('#topology-resources-header').hide();
- $('#topology-resources').hide();
- }
- });
+ var displayResource = response["schedulerDisplayResource"];
+ if (!displayResource){
+ $('#topology-resources-header').hide();
+ $('#topology-resources').hide();
+ }
topologyActions.append(Mustache.render($(template).filter("#topology-actions-template").html(),buttonJsonData));
topologyStats.append(Mustache.render($(template).filter("#topology-stats-template").html(),response));
//window, emitted, transferred, complete latency, acked, failed
[15/15] storm git commit: Add missing comma
Posted by bo...@apache.org.
Add missing comma
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7fb80a1d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7fb80a1d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7fb80a1d
Branch: refs/heads/master
Commit: 7fb80a1d1e96d15ef10610e4235840a1fc446724
Parents: cbf64fc
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Nov 20 16:25:28 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Nov 20 16:25:28 2015 -0600
----------------------------------------------------------------------
docs/documentation/ui-rest-api.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7fb80a1d/docs/documentation/ui-rest-api.md
----------------------------------------------------------------------
diff --git a/docs/documentation/ui-rest-api.md b/docs/documentation/ui-rest-api.md
index 1fabcf5..7de2bcb 100644
--- a/docs/documentation/ui-rest-api.md
+++ b/docs/documentation/ui-rest-api.md
@@ -142,7 +142,7 @@ Sample response:
"usedMem": 1280,
"usedCPU": 160
}
- ]
+ ],
"schedulerDisplayResource": true
}
```
[14/15] storm git commit: Added STORM-1198 to Changelog
Posted by bo...@apache.org.
Added STORM-1198 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cbf64fc3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cbf64fc3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cbf64fc3
Branch: refs/heads/master
Commit: cbf64fc3f6ed3d0ba65c5b3c69cb95853631d52a
Parents: 0646286
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Nov 20 16:06:40 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Nov 20 16:06:40 2015 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cbf64fc3/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 18a1a81..7cd2419 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-1198: Web UI to show resource usages and Total Resources on all supervisors
* STORM-1167: Add windowing support for storm core.
* STORM-1215: Use Async Loggers to avoid locking and logging overhead
* STORM-1204: Logviewer should graceful report page-not-found instead of 500 for bad topo-id etc
[02/15] storm git commit: Update rest api doc for supervisor summary
Posted by bo...@apache.org.
Update rest api doc for supervisor summary
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dda1c32d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dda1c32d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dda1c32d
Branch: refs/heads/master
Commit: dda1c32d6b6c39c7e3027fab81f2cfce5f9927e7
Parents: 400f215
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Nov 11 19:50:16 2015 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Thu Nov 19 12:35:22 2015 -0600
----------------------------------------------------------------------
docs/documentation/ui-rest-api.md | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/dda1c32d/docs/documentation/ui-rest-api.md
----------------------------------------------------------------------
diff --git a/docs/documentation/ui-rest-api.md b/docs/documentation/ui-rest-api.md
index e1c68c1..f2d36a9 100644
--- a/docs/documentation/ui-rest-api.md
+++ b/docs/documentation/ui-rest-api.md
@@ -120,6 +120,10 @@ Response fields:
|uptimeSeconds| Integer| Shows how long the supervisor is running in seconds|
|slotsTotal| Integer| Total number of available worker slots for this supervisor|
|slotsUsed| Integer| Number of worker slots used on this supervisor|
+|totalMem| Double| Total memory capacity on this supervisor|
+|totalCpu| Double| Total CPU capacity on this supervisor|
+|usedMem| Double| Used memory capacity on this supervisor|
+|usedCpu| Double| Used CPU capacity on this supervisor|
Sample response:
@@ -132,7 +136,11 @@ Sample response:
"uptime": "5m 58s",
"uptimeSeconds": 358,
"slotsTotal": 4,
- "slotsUsed": 3
+ "slotsUsed": 3,
+ "totalMem": 3000,
+ "totalCpu": 400,
+ "usedMem": 1280,
+ "usedCPU": 160
}
]
}