You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/05/28 02:10:02 UTC
git commit: Adding metrics details for Accumulo and Storm as well as
enabling Ganglia for Storm.
Repository: incubator-slider
Updated Branches:
refs/heads/develop b88cf33c7 -> 9b792e5cf
Adding metrics details for Accumulo and Storm as well as enabling Ganglia for Storm.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9b792e5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9b792e5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9b792e5c
Branch: refs/heads/develop
Commit: 9b792e5cfeda5bbde3b84dc20e2281390123bdbe
Parents: b88cf33
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue May 27 17:09:44 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue May 27 17:09:44 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo-v1_5/jmx_metrics.json | 41 ++++++++++++++++++++
app-packages/accumulo-v1_5/metainfo.xml | 8 ++++
app-packages/storm-v0_91/appConfig.json | 7 ++--
app-packages/storm-v0_91/ganglia_metrics.json | 31 +++++++++++++++
app-packages/storm-v0_91/jmx_metrics.json | 31 +++++++++++++++
.../storm-v0_91/package/scripts/params.py | 9 +++--
6 files changed, 121 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9b792e5c/app-packages/accumulo-v1_5/jmx_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo-v1_5/jmx_metrics.json b/app-packages/accumulo-v1_5/jmx_metrics.json
new file mode 100644
index 0000000..05a1bd9
--- /dev/null
+++ b/app-packages/accumulo-v1_5/jmx_metrics.json
@@ -0,0 +1,41 @@
+{
+ "Component": {
+ "ACCUMULO_MASTER": {
+ "masterGoalState": {
+ "metric": "/stats/masterGoalState",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "masterState": {
+ "metric": "/stats/masterState",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "deadTabletServers": {
+ "metric": "/stats/deadTabletServers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "totals_ingestrate": {
+ "metric": "/stats/totals/ingestrate",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "totals_queryrate": {
+ "metric": "/stats/totals/queryrate",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "totals_diskrate": {
+ "metric": "/stats/totals/diskrate",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "badTabletServers": {
+ "metric": "/stats/badTabletServers",
+ "pointInTime": true,
+ "temporal": false
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9b792e5c/app-packages/accumulo-v1_5/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo-v1_5/metainfo.xml b/app-packages/accumulo-v1_5/metainfo.xml
index 0224b92..79bef1d 100644
--- a/app-packages/accumulo-v1_5/metainfo.xml
+++ b/app-packages/accumulo-v1_5/metainfo.xml
@@ -38,11 +38,19 @@
<name>org.apache.slider.monitor</name>
<value>http://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}</value>
</export>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml</value>
+ </export>
</exports>
</exportGroup>
</exportGroups>
<commandOrders>
<commandOrder>
+ <command>ACCUMULO_MASTER-START</command>
+ <requires>ACCUMULO_MONITOR-INSTALLED</requires>
+ </commandOrder>
+ <commandOrder>
<command>ACCUMULO_TSERVER-START</command>
<requires>ACCUMULO_MASTER-STARTED</requires>
</commandOrder>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9b792e5c/app-packages/storm-v0_91/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/appConfig.json b/app-packages/storm-v0_91/appConfig.json
index ce76b16..e51c9b8 100644
--- a/app-packages/storm-v0_91/appConfig.json
+++ b/app-packages/storm-v0_91/appConfig.json
@@ -14,6 +14,7 @@
"site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
+ "site.global.ganglia_enabled":"true",
"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.storm-site.topology.tuple.serializer": "backtype.storm.serialization.types.ListDelegateSerializer",
@@ -42,8 +43,8 @@
"site.storm-site.storm.local.mode.zmq": "false",
"site.storm-site.topology.max.task.parallelism": "null",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8669,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8669,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.drpc.queue.size": "128",
"site.storm-site.storm.zookeeper.retry.times": "5",
"site.storm-site.nimbus.monitor.freq.secs": "10",
@@ -81,7 +82,7 @@
"site.storm-site.topology.executor.receive.buffer.size": "1024",
"site.storm-site.topology.stats.sample.rate": "0.05",
"site.storm-site.topology.fall.back.on.java.serialization": "true",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8669,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.topology.enable.message.timeouts": "true",
"site.storm-site.storm.messaging.netty.max_wait_ms": "1000",
"site.storm-site.nimbus.topology.validator": "backtype.storm.nimbus.DefaultTopologyValidator",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9b792e5c/app-packages/storm-v0_91/ganglia_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/ganglia_metrics.json b/app-packages/storm-v0_91/ganglia_metrics.json
new file mode 100644
index 0000000..478649b
--- /dev/null
+++ b/app-packages/storm-v0_91/ganglia_metrics.json
@@ -0,0 +1,31 @@
+{
+ "Component": {
+ "NIMBUS_SERVER": {
+ "totalslots": {
+ "metric": "Total Slots",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "totalexecutors": {
+ "metric": "Total Executors",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "topologies": {
+ "metric": "Topologies",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "totaltasks": {
+ "metric": "Total Tasks",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "usedslots": {
+ "metric": "Used Slots",
+ "pointInTime": false,
+ "temporal": true
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9b792e5c/app-packages/storm-v0_91/jmx_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/jmx_metrics.json b/app-packages/storm-v0_91/jmx_metrics.json
new file mode 100644
index 0000000..fa97527
--- /dev/null
+++ b/app-packages/storm-v0_91/jmx_metrics.json
@@ -0,0 +1,31 @@
+{
+ "Component": {
+ "NIMBUS_SERVER": {
+ "FreeSlots": {
+ "metric": "$['slots.free']",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "Tasks": {
+ "metric": "$['tasks.total']",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "Executors": {
+ "metric": "$['executors.total']",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "Topologies": {
+ "metric": "$['topologies']",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "NimbusUptime": {
+ "metric": "$['nimbus.uptime']",
+ "pointInTime": true,
+ "temporal": false
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9b792e5c/app-packages/storm-v0_91/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/package/scripts/params.py b/app-packages/storm-v0_91/package/scripts/params.py
index 23e9112..d459117 100644
--- a/app-packages/storm-v0_91/package/scripts/params.py
+++ b/app-packages/storm-v0_91/package/scripts/params.py
@@ -41,8 +41,11 @@ rest_api_conf_file = format("{conf_dir}/config.yaml")
rest_lib_dir = format("{app_root}/contrib/storm-rest")
storm_bin = format("{app_root}/bin/storm")
-ganglia_installed = False
-
+ganglia_installed = config['configurations']['global']['ganglia_enabled']
+if ganglia_installed:
+ ganglia_report_interval = 60
+ ganglia_server = config['configurations']['global']['ganglia_server_host']
+
_authentication = config['configurations']['core-site']['hadoop.security.authentication']
security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
@@ -50,5 +53,5 @@ if security_enabled:
_hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['global']['kerberos_domain']
_storm_principal_name = config['configurations']['global']['storm_principal_name']
- storm_jaas_principal = _storm_principal_name.replace('_HOST',_hostname_lowercase)
+ storm_jaas_principal = _storm_principal_name.replace('_HOST', _hostname_lowercase)
storm_keytab_path = config['configurations']['global']['storm_keytab']