You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by go...@apache.org on 2015/01/28 00:35:52 UTC
incubator-slider git commit: SLIDER-769 Modify Slider app packages
for seamless integration with AMS provider (Ambari Metrics Service)
Repository: incubator-slider
Updated Branches:
refs/heads/develop 96f890b1b -> 618cdc4c4
SLIDER-769 Modify Slider app packages for seamless integration with AMS provider (Ambari Metrics Service)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/618cdc4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/618cdc4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/618cdc4c
Branch: refs/heads/develop
Commit: 618cdc4c45b77f4f2e82cb5b86aeb1300cc005c7
Parents: 96f890b
Author: Gour Saha <go...@apache.org>
Authored: Tue Jan 27 15:31:38 2015 -0800
Committer: Gour Saha <go...@apache.org>
Committed: Tue Jan 27 15:33:14 2015 -0800
----------------------------------------------------------------------
app-packages/hbase/appConfig-default.json | 7 ++--
.../hbase/appConfig-phoenix-default.json | 7 ++--
.../hbase/appConfig-secured-default.json | 7 ++--
app-packages/hbase/ganglia_metrics.json | 38 --------------------
app-packages/hbase/metainfo.xml | 8 ++---
app-packages/hbase/package/scripts/params.py | 11 ++++--
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 32 ++++++++++++++---
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 32 ++++++++++++++---
app-packages/hbase/timeline_metrics.json | 38 ++++++++++++++++++++
9 files changed, 113 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
index f065185..4a91884 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -11,10 +11,9 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
+ "site.global.metric_collector_host": "${NN_HOST}",
+ "site.global.metric_collector_port": "6188",
+ "site.global.metric_collector_lib": "",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-phoenix-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-phoenix-default.json b/app-packages/hbase/appConfig-phoenix-default.json
index d74562f..68df0cc 100644
--- a/app-packages/hbase/appConfig-phoenix-default.json
+++ b/app-packages/hbase/appConfig-phoenix-default.json
@@ -11,10 +11,9 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
+ "site.global.metric_collector_host": "${NN_HOST}",
+ "site.global.metric_collector_port": "6188",
+ "site.global.metric_collector_lib": "",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/appConfig-secured-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-secured-default.json b/app-packages/hbase/appConfig-secured-default.json
index 2a2b08f..553117e 100644
--- a/app-packages/hbase/appConfig-secured-default.json
+++ b/app-packages/hbase/appConfig-secured-default.json
@@ -11,10 +11,9 @@
"site.global.app_user": "${USER_NAME}",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
+ "site.global.metric_collector_host": "${NN_HOST}",
+ "site.global.metric_collector_port": "6188",
+ "site.global.metric_collector_lib": "",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/ganglia_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/ganglia_metrics.json b/app-packages/hbase/ganglia_metrics.json
deleted file mode 100644
index da73d48..0000000
--- a/app-packages/hbase/ganglia_metrics.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "Component": {
- "HBASE_REGIONSERVER": {
- "readRequestsCount": {
- "metric": "regionserver.Server.readRequestCount",
- "pointInTime": false,
- "temporal": true
- },
- "regions": {
- "metric": "regionserver.Server.regionCount",
- "pointInTime": false,
- "temporal": true
- },
- "flushQueueSize": {
- "metric": "regionserver.Server.flushQueueLength",
- "pointInTime": false,
- "temporal": true
- }
- },
- "HBASE_MASTER": {
- "cluster_requests": {
- "metric": "master.Server.clusterRequests",
- "pointInTime": false,
- "temporal": true
- },
- "splitTime_avg_time": {
- "metric": "master.FileSystem.HlogSplitTime_mean",
- "pointInTime": false,
- "temporal": true
- },
- "splitSize_avg_time": {
- "metric": "master.FileSystem.HlogSplitSize_mean",
- "pointInTime": false,
- "temporal": true
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index d5e07a7..1d01ae7 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -55,11 +55,7 @@
</export>
<export>
<name>org.apache.slider.metrics</name>
- <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
- </export>
- <export>
- <name>org.apache.slider.metrics.ui</name>
- <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+ <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value>
</export>
</exports>
</exportGroup>
@@ -81,7 +77,7 @@
<name>HBASE_MASTER</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics</appExports>
<componentExports>
<componentExport>
<name>org.apache.slider.jmx</name>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index 027630b..2da4389 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -26,6 +26,7 @@ import status_params
config = Script.get_config()
hbase_root = config['configurations']['global']['app_root']
+app_name = config['clusterName']
conf_dir = format("{hbase_root}/conf")
daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
@@ -59,8 +60,12 @@ client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf
master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf"))
-ganglia_server_host = default('/configurations/global/ganglia_server_host', '')
-ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663')
+metric_collector_host = config['configurations']['global']['metric_collector_host']
+metric_collector_port = config['configurations']['global']['metric_collector_port']
+metric_collector_lib = config['configurations']['global']['metric_collector_lib']
+has_metric_collector = 1
+if not metric_collector_lib:
+ has_metric_collector = 0
rest_port = config['configurations']['global']['hbase_rest_port']
rest_infoport = config['configurations']['global']['hbase_rest_infoport']
@@ -88,7 +93,7 @@ if security_enabled:
master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
-
+
master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
index a3eb1fc..2b37512 100644
--- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
@@ -28,26 +28,48 @@
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
+{% if has_metric_collector %}
+
+*.timeline.plugin.urls=file://{{metric_collector_lib}}
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.period=10
+hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+jvm.period=10
+jvm.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+rpc.period=10
+rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.sink.timeline.period=10
+hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.serviceName-prefix={{app_name}}-master
+
+{% else %}
+
# Configuration of the "hbase" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
hbase.period=10
-hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "jvm" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
jvm.period=10
-jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+jvm.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "rpc" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
rpc.period=10
-rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+rpc.servers={{metric_collector_host}}:{{metric_collector_port}}
#Ganglia following hadoop example
hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
@@ -59,4 +81,6 @@ hbase.sink.ganglia.period=10
.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}}
+
+{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
index a3eb1fc..1954373 100644
--- a/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+++ b/app-packages/hbase/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
@@ -28,26 +28,48 @@
# If this variable is left out, then the default is no expiration.
hbase.extendedperiod = 3600
+{% if has_metric_collector %}
+
+*.timeline.plugin.urls=file://{{metric_collector_lib}}
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.period=10
+hbase.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+jvm.period=10
+jvm.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+rpc.period=10
+rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
+
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.sink.timeline.period=10
+hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.serviceName-prefix={{app_name}}-rs
+
+{% else %}
+
# Configuration of the "hbase" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
hbase.period=10
-hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "jvm" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
jvm.period=10
-jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+jvm.servers={{metric_collector_host}}:{{metric_collector_port}}
# Configuration of the "rpc" context for ganglia
# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
rpc.period=10
-rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+rpc.servers={{metric_collector_host}}:{{metric_collector_port}}
#Ganglia following hadoop example
hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
@@ -59,4 +81,6 @@ hbase.sink.ganglia.period=10
.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}}
+hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}}
+
+{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/618cdc4c/app-packages/hbase/timeline_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/timeline_metrics.json b/app-packages/hbase/timeline_metrics.json
new file mode 100644
index 0000000..da73d48
--- /dev/null
+++ b/app-packages/hbase/timeline_metrics.json
@@ -0,0 +1,38 @@
+{
+ "Component": {
+ "HBASE_REGIONSERVER": {
+ "readRequestsCount": {
+ "metric": "regionserver.Server.readRequestCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "regions": {
+ "metric": "regionserver.Server.regionCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "flushQueueSize": {
+ "metric": "regionserver.Server.flushQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ }
+ },
+ "HBASE_MASTER": {
+ "cluster_requests": {
+ "metric": "master.Server.clusterRequests",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "splitTime_avg_time": {
+ "metric": "master.FileSystem.HlogSplitTime_mean",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "splitSize_avg_time": {
+ "metric": "master.FileSystem.HlogSplitSize_mean",
+ "pointInTime": false,
+ "temporal": true
+ }
+ }
+ }
+}