You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/11/30 10:01:08 UTC
incubator-eagle git commit: [EAGLE-816] Fix host problem in jmx
collector script
Repository: incubator-eagle
Updated Branches:
refs/heads/master f39ffe915 -> 68a3e8bde
[EAGLE-816] Fix host problem in jmx collector script
[EAGLE-816] Fix host problem in jmx collector script
Fix host name bug in jmx collector scripts, which current always only use current fadn instead of configured host, it's a bug.
Author: Hao Chen <ha...@apache.org>
Closes #701 from haoch/EAGLE-816.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/68a3e8bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/68a3e8bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/68a3e8bd
Branch: refs/heads/master
Commit: 68a3e8bde3418add34c0b585a206236ee227c6e0
Parents: f39ffe9
Author: Hao Chen <ha...@apache.org>
Authored: Wed Nov 30 18:00:51 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Wed Nov 30 18:00:51 2016 +0800
----------------------------------------------------------------------
.../hadoop_jmx_collector/hadoop_ha_checker.py | 19 +++++++++++++++++--
.../hadoop_jmx_collector/metric_collector.py | 20 ++++++++++----------
2 files changed, 27 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/68a3e8bd/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
index e8bf4ae..dcca28e 100644
--- a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
+++ b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
@@ -17,17 +17,19 @@
#
from metric_collector import MetricCollector, JmxReader, YarnWSReader, Runner
-import logging,socket
+import logging,socket,string
class HadoopNNHAChecker(MetricCollector):
def run(self):
hosts = []
-
+ host_name_list = []
for input in self.config["input"]:
if not input.has_key("host"):
input["host"] = socket.getfqdn()
if input.has_key("component") and input["component"] == "namenode":
hosts.append(input)
+ host_name_list.append(input["host"])
+
if not bool(hosts):
logging.warn("non hosts are configured as 'namenode' in 'input' config, exit")
return
@@ -35,7 +37,10 @@ class HadoopNNHAChecker(MetricCollector):
logging.info("Checking namenode HA: " + str(hosts))
total_count = len(hosts)
+ all_hosts_name = string.join(host_name_list,",")
+
self.collect({
+ "host": all_hosts_name,
"component": "namenode",
"metric": "hadoop.namenode.hastate.total.count",
"value": total_count
@@ -63,18 +68,21 @@ class HadoopNNHAChecker(MetricCollector):
logging.exception("failed to read jmx from " + host["host"] + ":" + host["port"])
failed_count += 1
self.collect({
+ "host": all_hosts_name,
"component": "namenode",
"metric": "hadoop.namenode.hastate.active.count",
"value": active_count
})
self.collect({
+ "host": all_hosts_name,
"component": "namenode",
"metric": "hadoop.namenode.hastate.standby.count",
"value": standby_count
})
self.collect({
+ "host": all_hosts_name,
"component": "namenode",
"metric": "hadoop.namenode.hastate.failed.count",
"value": failed_count
@@ -83,19 +91,23 @@ class HadoopNNHAChecker(MetricCollector):
class HadoopRMHAChecker(MetricCollector):
def run(self):
hosts = []
+ all_hosts = []
for input in self.config["input"]:
if not input.has_key("host"):
input["host"] = socket.getfqdn()
if input.has_key("component") and input["component"] == "resourcemanager":
hosts.append(input)
+ all_hosts.append(input["host"])
if not bool(hosts):
logging.warn("Non hosts are configured as 'resourcemanager' in 'input' config, exit")
return
logging.info("Checking resource manager HA: " + str(hosts))
total_count = len(hosts)
+ all_hosts_name = string.join(all_hosts,",")
self.collect({
+ "host": all_hosts_name,
"component": "resourcemanager",
"metric": "hadoop.resourcemanager.hastate.total.count",
"value": total_count
@@ -120,18 +132,21 @@ class HadoopRMHAChecker(MetricCollector):
failed_count += 1
self.collect({
+ "host": all_hosts_name,
"component": "resourcemanager",
"metric": "hadoop.resourcemanager.hastate.active.count",
"value": active_count
})
self.collect({
+ "host": all_hosts_name,
"component": "resourcemanager",
"metric": "hadoop.resourcemanager.hastate.standby.count",
"value": standby_count
})
self.collect({
+ "host": all_hosts_name,
"component": "resourcemanager",
"metric": "hadoop.resourcemanager.hastate.failed.count",
"value": failed_count
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/68a3e8bd/eagle-external/hadoop_jmx_collector/metric_collector.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/metric_collector.py b/eagle-external/hadoop_jmx_collector/metric_collector.py
index 2e1520a..6eeec5a 100644
--- a/eagle-external/hadoop_jmx_collector/metric_collector.py
+++ b/eagle-external/hadoop_jmx_collector/metric_collector.py
@@ -251,7 +251,7 @@ class MetricCollector(threading.Thread):
if msg.has_key("value"):
msg["value"] = float(str(msg["value"]))
if not msg.has_key("host") or len(msg["host"]) == 0:
- msg["host"] = self.fqdn
+ raise Exception("host is null: " + str(msg))
if not msg.has_key("site"):
msg["site"] = self.config["env"]["site"]
self.sender.send(msg)
@@ -327,18 +327,18 @@ class JmxMetricCollector(MetricCollector):
for input in self.input_components:
try:
beans = JmxReader(input["host"], input["port"], input["https"]).open().get_jmx_beans()
- self.on_beans(input["component"], beans)
+ self.on_beans(input, beans)
except Exception as e:
logging.exception("Failed to read jmx for " + str(input))
def filter_bean(self, bean, mbean_domain):
return mbean_domain in self.selected_domain
- def on_beans(self, component, beans):
+ def on_beans(self, source, beans):
for bean in beans:
- self.on_bean(component,bean)
+ self.on_bean(source,bean)
- def on_bean(self, component, bean):
+ def on_bean(self, source, bean):
# mbean is of the form "domain:key=value,...,foo=bar"
mbean = bean[u'name']
mbean_domain, mbean_attribute = mbean.rstrip().split(":", 1)
@@ -351,18 +351,19 @@ class JmxMetricCollector(MetricCollector):
metric_prefix_name = self.__build_metric_prefix(mbean_attribute, context)
for key, value in bean.iteritems():
- self.on_bean_kv(metric_prefix_name, component,key, value)
+ self.on_bean_kv(metric_prefix_name, source, key, value)
for listener in self.listeners:
- listener.on_bean(component, bean.copy())
+ listener.on_bean(source, bean.copy())
- def on_bean_kv(self, prefix,component, key, value):
+ def on_bean_kv(self, prefix,source, key, value):
# Skip Tags
if re.match(r'tag.*', key):
return
metric_name = (prefix + '.' + key).lower()
self.on_metric({
- "component": component,
+ "component": source["component"],
+ "host": source["host"],
"metric": metric_name,
"value": value
})
@@ -384,7 +385,6 @@ class JmxMetricCollector(MetricCollector):
metric_prefix_name = '.'.join([i[1] for i in mbean_list])
return ("hadoop." + metric_prefix_name).replace(" ", "").lower()
-
class JmxMetricListener:
def init(self, collector):
self.collector = collector