You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2014/11/21 00:53:29 UTC
ambari git commit: AMBARI-7681. Add Metrics Service to common
services stack. Additional fixes. (mpapirkovskyy)
Repository: ambari
Updated Branches:
refs/heads/branch-metrics-dev a3ed7a3fd -> dd066b945
AMBARI-7681. Add Metrics Service to common services stack. Additional fixes. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dd066b94
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dd066b94
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dd066b94
Branch: refs/heads/branch-metrics-dev
Commit: dd066b945490cb56fcdbf01782970056af9885c3
Parents: a3ed7a3
Author: Myroslav Papirkovskyy <mp...@hortonworks.com>
Authored: Thu Nov 20 10:07:23 2014 +0200
Committer: Myroslav Papirkovskyy <mp...@hortonworks.com>
Committed: Thu Nov 20 11:53:31 2014 +0200
----------------------------------------------------------------------
.../metrics2/sink/timeline/TimelineMetric.java | 8 ++++---
.../conf/unix/ambari-metrics-collector | 2 +-
.../pom.xml | 21 +++++++++++++++++--
.../metrics/timeline/PhoenixTransactSQL.java | 14 ++++++++++++-
.../main/python/core/application_metric_map.py | 8 +++++--
.../src/main/python/core/controller.py | 2 +-
.../src/main/python/core/emitter.py | 4 ++--
.../src/main/python/core/host_info.py | 9 +++++---
ambari-metrics/pom.xml | 6 +++---
.../templates/hadoop-metrics2.properties.j2 | 5 +++++
.../2.2/services/AMS/configuration/ams-env.xml | 2 --
.../AMS/package/templates/metric_groups.conf.j2 | 22 ++------------------
12 files changed, 63 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
index 1b35d92..68b4be8 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
@@ -127,7 +127,8 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
if (!metricName.equals(metric.metricName)) return false;
if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null)
return false;
- if (!appId.equals(metric.appId)) return false;
+ if (appId != null ? !appId.equals(metric.appId) : metric.appId != null)
+ return false;
if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null)
return false;
if (timestamp != metric.timestamp) return false;
@@ -140,7 +141,8 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
if (!metricName.equals(metric.metricName)) return false;
if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null)
return false;
- if (!appId.equals(metric.appId)) return false;
+ if (appId != null ? !appId.equals(metric.appId) : metric.appId != null)
+ return false;
if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null)
return false;
@@ -150,7 +152,7 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
@Override
public int hashCode() {
int result = metricName.hashCode();
- result = 31 * result + appId.hashCode();
+ result = 31 * result + (appId != null ? appId.hashCode() : 0);
result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0);
result = 31 * result + (hostName != null ? hostName.hashCode() : 0);
result = 31 * result + (int) (timestamp ^ (timestamp >>> 32));
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector b/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector
index eed1c9c..b6e17cf 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/conf/unix/ambari-metrics-collector
@@ -200,7 +200,7 @@ case "$1" in
rm -f "${PIDFILE}" >/dev/null 2>&1
fi
- nohup "${JAVA}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 &
+ nohup "${JAVA}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 &
PID=$!
write_pidfile "${PIDFILE}"
sleep 2
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml
index 06f84be..7efdb6b 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/pom.xml
@@ -56,6 +56,8 @@
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <includeScope>compile</includeScope>
+ <excludeScope>test</excludeScope>
</configuration>
</execution>
</executions>
@@ -150,7 +152,7 @@
<source>
<location>target/embedded/${hbase.folder}</location>
<excludes>
- <exclude>target/embedded/${hbase.folder}/bin/*</exclude>
+ <exclude>bin/*</exclude>
</excludes>
</source>
</sources>
@@ -220,7 +222,17 @@
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
- <version>4.2.0.2.2.0.0-2038</version>
+ <version>4.2.0.2.2.0.0-2041</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -324,6 +336,11 @@
<artifactId>jersey-guice</artifactId>
<version>1.11</version>
</dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>1.11</version>
+ </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java
index ed8f978..77bd0f9 100644
--- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java
+++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixTransactSQL.java
@@ -242,23 +242,34 @@ public class PhoenixTransactSQL {
int pos = 1;
if (condition.getMetricNames() != null) {
for (; pos <= condition.getMetricNames().size(); pos++) {
+ LOG.debug("Setting pos: " + pos + ", value = " + condition.getMetricNames().get(pos - 1));
stmt.setString(pos, condition.getMetricNames().get(pos - 1));
}
}
if (condition.getHostname() != null) {
+ LOG.debug("Setting pos: " + pos + ", value: " + condition.getHostname());
stmt.setString(pos++, condition.getHostname());
}
// TODO: Upper case all strings on POST
if (condition.getAppId() != null) {
- stmt.setString(pos++, condition.getAppId().toLowerCase());
+ // TODO: fix case of appId coming from host metrics
+ String appId = condition.getAppId();
+ if (!condition.getAppId().equals("HOST")) {
+ appId = appId.toLowerCase();
+ }
+ LOG.debug("Setting pos: " + pos + ", value: " + appId);
+ stmt.setString(pos++, appId);
}
if (condition.getInstanceId() != null) {
+ LOG.debug("Setting pos: " + pos + ", value: " + condition.getInstanceId());
stmt.setString(pos++, condition.getInstanceId());
}
if (condition.getStartTime() != null) {
+ LOG.debug("Setting pos: " + pos + ", value: " + condition.getStartTime());
stmt.setLong(pos++, condition.getStartTime());
}
if (condition.getEndTime() != null) {
+ LOG.debug("Setting pos: " + pos + ", value: " + condition.getEndTime());
stmt.setLong(pos, condition.getEndTime());
}
if (condition.getFetchSize() != null) {
@@ -268,6 +279,7 @@ public class PhoenixTransactSQL {
return stmt;
}
+
public static PreparedStatement prepareGetAggregateSqlStmt(
Connection connection, Condition condition) throws SQLException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
index 8c32661..0b3f1dd 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
@@ -95,8 +95,8 @@ class ApplicationMetricMap:
timeline_metric = {
"hostname" : self.hostname,
"metricname" : metricId,
- "appid" : self.get_app_id(appId),
- "instanceid" : self.get_instance_id(appId),
+ "appid" : "HOST",
+ "instanceid" : "",
"starttime" : self.get_start_time(appId, metricId),
"metrics" : metricData
}
@@ -125,3 +125,7 @@ class ApplicationMetricMap:
parts = app_id.split("_")
return parts[1] if len(parts) > 1 else ''
pass
+
+ def clear(self):
+ self.app_metric_map.clear()
+ pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
index 3d1f487..d5299d1 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/controller.py
@@ -70,7 +70,7 @@ class Controller(threading.Thread):
def enqueque_events(self):
# Queue events for up to a minute
for event in self.events_cache:
- t = Timer(event.get_collect_interval(), self.metric_collector.process_event(event))
+ t = Timer(event.get_collect_interval(), self.metric_collector.process_event, args=(event,))
t.start()
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
index a107374..7045c2e 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
@@ -55,7 +55,7 @@ class Emitter(threading.Thread):
pass
response = self.push_metrics(json_data)
- if response and response.getcode() == '200':
+ if response and response.getcode() == 200:
retry_count = MAX_RETRY_COUNT
self.application_metric_map.clear()
self.application_metric_map.release_lock()
@@ -83,7 +83,7 @@ class Emitter(threading.Thread):
response = urllib2.urlopen(req, timeout=int(self.send_interval - 10))
if response:
logger.debug("POST response from server: retcode = {0}".format(response.getcode()))
- logger.debug(response.read())
+ logger.debug(str(response.read()))
pass
return response
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
index 4b39119..7ca11b6 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
@@ -74,7 +74,8 @@ class HostInfo():
'mem_cached' : mem_stats.cached if hasattr(mem_stats, 'cached') else '',
'swap_free' : swap_stats.free if hasattr(mem_stats, 'free') else '',
'disk_free' : disk_usage.get("disk_free"),
- 'part_max_used' : disk_usage.get("max_part_used")[0],
+ # todo: cannot send string
+ #'part_max_used' : disk_usage.get("max_part_used")[0],
'disk_total' : disk_usage.get("disk_total")
}
pass
@@ -127,8 +128,10 @@ class HostInfo():
return { "disk_total" : bytes2human(combined_disk_total),
"disk_used" : bytes2human(combined_disk_used),
"disk_free" : bytes2human(combined_disk_free),
- "disk_percent" : bytes2human(combined_disk_percent),
- "max_part_used" : max_percent_usage }
+ "disk_percent" : bytes2human(combined_disk_percent)
+ # todo: cannot send string
+ #"max_part_used" : max_percent_usage }
+ }
pass
def get_host_static_info(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml
index 67f5892..2ee4c1e 100644
--- a/ambari-metrics/pom.xml
+++ b/ambari-metrics/pom.xml
@@ -35,8 +35,8 @@
<python.ver>python >= 2.6</python.ver>
<deb.python.ver>python (>= 2.6)</deb.python.ver>
<!--TODO change to HDP URL-->
- <hbase.tar>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0/tars/hbase-0.98.4.2.2.0.0-1995-hadoop2.tar.gz</hbase.tar>
- <hbase.folder>hbase-0.98.4.2.2.0.0-1995-hadoop2</hbase.folder>
+ <hbase.tar>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0/tars/hbase-0.98.4.2.2.0.0-2041-hadoop2.tar.gz</hbase.tar>
+ <hbase.folder>hbase-0.98.4.2.2.0.0-2041-hadoop2</hbase.folder>
</properties>
<repositories>
<repository>
@@ -196,4 +196,4 @@
</profiles>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
index d1b82b6..4b35ff4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -66,6 +66,11 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
{% if has_metric_collector %}
+*.period=60
+
+*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.TimelineMetricsSink
+*.sink.timeline.period=10
+
datanode.sink.timeline.collector={{metric_collector_host}}:8188
namenode.sink.timeline.collector={{metric_collector_host}}:8188
resourcemanager.sink.timeline.collector={{metric_collector_host}}:8188
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml
index 746db5c..fda1df0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/configuration/ams-env.xml
@@ -34,8 +34,6 @@
# The java implementation to use. Java 1.6 required.
export JAVA_HOME={{java64_home}}
- export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:{{hadoop_native_lib}}
-
#TODO
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/dd066b94/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
index f86343f..aa03d19 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
@@ -18,26 +18,8 @@
{
"host_metric_groups": {
- "cpu_info": {
- "collect_every": "15",
- "metrics": [
- {
- "name": "cpu_user",
- "value_threshold": "1.0"
- }
- ]
- },
- "disk_info": {
- "collect_every": "30",
- "metrics": [
- {
- "name": "disk_free",
- "value_threshold": "5.0"
- }
- ]
- },
- "network_info": {
- "collect_every": "20",
+ "all": {
+ "collect_every": "10",
"metrics": [
{
"name": "bytes_out",