You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/24 14:19:11 UTC
[04/50] ambari git commit: AMBARI-20103 : Reduce AMS HBase metrics
through whitelist. (avijayan)
AMBARI-20103 : Reduce AMS HBase metrics through whitelist. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b986a3c2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b986a3c2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b986a3c2
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: b986a3c2db753a006718592c7689590bc842c6c6
Parents: ef0ae8b
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Feb 21 16:17:13 2017 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Feb 21 16:17:13 2017 -0800
----------------------------------------------------------------------
ambari-metrics/ambari-metrics-assembly/pom.xml | 12 ++
.../src/main/assembly/collector-windows.xml | 1 +
.../src/main/assembly/collector.xml | 1 +
.../conf/unix/amshbase_metrics_whitelist | 162 +++++++++++++++++++
.../conf/windows/amshbase_metrics_whitelist | 162 +++++++++++++++++++
.../ambari-metrics-timelineservice/pom.xml | 5 +
.../timeline/HBaseTimelineMetricStore.java | 2 +-
.../timeline/TimelineMetricConfiguration.java | 39 +++++
.../metrics/timeline/TimelineMetricsFilter.java | 45 +++---
.../timeline/TimelineMetricsFilterTest.java | 59 ++++++-
.../timeline/discovery/TestMetadataManager.java | 15 +-
11 files changed, 475 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml
index cdcc2a7..a4b87de 100644
--- a/ambari-metrics/ambari-metrics-assembly/pom.xml
+++ b/ambari-metrics/ambari-metrics-assembly/pom.xml
@@ -338,6 +338,9 @@
<location>${collector.dir}/conf/unix/metrics_whitelist</location>
</source>
<source>
+ <location>${collector.dir}/conf/unix/amshbase_metrics_whitelist</location>
+ </source>
+ <source>
<location>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</location>
</source>
</sources>
@@ -882,6 +885,15 @@
</mapper>
</data>
<data>
+ <src>${collector.dir}/conf/unix/amshbase_metrics_whitelist</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <filemode>755</filemode>
+ <prefix>/etc/ambari-metrics-collector/conf</prefix>
+ </mapper>
+ </data>
+ <data>
<src>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</src>
<type>file</type>
<mapper>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
index 1ea6c46..8b7a021 100644
--- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml
@@ -46,6 +46,7 @@
<include>ams-site.xml</include>
<include>log4j.properties</include>
<include>metrics_whitelist</include>
+ <include>amshbase_metrics_whitelist</include>
</includes>
</fileSet>
<fileSet>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml
index 0997b48..7f338be 100644
--- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml
+++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml
@@ -45,6 +45,7 @@
<include>ams-site.xml</include>
<include>log4j.properties</include>
<include>metrics_whitelist</include>
+ <include>amshbase_metrics_whitelist</include>
</includes>
</fileSet>
<fileSet>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist
new file mode 100644
index 0000000..6cf4319
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist
@@ -0,0 +1,162 @@
+jvm.Master.JvmMetrics.ThreadsBlocked
+jvm.Master.JvmMetrics.ThreadsNew
+jvm.Master.JvmMetrics.ThreadsRunnable
+jvm.Master.JvmMetrics.ThreadsTerminated
+jvm.Master.JvmMetrics.ThreadsTimedWaiting
+jvm.Master.JvmMetrics.ThreadsWaiting
+jvm.RegionServer.JvmMetrics.GcCount
+jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcCountParNew
+jvm.RegionServer.JvmMetrics.GcTimeMillis
+jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcTimeMillisParNew
+jvm.RegionServer.JvmMetrics.MemHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemHeapMaxM
+jvm.RegionServer.JvmMetrics.MemHeapUsedM
+jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemNonHeapMaxM
+jvm.RegionServer.JvmMetrics.MemNonHeapUsedM
+jvm.RegionServer.JvmMetrics.ThreadsBlocked
+jvm.RegionServer.JvmMetrics.ThreadsNew
+jvm.RegionServer.JvmMetrics.ThreadsRunnable
+jvm.RegionServer.JvmMetrics.ThreadsTerminated
+jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting
+jvm.RegionServer.JvmMetrics.ThreadsWaiting
+master.AssignmentManger.ritCount
+master.AssignmentManger.ritCountOverThreshold
+master.AssignmentManger.ritOldestAge
+master.Master.TotalCallTime_num_ops
+master.Master.exceptions
+master.Server.averageLoad
+master.Server.numDeadRegionServers
+master.Server.numRegionServers
+regionserver.RegionServer.ProcessCallTime_75th_percentile
+regionserver.RegionServer.ProcessCallTime_95th_percentile
+regionserver.RegionServer.ProcessCallTime_99th_percentile
+regionserver.RegionServer.ProcessCallTime_max
+regionserver.RegionServer.ProcessCallTime_mean
+regionserver.RegionServer.ProcessCallTime_median
+regionserver.RegionServer.QueueCallTime_75th_percentile
+regionserver.RegionServer.QueueCallTime_95th_percentile
+regionserver.RegionServer.QueueCallTime_99th_percentile
+regionserver.RegionServer.QueueCallTime_max
+regionserver.RegionServer.QueueCallTime_mean
+regionserver.RegionServer.QueueCallTime_median
+regionserver.RegionServer.TotalCallTime_num_ops
+regionserver.RegionServer.authenticationFailures
+regionserver.RegionServer.authenticationSuccesses
+regionserver.RegionServer.authorizationFailures
+regionserver.RegionServer.authorizationSuccesses
+regionserver.RegionServer.exceptions
+regionserver.RegionServer.numActiveHandler
+regionserver.RegionServer.numCallsInGeneralQueue
+regionserver.RegionServer.numCallsInPriorityQueue
+regionserver.RegionServer.numCallsInReplicationQueue
+regionserver.RegionServer.numOpenConnections
+regionserver.RegionServer.queueSize
+regionserver.RegionServer.receivedBytes
+regionserver.RegionServer.sentBytes
+regionserver.Server.Append_75th_percentile
+regionserver.Server.Append_95th_percentile
+regionserver.Server.Append_99th_percentile
+regionserver.Server.Append_max
+regionserver.Server.Append_mean
+regionserver.Server.Append_median
+regionserver.Server.Append_min
+regionserver.Server.Append_num_ops
+regionserver.Server.Delete_75th_percentile
+regionserver.Server.Delete_95th_percentile
+regionserver.Server.Delete_99th_percentile
+regionserver.Server.Delete_max
+regionserver.Server.Delete_mean
+regionserver.Server.Delete_median
+regionserver.Server.Delete_min
+regionserver.Server.Delete_num_ops
+regionserver.Server.Get_75th_percentile
+regionserver.Server.Get_95th_percentile
+regionserver.Server.Get_99th_percentile
+regionserver.Server.Get_max
+regionserver.Server.Get_mean
+regionserver.Server.Get_median
+regionserver.Server.Get_min
+regionserver.Server.Get_num_ops
+regionserver.Server.Increment_75th_percentile
+regionserver.Server.Increment_95th_percentile
+regionserver.Server.Increment_99th_percentile
+regionserver.Server.Increment_max
+regionserver.Server.Increment_mean
+regionserver.Server.Increment_median
+regionserver.Server.Increment_min
+regionserver.Server.Increment_num_ops
+regionserver.Server.Mutate_75th_percentile
+regionserver.Server.Mutate_95th_percentile
+regionserver.Server.Mutate_99th_percentile
+regionserver.Server.Mutate_max
+regionserver.Server.Mutate_mean
+regionserver.Server.Mutate_median
+regionserver.Server.Mutate_min
+regionserver.Server.Mutate_num_ops
+regionserver.Server.Replay_75th_percentile
+regionserver.Server.Replay_95th_percentile
+regionserver.Server.Replay_99th_percentile
+regionserver.Server.Replay_max
+regionserver.Server.Replay_mean
+regionserver.Server.Replay_median
+regionserver.Server.Replay_min
+regionserver.Server.Replay_num_ops
+regionserver.Server.ScanNext_num_ops
+regionserver.Server.ScanTime_75th_percentile
+regionserver.Server.ScanTime_95th_percentile
+regionserver.Server.ScanTime_99th_percentile
+regionserver.Server.ScanTime_max
+regionserver.Server.ScanTime_mean
+regionserver.Server.ScanTime_median
+regionserver.Server.blockCacheCount
+regionserver.Server.blockCacheCountHitPercent
+regionserver.Server.blockCacheEvictionCount
+regionserver.Server.blockCacheExpressHitPercent
+regionserver.Server.blockCacheFreeSize
+regionserver.Server.blockCacheHitCount
+regionserver.Server.blockCacheMissCount
+regionserver.Server.blockCacheSize
+regionserver.Server.compactionQueueLength
+regionserver.Server.flushQueueLength
+regionserver.Server.hlogFileCount
+regionserver.Server.hlogFileSize
+regionserver.Server.memStoreSize
+regionserver.Server.percentFilesLocal
+regionserver.Server.readRequestCount
+regionserver.Server.regionCount
+regionserver.Server.slowDeleteCount
+regionserver.Server.slowGetCount
+regionserver.Server.slowPutCount
+regionserver.Server.splitQueueLength
+regionserver.Server.staticBloomSize
+regionserver.Server.staticIndexSize
+regionserver.Server.storeCount
+regionserver.Server.storeFileCount
+regionserver.Server.storeFileIndexSize
+regionserver.Server.storeFileSize
+regionserver.Server.totalRequestCount
+regionserver.Server.writeRequestCount
+regionserver.WAL.AppendSize_75th_percentile
+regionserver.WAL.AppendSize_95th_percentile
+regionserver.WAL.AppendSize_99th_percentile
+regionserver.WAL.AppendSize_max
+regionserver.WAL.AppendSize_mean
+regionserver.WAL.AppendSize_median
+regionserver.WAL.AppendTime_75th_percentile
+regionserver.WAL.AppendTime_95th_percentile
+regionserver.WAL.AppendTime_99th_percentile
+regionserver.WAL.AppendTime_max
+regionserver.WAL.AppendTime_mean
+regionserver.WAL.AppendTime_median
+regionserver.WAL.SyncTime_75th_percentile
+regionserver.WAL.SyncTime_95th_percentile
+regionserver.WAL.SyncTime_99th_percentile
+regionserver.WAL.SyncTime_max
+regionserver.WAL.SyncTime_mean
+regionserver.WAL.SyncTime_median
+regionserver.WAL.SyncTime_num_ops
+regionserver.WAL.appendCount
+regionserver.WAL.slowAppendCount
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist
new file mode 100644
index 0000000..6cf4319
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist
@@ -0,0 +1,162 @@
+jvm.Master.JvmMetrics.ThreadsBlocked
+jvm.Master.JvmMetrics.ThreadsNew
+jvm.Master.JvmMetrics.ThreadsRunnable
+jvm.Master.JvmMetrics.ThreadsTerminated
+jvm.Master.JvmMetrics.ThreadsTimedWaiting
+jvm.Master.JvmMetrics.ThreadsWaiting
+jvm.RegionServer.JvmMetrics.GcCount
+jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcCountParNew
+jvm.RegionServer.JvmMetrics.GcTimeMillis
+jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep
+jvm.RegionServer.JvmMetrics.GcTimeMillisParNew
+jvm.RegionServer.JvmMetrics.MemHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemHeapMaxM
+jvm.RegionServer.JvmMetrics.MemHeapUsedM
+jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM
+jvm.RegionServer.JvmMetrics.MemNonHeapMaxM
+jvm.RegionServer.JvmMetrics.MemNonHeapUsedM
+jvm.RegionServer.JvmMetrics.ThreadsBlocked
+jvm.RegionServer.JvmMetrics.ThreadsNew
+jvm.RegionServer.JvmMetrics.ThreadsRunnable
+jvm.RegionServer.JvmMetrics.ThreadsTerminated
+jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting
+jvm.RegionServer.JvmMetrics.ThreadsWaiting
+master.AssignmentManger.ritCount
+master.AssignmentManger.ritCountOverThreshold
+master.AssignmentManger.ritOldestAge
+master.Master.TotalCallTime_num_ops
+master.Master.exceptions
+master.Server.averageLoad
+master.Server.numDeadRegionServers
+master.Server.numRegionServers
+regionserver.RegionServer.ProcessCallTime_75th_percentile
+regionserver.RegionServer.ProcessCallTime_95th_percentile
+regionserver.RegionServer.ProcessCallTime_99th_percentile
+regionserver.RegionServer.ProcessCallTime_max
+regionserver.RegionServer.ProcessCallTime_mean
+regionserver.RegionServer.ProcessCallTime_median
+regionserver.RegionServer.QueueCallTime_75th_percentile
+regionserver.RegionServer.QueueCallTime_95th_percentile
+regionserver.RegionServer.QueueCallTime_99th_percentile
+regionserver.RegionServer.QueueCallTime_max
+regionserver.RegionServer.QueueCallTime_mean
+regionserver.RegionServer.QueueCallTime_median
+regionserver.RegionServer.TotalCallTime_num_ops
+regionserver.RegionServer.authenticationFailures
+regionserver.RegionServer.authenticationSuccesses
+regionserver.RegionServer.authorizationFailures
+regionserver.RegionServer.authorizationSuccesses
+regionserver.RegionServer.exceptions
+regionserver.RegionServer.numActiveHandler
+regionserver.RegionServer.numCallsInGeneralQueue
+regionserver.RegionServer.numCallsInPriorityQueue
+regionserver.RegionServer.numCallsInReplicationQueue
+regionserver.RegionServer.numOpenConnections
+regionserver.RegionServer.queueSize
+regionserver.RegionServer.receivedBytes
+regionserver.RegionServer.sentBytes
+regionserver.Server.Append_75th_percentile
+regionserver.Server.Append_95th_percentile
+regionserver.Server.Append_99th_percentile
+regionserver.Server.Append_max
+regionserver.Server.Append_mean
+regionserver.Server.Append_median
+regionserver.Server.Append_min
+regionserver.Server.Append_num_ops
+regionserver.Server.Delete_75th_percentile
+regionserver.Server.Delete_95th_percentile
+regionserver.Server.Delete_99th_percentile
+regionserver.Server.Delete_max
+regionserver.Server.Delete_mean
+regionserver.Server.Delete_median
+regionserver.Server.Delete_min
+regionserver.Server.Delete_num_ops
+regionserver.Server.Get_75th_percentile
+regionserver.Server.Get_95th_percentile
+regionserver.Server.Get_99th_percentile
+regionserver.Server.Get_max
+regionserver.Server.Get_mean
+regionserver.Server.Get_median
+regionserver.Server.Get_min
+regionserver.Server.Get_num_ops
+regionserver.Server.Increment_75th_percentile
+regionserver.Server.Increment_95th_percentile
+regionserver.Server.Increment_99th_percentile
+regionserver.Server.Increment_max
+regionserver.Server.Increment_mean
+regionserver.Server.Increment_median
+regionserver.Server.Increment_min
+regionserver.Server.Increment_num_ops
+regionserver.Server.Mutate_75th_percentile
+regionserver.Server.Mutate_95th_percentile
+regionserver.Server.Mutate_99th_percentile
+regionserver.Server.Mutate_max
+regionserver.Server.Mutate_mean
+regionserver.Server.Mutate_median
+regionserver.Server.Mutate_min
+regionserver.Server.Mutate_num_ops
+regionserver.Server.Replay_75th_percentile
+regionserver.Server.Replay_95th_percentile
+regionserver.Server.Replay_99th_percentile
+regionserver.Server.Replay_max
+regionserver.Server.Replay_mean
+regionserver.Server.Replay_median
+regionserver.Server.Replay_min
+regionserver.Server.Replay_num_ops
+regionserver.Server.ScanNext_num_ops
+regionserver.Server.ScanTime_75th_percentile
+regionserver.Server.ScanTime_95th_percentile
+regionserver.Server.ScanTime_99th_percentile
+regionserver.Server.ScanTime_max
+regionserver.Server.ScanTime_mean
+regionserver.Server.ScanTime_median
+regionserver.Server.blockCacheCount
+regionserver.Server.blockCacheCountHitPercent
+regionserver.Server.blockCacheEvictionCount
+regionserver.Server.blockCacheExpressHitPercent
+regionserver.Server.blockCacheFreeSize
+regionserver.Server.blockCacheHitCount
+regionserver.Server.blockCacheMissCount
+regionserver.Server.blockCacheSize
+regionserver.Server.compactionQueueLength
+regionserver.Server.flushQueueLength
+regionserver.Server.hlogFileCount
+regionserver.Server.hlogFileSize
+regionserver.Server.memStoreSize
+regionserver.Server.percentFilesLocal
+regionserver.Server.readRequestCount
+regionserver.Server.regionCount
+regionserver.Server.slowDeleteCount
+regionserver.Server.slowGetCount
+regionserver.Server.slowPutCount
+regionserver.Server.splitQueueLength
+regionserver.Server.staticBloomSize
+regionserver.Server.staticIndexSize
+regionserver.Server.storeCount
+regionserver.Server.storeFileCount
+regionserver.Server.storeFileIndexSize
+regionserver.Server.storeFileSize
+regionserver.Server.totalRequestCount
+regionserver.Server.writeRequestCount
+regionserver.WAL.AppendSize_75th_percentile
+regionserver.WAL.AppendSize_95th_percentile
+regionserver.WAL.AppendSize_99th_percentile
+regionserver.WAL.AppendSize_max
+regionserver.WAL.AppendSize_mean
+regionserver.WAL.AppendSize_median
+regionserver.WAL.AppendTime_75th_percentile
+regionserver.WAL.AppendTime_95th_percentile
+regionserver.WAL.AppendTime_99th_percentile
+regionserver.WAL.AppendTime_max
+regionserver.WAL.AppendTime_mean
+regionserver.WAL.AppendTime_median
+regionserver.WAL.SyncTime_75th_percentile
+regionserver.WAL.SyncTime_95th_percentile
+regionserver.WAL.SyncTime_99th_percentile
+regionserver.WAL.SyncTime_max
+regionserver.WAL.SyncTime_mean
+regionserver.WAL.SyncTime_median
+regionserver.WAL.SyncTime_num_ops
+regionserver.WAL.appendCount
+regionserver.WAL.slowAppendCount
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index 9d269e7..d848eab 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -211,6 +211,9 @@
<location>conf/unix/metrics_whitelist</location>
</source>
<source>
+ <location>conf/unix/amshbase_metrics_whitelist</location>
+ </source>
+ <source>
<location>target/embedded/${hbase.folder}/conf/hbase-site.xml</location>
</source>
</sources>
@@ -252,7 +255,9 @@
<configuration>
<excludes>
<exclude>conf/unix/metrics_whitelist</exclude>
+ <exclude>conf/unix/amshbase_metrics_whitelist</exclude>
<exclude>conf/windows/metrics_whitelist</exclude>
+ <exclude>conf/windows/amshbase_metrics_whitelist</exclude>
</excludes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index ccce3ff..575b8ea 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -114,7 +114,7 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
}
//Initialize whitelisting & blacklisting if needed
- TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
defaultTopNHostsLimit = Integer.parseInt(metricsConf.get(DEFAULT_TOPN_HOSTS_LIMIT, "20"));
if (Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"))) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 92d595e..b7b9968 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -17,17 +17,25 @@
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
/**
* Configuration class that reads properties from ams-site.xml. All values
@@ -265,6 +273,8 @@ public class TimelineMetricConfiguration {
public static final String DEFAULT_INSTANCE_PORT = "12001";
+ public static final String AMSHBASE_METRICS_WHITESLIST_FILE = "amshbase_metrics_whitelist";
+
private Configuration hbaseConf;
private Configuration metricsConf;
private Configuration amsEnvConf;
@@ -426,4 +436,33 @@ public class TimelineMetricConfiguration {
public boolean isSecurityEnabled() {
return hbaseConf.get("hbase.security.authentication", "").equals("kerberos");
}
+
+ public Set<String> getAmshbaseWhitelist() {
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = getClass().getClassLoader();
+ }
+
+ BufferedReader br = null;
+ String strLine;
+ Set<String> whitelist = new HashSet<>();
+
+ try(InputStream inputStream = classLoader.getResourceAsStream(AMSHBASE_METRICS_WHITESLIST_FILE)) {
+ br = new BufferedReader(new InputStreamReader(inputStream));
+
+ while ((strLine = br.readLine()) != null) {
+ strLine = strLine.trim();
+ if (StringUtils.isEmpty(strLine)) {
+ continue;
+ }
+ whitelist.add(strLine);
+ }
+ } catch (IOException ioEx) {
+ LOG.error("Unable to parse ams-hbase metric whitelist file", ioEx);
+ return Collections.EMPTY_SET;
+ }
+
+ return whitelist;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
index d05353d..0fe979e 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,6 +28,7 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashSet;
@@ -41,35 +43,44 @@ public class TimelineMetricsFilter {
private static Set<String> whitelistedMetrics;
private static Set<Pattern> whitelistedMetricPatterns;
- private static boolean metricWhitelistingEnabled;
- private static boolean appBlacklistingEnabled;
private static Set<String> blacklistedApps;
private static String patternPrefix = "._p_";
+ private static Set<String> amshbaseWhitelist;
private static final Log LOG = LogFactory.getLog(TimelineMetricsFilter.class);
- public static void initializeMetricFilter(Configuration metricsConf) {
+ public static void initializeMetricFilter(TimelineMetricConfiguration configuration) {
+
+ Configuration metricsConf = null;
+ try {
+ metricsConf = configuration.getMetricsConf();
+ } catch (Exception e) {
+ LOG.error("Error fetching metrics configuration for getting whitelisting information");
+ return;
+ }
whitelistedMetrics = new HashSet<String>();
whitelistedMetricPatterns = new HashSet<Pattern>();
blacklistedApps = new HashSet<>();
- metricWhitelistingEnabled = false;
- appBlacklistingEnabled = false;
+ amshbaseWhitelist = new HashSet<>();
String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, "");
if (!StringUtils.isEmpty(whitelistFile)) {
- metricWhitelistingEnabled = true;
readMetricWhitelistFromFile(whitelistFile);
}
String appsBlacklist = metricsConf.get(TIMELINE_METRICS_APPS_BLACKLIST, "");
if (!StringUtils.isEmpty(appsBlacklist)) {
- appBlacklistingEnabled = true;
for (String app : appsBlacklist.split(",")) {
blacklistedApps.add(app);
}
LOG.info("Blacklisted apps : " + blacklistedApps.toString());
}
+
+ amshbaseWhitelist = configuration.getAmshbaseWhitelist();
+ if (CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
+ LOG.info("Whitelisting " + amshbaseWhitelist.size() + " ams-hbase metrics");
+ }
}
private static void readMetricWhitelistFromFile(String whitelistFile) {
@@ -108,16 +119,17 @@ public class TimelineMetricsFilter {
public static boolean acceptMetric(TimelineMetric metric) {
- if (!isEnabled()) {
- return true;
- }
-
// App takes precedence.
- if (blacklistedApps.contains(metric.getAppId())) {
+ if (CollectionUtils.isNotEmpty(blacklistedApps) && blacklistedApps.contains(metric.getAppId())) {
return false;
}
- if (!metricWhitelistingEnabled) {
+ //Special Case appId = ams-hbase whitelisting.
+ if ("ams-hbase".equals(metric.getAppId()) && CollectionUtils.isNotEmpty(amshbaseWhitelist)) {
+ return amshbaseWhitelist.contains(metric.getMetricName());
+ }
+
+ if (CollectionUtils.isEmpty(whitelistedMetrics) && CollectionUtils.isEmpty(whitelistedMetricPatterns)) {
return true;
}
@@ -139,15 +151,8 @@ public class TimelineMetricsFilter {
public static void addToWhitelist(String metricName) {
if (StringUtils.isNotEmpty(metricName)) {
- if (!metricWhitelistingEnabled) {
- metricWhitelistingEnabled = true;
- }
-
whitelistedMetrics.add(metricName);
}
}
- public static boolean isEnabled() {
- return appBlacklistingEnabled || metricWhitelistingEnabled;
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
index 049d473..73c66fe 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java
@@ -17,20 +17,31 @@
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.easymock.EasyMock;
import org.junit.Test;
+
import java.net.URL;
+import java.util.HashSet;
+import java.util.Set;
public class TimelineMetricsFilterTest {
@Test
- public void testAppBlacklisting() {
+ public void testAppBlacklisting() throws Exception{
Configuration metricsConf = new Configuration();
metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
- TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
TimelineMetric timelineMetric = new TimelineMetric();
@@ -48,10 +59,14 @@ public class TimelineMetricsFilterTest {
public void testMetricWhitelisting() throws Exception {
Configuration metricsConf = new Configuration();
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+
URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
- TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
TimelineMetric timelineMetric = new TimelineMetric();
@@ -73,11 +88,14 @@ public class TimelineMetricsFilterTest {
Configuration metricsConf = new Configuration();
metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus");
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat");
metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath());
- TimelineMetricsFilter.initializeMetricFilter(metricsConf);
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
TimelineMetric timelineMetric = new TimelineMetric();
@@ -96,4 +114,37 @@ public class TimelineMetricsFilterTest {
Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
}
+ @Test
+ public void testAmshbaseWhitelisting() throws Exception {
+
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+
+ Configuration metricsConf = new Configuration();
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+
+ Set<String> whitelist = new HashSet();
+ whitelist.add("regionserver.Server.Delete_99th_percentile");
+ whitelist.add("regionserver.Server.Delete_max");
+ whitelist.add("regionserver.Server.Delete_mean");
+ expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once();
+
+ replay(configuration);
+
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
+
+ TimelineMetric timelineMetric = new TimelineMetric();
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_max");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("regionserver.Server.Delete_min3333");
+ timelineMetric.setAppId("ams-hbase");
+ Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric));
+
+ timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM");
+ timelineMetric.setAppId("hbase");
+ Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
index c4cf51d..b243e0b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
@@ -17,18 +17,23 @@
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsFilter;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils;
+import org.easymock.EasyMock;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.net.URISyntaxException;
import java.sql.SQLException;
import java.util.Map;
import java.util.Set;
@@ -38,7 +43,7 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
TimelineMetricMetadataManager metadataManager;
@Before
- public void insertDummyRecords() throws IOException, SQLException {
+ public void insertDummyRecords() throws IOException, SQLException, URISyntaxException {
// Initialize new manager
metadataManager = new TimelineMetricMetadataManager(hdb, new Configuration());
final long now = System.currentTimeMillis();
@@ -87,7 +92,11 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
}});
timelineMetrics.getMetrics().add(metric3);
- TimelineMetricsFilter.initializeMetricFilter(new Configuration());
+ Configuration metricsConf = new Configuration();
+ TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class);
+ expect(configuration.getMetricsConf()).andReturn(metricsConf).once();
+ replay(configuration);
+ TimelineMetricsFilter.initializeMetricFilter(configuration);
TimelineMetricsFilter.addToWhitelist("dummy_metric1");
TimelineMetricsFilter.addToWhitelist("dummy_metric2");