You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/02/21 18:21:49 UTC
ambari git commit: AMBARI-20071 : Hadoop metrics sink prints lots of
logs if collector is unavailable. (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 381d43cad -> 4279738af
AMBARI-20071 : Hadoop metrics sink prints lots of logs if collector is unavailable. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4279738a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4279738a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4279738a
Branch: refs/heads/branch-2.5
Commit: 4279738af80a4b448ba24906e2144b5cd0806b63
Parents: 381d43c
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Feb 20 15:01:29 2017 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Feb 21 10:11:53 2017 -0800
----------------------------------------------------------------------
.../timeline/AbstractTimelineMetricsSink.java | 22 +++++++++++++-------
.../timeline/HadoopTimelineMetricsSink.java | 11 +++++++---
2 files changed, 23 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4279738a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index f5a02e4..a1fd008 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -82,6 +82,8 @@ public abstract class AbstractTimelineMetricsSink {
protected static final AtomicInteger failedCollectorConnectionsCounter = new AtomicInteger(0);
public static int NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 100;
+ protected static final AtomicInteger nullCollectorCounter = new AtomicInteger(0);
+ public static int NUMBER_OF_NULL_COLLECTOR_EXCEPTIONS = 20;
public int ZK_CONNECT_TRY_COUNT = 10;
public int ZK_SLEEP_BETWEEN_RETRY_TIME = 2000;
public boolean shardExpired = true;
@@ -214,7 +216,7 @@ public abstract class AbstractTimelineMetricsSink {
collectorHost = targetCollectorHostSupplier.get();
// Last X attempts have failed - force refresh
if (failedCollectorConnectionsCounter.get() > RETRY_COUNT_BEFORE_COLLECTOR_FAILOVER) {
- LOG.info("Removing collector " + collectorHost + " from allKnownLiveCollectors.");
+ LOG.debug("Removing collector " + collectorHost + " from allKnownLiveCollectors.");
allKnownLiveCollectors.remove(collectorHost);
targetCollectorHostSupplier = null;
collectorHost = findPreferredCollectHost();
@@ -224,8 +226,15 @@ public abstract class AbstractTimelineMetricsSink {
}
if (collectorHost == null) {
- LOG.warn("No live collector to send metrics to. Metrics to be sent will be discarded.");
+ if (nullCollectorCounter.getAndIncrement() == 0) {
+ LOG.info("No live collector to send metrics to. Metrics to be sent will be discarded. " +
+ "This message will be skipped for the next " + NUMBER_OF_NULL_COLLECTOR_EXCEPTIONS + " times.");
+ } else {
+ nullCollectorCounter.compareAndSet(NUMBER_OF_NULL_COLLECTOR_EXCEPTIONS, 0);
+ }
return false;
+ } else {
+ nullCollectorCounter.set(0);
}
String connectUrl = getCollectorUri(collectorHost);
@@ -356,7 +365,7 @@ public abstract class AbstractTimelineMetricsSink {
if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null
&& (currentTime - lastFailedZkRequestTime) > zookeeperBackoffTimeMillis) {
- LOG.info("No live collectors from configuration. Requesting zookeeper...");
+ LOG.debug("No live collectors from configuration. Requesting zookeeper...");
allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode());
boolean noNewCollectorFromZk = true;
for (String collectorHostFromZk : allKnownLiveCollectors) {
@@ -366,7 +375,7 @@ public abstract class AbstractTimelineMetricsSink {
}
}
if (noNewCollectorFromZk) {
- LOG.info("No new collector was found from Zookeeper. Will not request zookeeper for " + zookeeperBackoffTimeMillis + " millis");
+ LOG.debug("No new collector was found from Zookeeper. Will not request zookeeper for " + zookeeperBackoffTimeMillis + " millis");
lastFailedZkRequestTime = System.currentTimeMillis();
}
}
@@ -396,7 +405,7 @@ public abstract class AbstractTimelineMetricsSink {
shardExpired = true;
return collectorHost;
}
- LOG.warn("Couldn't find any live collectors. Returning null");
+ LOG.debug("Couldn't find any live collectors. Returning null");
shardExpired = true;
return null;
}
@@ -416,7 +425,7 @@ public abstract class AbstractTimelineMetricsSink {
}
break; // Found at least 1 live collector
} catch (MetricCollectorUnavailableException e) {
- LOG.info("Collector " + hostStr + " is not longer live. Removing " +
+ LOG.debug("Collector " + hostStr + " is not longer live. Removing " +
"it from list of know live collector hosts : " + allKnownLiveCollectors);
allKnownLiveCollectors.remove(hostStr);
}
@@ -473,7 +482,6 @@ public abstract class AbstractTimelineMetricsSink {
LOG.debug(errorMessage);
LOG.debug(ioe);
String warnMsg = "Unable to connect to collector to find live nodes.";
- LOG.warn(warnMsg);
throw new MetricCollectorUnavailableException(warnMsg);
}
return collectors;
http://git-wip-us.apache.org/repos/asf/ambari/blob/4279738a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
index 1936977..988f15d 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
@@ -116,10 +116,15 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
String preferredCollectorHost = findPreferredCollectHost();
collectorUri = constructTimelineMetricUri(protocol, preferredCollectorHost, port);
containerMetricsUri = constructContainerMetricUri(protocol, preferredCollectorHost, port);
+
+ if (StringUtils.isNotEmpty(preferredCollectorHost)) {
+ LOG.info("Collector Uri: " + collectorUri);
+ LOG.info("Container Metrics Uri: " + containerMetricsUri);
+ } else {
+ LOG.info("No suitable collector found.");
+ }
}
- LOG.info("Collector Uri: " + collectorUri);
- LOG.info("Container Metrics Uri: " + containerMetricsUri);
timeoutSeconds = conf.getInt(METRICS_POST_TIMEOUT_SECONDS, DEFAULT_POST_TIMEOUT_SECONDS);
@@ -463,7 +468,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
executorService.submit(new Runnable() {
@Override
public void run() {
- LOG.info("Closing HadoopTimelineMetricSink. Flushing metrics to collector...");
+ LOG.debug("Closing HadoopTimelineMetricSink. Flushing metrics to collector...");
TimelineMetrics metrics = metricsCache.getAllMetrics();
if (metrics != null) {
emitMetrics(metrics);