You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2016/11/15 19:14:01 UTC
[08/28] ambari git commit: AMBARI-18170 : Kafka Metrics do not show
up in AMS HA enabled cluster. (avijayan)
AMBARI-18170 : Kafka Metrics do not show up in AMS HA enabled cluster. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a7f9090a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a7f9090a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a7f9090a
Branch: refs/heads/branch-2.5
Commit: a7f9090a349f1a11460e114e1bf9066b7267201e
Parents: a105d58
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Nov 14 20:42:18 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Nov 15 11:02:13 2016 -0800
----------------------------------------------------------------------
.../sink/kafka/KafkaTimelineMetricsReporter.java | 13 +++----------
.../sink/kafka/KafkaTimelineMetricsReporterTest.java | 2 +-
.../KAFKA/0.8.1/configuration/kafka-broker.xml | 4 ++--
.../KAFKA/0.8.1/package/scripts/kafka.py | 2 +-
.../KAFKA/0.8.1/package/scripts/params.py | 8 ++++++++
.../stacks/2.2/configs/ranger-admin-upgrade.json | 2 +-
.../stacks/2.2/configs/ranger-usersync-upgrade.json | 2 +-
.../app/assets/data/stacks/HDP-2.2/configurations.json | 6 +++---
8 files changed, 20 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
index 11a1c75..9c16564 100644
--- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
@@ -61,7 +61,7 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
private static final String TIMELINE_METRICS_SEND_INTERVAL_PROPERTY = "kafka.timeline.metrics.sendInterval";
private static final String TIMELINE_METRICS_MAX_ROW_CACHE_SIZE_PROPERTY = "kafka.timeline.metrics.maxRowCacheSize";
- private static final String TIMELINE_HOST_PROPERTY = "kafka.timeline.metrics.host";
+ private static final String TIMELINE_HOSTS_PROPERTY = "kafka.timeline.metrics.hosts";
private static final String TIMELINE_PORT_PROPERTY = "kafka.timeline.metrics.port";
private static final String TIMELINE_PROTOCOL_PROPERTY = "kafka.timeline.metrics.protocol";
private static final String TIMELINE_REPORTER_ENABLED_PROPERTY = "kafka.timeline.metrics.reporter.enabled";
@@ -74,7 +74,6 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
private volatile boolean initialized = false;
private boolean running = false;
private final Object lock = new Object();
- private String collectorUri;
private String hostname;
private String metricCollectorPort;
private String collectors;
@@ -147,17 +146,12 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
int maxRowCacheSize = props.getInt(TIMELINE_METRICS_MAX_ROW_CACHE_SIZE_PROPERTY, MAX_RECS_PER_NAME_DEFAULT);
zookeeperQuorum = props.getString("zookeeper.connect");
- collectors = props.getString(TIMELINE_HOST_PROPERTY, TIMELINE_DEFAULT_HOST);
- metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL);
-
- String metricCollectorHost = props.getString(TIMELINE_HOST_PROPERTY, TIMELINE_DEFAULT_HOST);
metricCollectorPort = props.getString(TIMELINE_PORT_PROPERTY, TIMELINE_DEFAULT_PORT);
+ collectors = props.getString(TIMELINE_HOSTS_PROPERTY, TIMELINE_DEFAULT_HOST + ":" + metricCollectorPort);
+ metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL);
setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval));
- collectorUri = constructTimelineMetricUri(metricCollectorProtocol,
- metricCollectorHost, metricCollectorPort);
-
if (metricCollectorProtocol.contains("https")) {
String trustStorePath = props.getString(SSL_KEYSTORE_PATH_PROPERTY).trim();
String trustStoreType = props.getString(SSL_KEYSTORE_TYPE_PROPERTY).trim();
@@ -181,7 +175,6 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
startReporter(metricsConfig.pollingIntervalSecs());
}
if (LOG.isDebugEnabled()) {
- LOG.debug("CollectorUri = " + collectorUri);
LOG.debug("MetricsSendInterval = " + metricsSendInterval);
LOG.debug("MaxRowCacheSize = " + maxRowCacheSize);
LOG.debug("Excluded metrics prefixes = " + excludedMetricsStr);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
index 9027716..c569d88 100644
--- a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
+++ b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
@@ -79,7 +79,7 @@ public class KafkaTimelineMetricsReporterTest {
properties.setProperty("zookeeper.connect", "localhost:2181");
properties.setProperty("kafka.timeline.metrics.sendInterval", "5900");
properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000");
- properties.setProperty("kafka.timeline.metrics.host", "localhost");
+ properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188");
properties.setProperty("kafka.timeline.metrics.port", "6188");
properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
index f7da62c..ab4d701 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
@@ -357,8 +357,8 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>kafka.timeline.metrics.host</name>
- <value>{{metric_collector_host}}</value>
+ <name>kafka.timeline.metrics.hosts</name>
+ <value>{{metric_collector_hosts}}</value>
<description>Timeline host</description>
<on-ambari-upgrade add="true"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
index f0c8289..635cbe9 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
@@ -99,7 +99,7 @@ def kafka(upgrade_type=None):
kafka_server_config['host.name'] = params.hostname
if params.has_metric_collector:
- kafka_server_config['kafka.timeline.metrics.host'] = params.metric_collector_host
+ kafka_server_config['kafka.timeline.metrics.hosts'] = params.metric_collector_hosts
kafka_server_config['kafka.timeline.metrics.port'] = params.metric_collector_port
kafka_server_config['kafka.timeline.metrics.protocol'] = params.metric_collector_protocol
kafka_server_config['kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
index 11c1ad0..5c26796 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
@@ -144,6 +144,14 @@ if has_metric_collector:
else:
metric_collector_protocol = 'http'
pass
+
+ # Collector hosts
+ metric_collector_hosts = ""
+ if ams_collector_hosts:
+ for host in ams_collector_hosts:
+ metric_collector_hosts += host + ':' + metric_collector_port + ','
+ metric_collector_hosts = metric_collector_hosts[:-1]
+
# Security-related params
security_enabled = config['configurations']['cluster-env']['security_enabled']
kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json
index d348645..0a49368 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json
@@ -455,7 +455,7 @@
"replica.high.watermark.checkpoint.interval.ms": "5000",
"zookeeper.connect": "c6407.ambari.apache.org:2181,c6408.ambari.apache.org:2181,c6409.ambari.apache.org:2181",
"controlled.shutdown.retry.backoff.ms": "5000",
- "kafka.timeline.metrics.host": "{{metric_collector_host}}",
+ "kafka.timeline.metrics.host": "{{metric_collector_hosts}}",
"kafka.timeline.metrics.reporter.sendInterval": "5900",
"num.partitions": "1",
"log.flush.interval.messages": "10000",
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json
index 27af82f..b6615da 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json
@@ -450,7 +450,7 @@
"replica.high.watermark.checkpoint.interval.ms": "5000",
"zookeeper.connect": "c6407.ambari.apache.org:2181,c6408.ambari.apache.org:2181,c6409.ambari.apache.org:2181",
"controlled.shutdown.retry.backoff.ms": "5000",
- "kafka.timeline.metrics.host": "{{metric_collector_host}}",
+ "kafka.timeline.metrics.host": "{{metric_collector_hosts}}",
"kafka.timeline.metrics.reporter.sendInterval": "5900",
"num.partitions": "1",
"log.flush.interval.messages": "10000",
http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json
----------------------------------------------------------------------
diff --git a/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json b/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json
index 85d4a9f..e0aa9eb 100644
--- a/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json
+++ b/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json
@@ -8800,13 +8800,13 @@
}
},
{
- "href" : "http://c6401:8080/api/v1/stacks/HDP/versions/2.2/services/KAFKA/configurations/kafka.timeline.metrics.host",
+ "href" : "http://c6401:8080/api/v1/stacks/HDP/versions/2.2/services/KAFKA/configurations/kafka.timeline.metrics.hosts",
"StackConfigurations" : {
"final" : "false",
"property_description" : "Timeline host",
- "property_name" : "kafka.timeline.metrics.host",
+ "property_name" : "kafka.timeline.metrics.hosts",
"property_type" : [ ],
- "property_value" : "{{metric_collector_host}}",
+ "property_value" : "{{metric_collector_hosts}}",
"service_name" : "KAFKA",
"stack_name" : "HDP",
"stack_version" : "2.2",