You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2017/04/26 18:35:57 UTC
ambari git commit: AMBARI-20820 Stack side changes to use instanceId
for cluster based segregation of data (AMS) (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 8a7314583 -> 362d6ecab
AMBARI-20820 Stack side changes to use instanceId for cluster based segregation of data (AMS) (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/362d6eca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/362d6eca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/362d6eca
Branch: refs/heads/trunk
Commit: 362d6ecaba8e1909e26b79928be91ae8a8103461
Parents: 8a73145
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Apr 26 21:35:45 2017 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Apr 26 21:35:45 2017 +0300
----------------------------------------------------------------------
.../timeline/AbstractTimelineMetricsSink.java | 2 ++
.../sink/flume/FlumeTimelineMetricsSink.java | 10 +++++-
.../timeline/HadoopTimelineMetricsSink.java | 9 +++--
.../timeline/HadoopTimelineMetricsSinkTest.java | 14 +++++++-
.../main/python/core/application_metric_map.py | 7 ++--
.../src/main/python/core/config_reader.py | 6 ++++
.../src/main/python/core/emitter.py | 4 ++-
.../kafka/KafkaTimelineMetricsReporter.java | 10 ++++++
.../kafka/KafkaTimelineMetricsReporterTest.java | 4 +++
.../storm/StormTimelineMetricsReporter.java | 7 ++++
.../sink/storm/StormTimelineMetricsSink.java | 7 ++++
.../storm/StormTimelineMetricsReporter.java | 8 ++++-
.../sink/storm/StormTimelineMetricsSink.java | 8 ++++-
.../system/impl/AmbariMetricSinkImpl.java | 36 ++++++++++++--------
.../1.6.1.2.2.0/package/scripts/params.py | 15 ++++++--
.../hadoop-metrics2-accumulo.properties.j2 | 2 ++
.../0.1.0/package/scripts/params.py | 13 ++++---
.../hadoop-metrics2-hbase.properties.j2 | 2 ++
.../package/templates/metric_monitor.ini.j2 | 2 ++
.../FLUME/1.4.0.2.0/package/scripts/params.py | 21 +++++++-----
.../templates/flume-metrics2.properties.j2 | 3 ++
.../0.96.0.2.0/package/scripts/params_linux.py | 14 ++++++--
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 2 ++
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 3 +-
.../hadoop-metrics2.properties.xml | 2 ++
.../0.12.0.2.0/package/scripts/params_linux.py | 17 ++++++---
.../hadoop-metrics2-hivemetastore.properties.j2 | 2 ++
.../hadoop-metrics2-hiveserver2.properties.j2 | 2 ++
.../templates/hadoop-metrics2-llapdaemon.j2 | 4 ++-
.../hadoop-metrics2-llaptaskscheduler.j2 | 4 ++-
.../2.1.0.3.0/package/scripts/params_linux.py | 16 ++++++---
.../hadoop-metrics2-hivemetastore.properties.j2 | 2 ++
.../hadoop-metrics2-hiveserver2.properties.j2 | 2 ++
.../templates/hadoop-metrics2-llapdaemon.j2 | 4 ++-
.../hadoop-metrics2-llaptaskscheduler.j2 | 4 ++-
.../KAFKA/0.8.1/configuration/kafka-broker.xml | 10 ++++++
.../KAFKA/0.8.1/package/scripts/params.py | 15 ++++++--
.../STORM/0.9.1/package/scripts/params_linux.py | 12 +++++--
.../0.9.1/package/templates/config.yaml.j2 | 3 ++
.../templates/storm-metrics2.properties.j2 | 2 ++
.../2.0.6/hooks/before-START/scripts/params.py | 13 +++++--
.../templates/hadoop-metrics2.properties.j2 | 2 ++
.../hadoop-metrics2.properties.xml | 2 ++
.../3.0/hooks/before-START/scripts/params.py | 14 ++++++--
.../templates/hadoop-metrics2.properties.j2 | 2 ++
.../python/stacks/2.0.6/configs/default.json | 4 +--
46 files changed, 278 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 46f32f9..2c6fae2 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -79,6 +79,8 @@ public abstract class AbstractTimelineMetricsSink {
public static final String SSL_KEYSTORE_TYPE_PROPERTY = "truststore.type";
public static final String SSL_KEYSTORE_PASSWORD_PROPERTY = "truststore.password";
public static final String COLLECTOR_LIVE_NODES_PATH = "/ws/v1/timeline/metrics/livenodes";
+ public static final String INSTANCE_ID_PROPERTY = "instanceId";
+ public static final String SET_INSTANCE_ID_PROPERTY = "set.instanceId";
protected static final AtomicInteger failedCollectorConnectionsCounter = new AtomicInteger(0);
public static int NUMBER_OF_SKIPPED_COLLECTOR_EXCEPTIONS = 100;
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
index 3fdf3f4..904c916 100644
--- a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
@@ -61,6 +61,8 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem
private final static String COUNTER_METRICS_PROPERTY = "counters";
private final Set<String> counterMetrics = new HashSet<String>();
private int timeoutSeconds = 10;
+ private boolean setInstanceId;
+ private String instanceId;
@Override
public void start() {
@@ -106,6 +108,8 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem
zookeeperQuorum = configuration.getProperty("zookeeper.quorum");
protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http");
port = configuration.getProperty(COLLECTOR_PORT, "6188");
+ setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false"));
+ instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY, "");
// Initialize the collector write strategy
super.init();
@@ -227,7 +231,11 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostname);
- timelineMetric.setInstanceId(component);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId + component);
+ } else {
+ timelineMetric.setInstanceId(component);
+ }
timelineMetric.setAppId("FLUME_HANDLER");
timelineMetric.setStartTime(currentTimeMillis);
timelineMetric.getMetricValues().put(currentTimeMillis, Double.parseDouble(attributeValue));
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
index 8e0de03..11e16c2 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
@@ -52,6 +52,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
private TimelineMetricsCache metricsCache;
private String hostName = "UNKNOWN.example.com";
private String instanceId = null;
+ private boolean setInstanceId;
private String serviceName = "";
private Collection<String> collectorHosts;
private String collectorUri;
@@ -95,8 +96,8 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
}
serviceName = getServiceName(conf);
- String inst = conf.getString("instanceId", "");
- instanceId = StringUtils.isEmpty(inst) ? null : inst;
+ instanceId = conf.getString(INSTANCE_ID_PROPERTY);
+ setInstanceId = conf.getBoolean(SET_INSTANCE_ID_PROPERTY, false);
LOG.info("Identified hostname = " + hostName + ", serviceName = " + serviceName);
// Initialize the collector write strategy
@@ -321,7 +322,9 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
timelineMetric.setMetricName(name);
timelineMetric.setHostName(hostName);
timelineMetric.setAppId(serviceName);
- timelineMetric.setInstanceId(instanceId);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setStartTime(startTime);
timelineMetric.setType(metric.type() != null ? metric.type().name() : null);
timelineMetric.getMetricValues().put(startTime, value.doubleValue());
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
index 5777639..4a009dc 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
@@ -50,12 +50,15 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.TreeMap;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PORT;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_HOSTS_PROPERTY;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PROTOCOL;
+import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.INSTANCE_ID_PROPERTY;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.MAX_METRIC_ROW_CACHE_SIZE;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.METRICS_SEND_INTERVAL;
+import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.SET_INSTANCE_ID_PROPERTY;
import static org.easymock.EasyMock.anyInt;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
@@ -81,13 +84,17 @@ public class HadoopTimelineMetricsSinkTest {
}
@Test
- @PrepareForTest({URL.class, OutputStream.class, AbstractTimelineMetricsSink.class, HttpURLConnection.class})
+ @PrepareForTest({URL.class, OutputStream.class, AbstractTimelineMetricsSink.class, HttpURLConnection.class, TimelineMetric.class, HadoopTimelineMetricsSink.class})
public void testPutMetrics() throws Exception {
HadoopTimelineMetricsSink sink = new HadoopTimelineMetricsSink();
HttpURLConnection connection = PowerMock.createNiceMock(HttpURLConnection.class);
URL url = PowerMock.createNiceMock(URL.class);
InputStream is = IOUtils.toInputStream(gson.toJson(Collections.singletonList("localhost")));
+ TimelineMetric timelineMetric = PowerMock.createNiceMock(TimelineMetric.class);
+ expectNew(TimelineMetric.class).andReturn(timelineMetric).times(2);
+ expect(timelineMetric.getMetricValues()).andReturn(new TreeMap<Long, Double>()).anyTimes();
+ expect(timelineMetric.getMetricName()).andReturn("metricName").anyTimes();
expectNew(URL.class, anyString()).andReturn(url).anyTimes();
expect(url.openConnection()).andReturn(connection).anyTimes();
expect(connection.getInputStream()).andReturn(is).anyTimes();
@@ -106,6 +113,8 @@ public class HadoopTimelineMetricsSinkTest {
expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes();
expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(1000).anyTimes();
+ expect(conf.getBoolean(eq(SET_INSTANCE_ID_PROPERTY), eq(false))).andReturn(true).anyTimes();
+ expect(conf.getString(eq(INSTANCE_ID_PROPERTY))).andReturn("instanceId").anyTimes();
conf.setListDelimiter(eq(','));
expectLastCall().anyTimes();
@@ -145,6 +154,9 @@ public class HadoopTimelineMetricsSinkTest {
expect(record.metrics()).andReturn(Arrays.asList(metric)).anyTimes();
+ timelineMetric.setInstanceId(eq("instanceId"));
+ EasyMock.expectLastCall();
+
replay(conf, record, metric);
replayAll();
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
index 0052808..34a6787 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/application_metric_map.py
@@ -66,7 +66,7 @@ class ApplicationMetricMap:
del self.app_metric_map[ app_id ]
pass
- def flatten(self, application_id = None, clear_once_flattened = False):
+ def flatten(self, application_id = None, clear_once_flattened = False, set_instanceid = False, instanceid = None):
"""
Return flatten dict to caller in json format.
Json format:
@@ -89,11 +89,14 @@ class ApplicationMetricMap:
for appId, metrics in local_metric_map.iteritems():
for metricId, metricData in dict(metrics).iteritems():
# Create a timeline metric object
+ result_instanceid = ""
+ if set_instanceid:
+ result_instanceid = instanceid
timeline_metric = {
"hostname" : self.hostname,
"metricname" : metricId,
"appid" : "HOST",
- "instanceid" : "",
+ "instanceid" : result_instanceid,
"starttime" : self.get_start_time(appId, metricId),
"metrics" : metricData
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
index 5686c50..2670e76 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
@@ -239,9 +239,15 @@ class Configuration:
def is_server_https_enabled(self):
return "true" == str(self.get("collector", "https_enabled")).lower()
+ def is_set_instanceid(self):
+ return "true" == str(self.get("default", "set.instanceId", 'false')).lower()
+
def get_server_host(self):
return self.get("collector", "host")
+ def get_instanceid(self):
+ return self.get("default", "instanceid")
+
def get_server_port(self):
try:
return int(self.get("collector", "port"))
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
index ba3f18e..e2a7f0d 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
@@ -46,6 +46,8 @@ class Emitter(threading.Thread):
self.collector_port = config.get_server_port()
self.all_metrics_collector_hosts = config.get_metrics_collector_hosts()
self.is_server_https_enabled = config.is_server_https_enabled()
+ self.set_instanceid = config.is_set_instanceid()
+ self.instanceid = config.get_instanceid()
if self.is_server_https_enabled:
self.ca_certs = config.get_ca_certs()
@@ -74,7 +76,7 @@ class Emitter(threading.Thread):
# This call will acquire lock on the map and clear contents before returning
# After configured number of retries the data will not be sent to the
# collector
- json_data = self.application_metric_map.flatten(None, True)
+ json_data = self.application_metric_map.flatten(None, True, set_instanceid=self.set_instanceid, instanceid=self.instanceid)
if json_data is None:
logger.info("Nothing to emit, resume waiting.")
return
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
index 9d492cb..211e9cd 100644
--- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
@@ -70,6 +70,8 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
private static final String TIMELINE_METRICS_SSL_KEYSTORE_PATH_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SSL_KEYSTORE_PATH_PROPERTY;
private static final String TIMELINE_METRICS_SSL_KEYSTORE_TYPE_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SSL_KEYSTORE_TYPE_PROPERTY;
private static final String TIMELINE_METRICS_SSL_KEYSTORE_PASSWORD_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SSL_KEYSTORE_PASSWORD_PROPERTY;
+ private static final String TIMELINE_METRICS_KAFKA_INSTANCE_ID_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + INSTANCE_ID_PROPERTY;
+ private static final String TIMELINE_METRICS_KAFKA_SET_INSTANCE_ID_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SET_INSTANCE_ID_PROPERTY;
private static final String TIMELINE_DEFAULT_HOST = "localhost";
private static final String TIMELINE_DEFAULT_PORT = "6188";
private static final String TIMELINE_DEFAULT_PROTOCOL = "http";
@@ -87,6 +89,8 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
private TimelineMetricsCache metricsCache;
private int timeoutSeconds = 10;
private String zookeeperQuorum = null;
+ private boolean setInstanceId;
+ private String instanceId;
private String[] excludedMetricsPrefixes;
private String[] includedMetricsPrefixes;
@@ -162,6 +166,9 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
collectorHosts = parseHostsStringIntoCollection(props.getString(TIMELINE_HOSTS_PROPERTY, TIMELINE_DEFAULT_HOST));
metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL);
+ instanceId = props.getString(TIMELINE_METRICS_KAFKA_INSTANCE_ID_PROPERTY);
+ setInstanceId = props.getBoolean(TIMELINE_METRICS_KAFKA_SET_INSTANCE_ID_PROPERTY);
+
setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval));
if (metricCollectorProtocol.contains("https")) {
@@ -315,6 +322,9 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostname);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setAppId(component);
timelineMetric.setStartTime(currentTimeMillis);
timelineMetric.setType(ClassUtils.getShortCanonicalName(attributeValue, "Number"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
index e1ac48c..b05190c 100644
--- a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
+++ b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
@@ -84,6 +84,8 @@ public class KafkaTimelineMetricsReporterTest {
properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
+ properties.setProperty("kafka.timeline.metrics.instanceId", "cluster");
+ properties.setProperty("kafka.timeline.metrics.set.instanceId", "false");
props = new VerifiableProperties(properties);
}
@@ -120,6 +122,8 @@ public class KafkaTimelineMetricsReporterTest {
properties.setProperty("kafka.timeline.metrics.truststore.path", "");
properties.setProperty("kafka.timeline.metrics.truststore.type", "");
properties.setProperty("kafka.timeline.metrics.truststore.password", "");
+ properties.setProperty("kafka.timeline.metrics.instanceId", "cluster");
+ properties.setProperty("kafka.timeline.metrics.set.instanceId", "false");
kafkaTimelineMetricsReporter.init(new VerifiableProperties(properties));
kafkaTimelineMetricsReporter.stopReporter();
verifyAll();
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
index 9a55f10..08f0598 100644
--- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
@@ -50,6 +50,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
private Collection<String> collectorHosts;
private String zkQuorum;
private String protocol;
+ private boolean setInstanceId;
+ private String instanceId;
private NimbusClient nimbusClient;
private String applicationId;
private int timeoutSeconds;
@@ -126,6 +128,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
Integer.parseInt(cf.get(METRICS_POST_TIMEOUT_SECONDS).toString()) :
DEFAULT_POST_TIMEOUT_SECONDS;
applicationId = cf.get(APP_ID).toString();
+ setInstanceId = Boolean.getBoolean(cf.get(SET_INSTANCE_ID_PROPERTY).toString());
+ instanceId = cf.get(INSTANCE_ID_PROPERTY).toString();
collectorUri = constructTimelineMetricUri(protocol, findPreferredCollectHost(), port);
if (protocol.contains("https")) {
@@ -196,6 +200,9 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostname);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setAppId(component);
timelineMetric.setStartTime(currentTimeMillis);
timelineMetric.getMetricValues().put(currentTimeMillis, Double.parseDouble(attributeValue));
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
index 60c1427..20f60e1 100644
--- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
@@ -59,6 +59,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
private String port;
private String topologyName;
private String applicationId;
+ private boolean setInstanceId;
+ private String instanceId;
@Override
protected String getCollectorUri(String host) {
@@ -122,6 +124,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http");
port = configuration.getProperty(COLLECTOR_PORT, "6188");
+ instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY);
+ setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false"));
// Initialize the collector write strategy
super.init();
@@ -243,6 +247,9 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostName);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setAppId(applicationId);
timelineMetric.setStartTime(currentTimeMillis);
timelineMetric.setType(ClassUtils.getShortCanonicalName(
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
index 535fae0..14f160b 100644
--- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
@@ -46,6 +46,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
private Collection<String> collectorHosts;
private String zkQuorum;
private String protocol;
+ private boolean setInstanceId;
+ private String instanceId;
private String applicationId;
private int timeoutSeconds;
@@ -115,7 +117,8 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
Integer.parseInt(configuration.getProperty(METRICS_POST_TIMEOUT_SECONDS)) :
DEFAULT_POST_TIMEOUT_SECONDS;
applicationId = configuration.getProperty(CLUSTER_REPORTER_APP_ID, DEFAULT_CLUSTER_REPORTER_APP_ID);
-
+ setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY));
+ instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY);
if (protocol.contains("https")) {
String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim();
String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim();
@@ -226,6 +229,9 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostname);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setAppId(component);
timelineMetric.setStartTime(currentTimeMillis);
timelineMetric.setType(ClassUtils.getShortCanonicalName(attributeValue, "Number"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
index f58f549..425201c 100644
--- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
@@ -68,6 +68,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
private String port;
private String topologyName;
private String applicationId;
+ private String instanceId;
+ private boolean setInstanceId;
@Override
protected String getCollectorUri(String host) {
@@ -133,7 +135,8 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http");
port = configuration.getProperty(COLLECTOR_PORT, "6188");
-
+ instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY);
+ setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false"));
// Initialize the collector write strategy
super.init();
@@ -332,6 +335,9 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostName);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setAppId(applicationId);
timelineMetric.setStartTime(currentTimeMillis);
timelineMetric.setType(ClassUtils.getShortCanonicalName(
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
index f268149..8d1f63f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.metrics.system.impl;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -71,6 +72,8 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
private AmbariManagementController ambariManagementController;
private TimelineMetricsCache timelineMetricsCache;
private boolean isInitialized = false;
+ private boolean setInstanceId = false;
+ private String instanceId;
public AmbariMetricSinkImpl(AmbariManagementController amc) {
this.ambariManagementController = amc;
@@ -98,28 +101,30 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
for (Map.Entry<String, Cluster> kv : clusters.getClusters().entrySet()) {
String clusterName = kv.getKey();
+ instanceId = clusterName;
Cluster c = kv.getValue();
Resource.Type type = Resource.Type.ServiceConfigVersion;
//If Metrics Collector VIP settings are configured, use that.
- boolean vipHostConfigPresent = false;
- boolean vipPortConfigPresent = false;
+ boolean externalHostConfigPresent = false;
+ boolean externalPortConfigPresent = false;
Config clusterEnv = c.getDesiredConfigByType(ConfigHelper.CLUSTER_ENV);
if (clusterEnv != null) {
Map<String, String> configs = clusterEnv.getProperties();
- String metricsCollectorVipHost = configs.get("metrics_collector_vip_host");
- if (StringUtils.isNotEmpty(metricsCollectorVipHost)) {
- LOG.info("Setting Metrics Collector Vip Host : " + metricsCollectorVipHost);
- collectorHosts.add(metricsCollectorVipHost);
- vipHostConfigPresent = true;
+ String metricsCollectorExternalHosts = configs.get("metrics_collector_external_hosts");
+ if (StringUtils.isNotEmpty(metricsCollectorExternalHosts)) {
+ LOG.info("Setting Metrics Collector External Host : " + metricsCollectorExternalHosts);
+ collectorHosts.addAll(Arrays.asList(metricsCollectorExternalHosts.split(",")));
+ externalHostConfigPresent = true;
+ setInstanceId = true;
}
- String metricsCollectorVipPort = configs.get("metrics_collector_vip_port");
- if (StringUtils.isNotEmpty(metricsCollectorVipPort)) {
- LOG.info("Setting Metrics Collector Vip Port : " + metricsCollectorVipPort);
- port = metricsCollectorVipPort;
- vipPortConfigPresent = true;
+ String metricsCollectorExternalPort = configs.get("metrics_collector_external_port");
+ if (StringUtils.isNotEmpty(metricsCollectorExternalPort)) {
+ LOG.info("Setting Metrics Collector External Port : " + metricsCollectorExternalPort);
+ port = metricsCollectorExternalPort;
+ externalPortConfigPresent = true;
}
}
@@ -140,7 +145,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
ambariManagementController);
try {
- if ( !vipHostConfigPresent ) {
+ if ( !externalHostConfigPresent ) {
//get collector host(s)
Service service = c.getService(ambariMetricsServiceName);
if (service != null) {
@@ -166,7 +171,7 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
if (config != null && config.get("type").equals("ams-site")) {
TreeMap<Object, Object> properties = (TreeMap<Object, Object>) config.get("properties");
String timelineWebappAddress = (String) properties.get("timeline.metrics.service.webapp.address");
- if (!vipPortConfigPresent && StringUtils.isNotEmpty(timelineWebappAddress) && timelineWebappAddress.contains(":")) {
+ if (!externalPortConfigPresent && StringUtils.isNotEmpty(timelineWebappAddress) && timelineWebappAddress.contains(":")) {
port = timelineWebappAddress.split(":")[1];
}
String httpPolicy = (String) properties.get("timeline.metrics.service.http.policy");
@@ -318,6 +323,9 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements
TimelineMetric timelineMetric = new TimelineMetric();
timelineMetric.setMetricName(attributeName);
timelineMetric.setHostName(hostName);
+ if (setInstanceId) {
+ timelineMetric.setInstanceId(instanceId);
+ }
timelineMetric.setAppId(component);
timelineMetric.setStartTime(currentTimeMillis);
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
index 53cf002..150b0a8 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
@@ -122,12 +122,21 @@ info_num_logs = config['configurations']['accumulo-log4j']['info_num_logs']
# metrics2 properties
ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present
ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
+set_instanceId = "false"
+cluster_name = config["clusterName"]
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
index f7926d0..6873c85 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
@@ -40,6 +40,8 @@ accumulo.sink.timeline.period={{metrics_collection_period}}
accumulo.sink.timeline.sendInterval={{metrics_report_interval}}000
accumulo.sink.timeline.collector.hosts={{ams_collector_hosts}}
accumulo.sink.timeline.port={{metric_collector_port}}
+accumulo.sink.timeline.instanceId={{cluster_name}}
+accumulo.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
accumulo.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 919f26d..50dde1c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -66,17 +66,22 @@ embedded_mode_multiple_instances = False
if not is_ams_distributed and len(ams_collector_list) > 1:
embedded_mode_multiple_instances = True
+set_instanceId = "false"
+cluster_name = config["clusterName"]
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_vip_host']
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
random_metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
index 8c6f86f..9729bbe 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
@@ -65,6 +65,8 @@ hbase.sink.timeline.collector.hosts={{ams_collector_hosts}}
hbase.sink.timeline.port={{metric_collector_port}}
hbase.sink.timeline.protocol={{metric_collector_protocol}}
hbase.sink.timeline.serviceName-prefix=ams
+hbase.sink.timeline.instanceId={{cluster_name}}
+hbase.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
index 1505f9b..769ad67 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
@@ -25,6 +25,8 @@ enable_value_threshold = false
skip_disk_patterns = {{skip_disk_metrics_patterns}}
skip_virtual_interfaces = {{skip_virtual_interfaces}}
skip_network_interfaces_patterns = {{skip_network_interfaces_patterns}}
+instanceId={{cluster_name}}
+set.instanceId={{set_instanceId}}
[emitter]
send_interval = {{metrics_report_interval}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
index b143941..86a290f 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
@@ -90,18 +90,23 @@ hostname = None
if config.has_key('hostname'):
hostname = config['hostname']
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+set_instanceId = "false"
+cluster_name = config["clusterName"]
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
has_metric_collector = not len(ams_collector_hosts) == 0
metric_collector_port = None
if has_metric_collector:
+ metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
index cf2bd6c..b876a3d 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
@@ -24,6 +24,9 @@ collectionFrequency={{metrics_collection_period}}000
maxRowCacheSize=10000
sendInterval={{metrics_report_interval}}000
+instanceId={{cluster_name}}
+set.instanceId={{set_instanceId}}
+
# HTTPS properties
truststore.path = {{metric_truststore_path}}
truststore.type = {{metric_truststore_type}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index 3177643..efea167 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -152,12 +152,20 @@ has_ganglia_server = not len(ganglia_server_hosts) == 0
if has_ganglia_server:
ganglia_server_host = ganglia_server_hosts[0]
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+set_instanceId = "false"
+cluster_name = config["clusterName"]
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
index e808f5d..24535c5 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
@@ -76,6 +76,8 @@ hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
hbase.sink.timeline.collector.hosts={{ams_collector_hosts}}
hbase.sink.timeline.protocol={{metric_collector_protocol}}
hbase.sink.timeline.port={{metric_collector_port}}
+hbase.sink.timeline.instanceId={{cluster_name}}
+hbase.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
index 7e1abbc..9076269 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
@@ -74,7 +74,8 @@ hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
hbase.sink.timeline.collector.hosts={{ams_collector_hosts}}
hbase.sink.timeline.protocol={{metric_collector_protocol}}
hbase.sink.timeline.port={{metric_collector_port}}
-
+hbase.sink.timeline.instanceId={{cluster_name}}
+hbase.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml
index 4aadb83..fae61d3 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml
@@ -86,6 +86,8 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}
*.sink.timeline.protocol={{metric_collector_protocol}}
*.sink.timeline.port={{metric_collector_port}}
+*.sink.timeline.instanceId={{cluster_name}}
+*.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 7615ee7..0f03155 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -534,12 +534,21 @@ hive_site_config = dict(config['configurations']['hive-site'])
########################################################
############# AMS related params #####################
########################################################
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
+set_instanceId = "false"
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
@@ -830,4 +839,4 @@ if enable_ranger_hive:
ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-properties/ranger.service.config.param.enable.hive.metastore.lookup', False)
if security_enabled:
- hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
\ No newline at end of file
+ hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
index 82f71c5..9328f9f 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
@@ -42,6 +42,8 @@
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
*.sink.timeline.slave.host.name = {{hostname}}
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
index f1bfa03..9a7f9dc 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
@@ -42,6 +42,8 @@
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
*.sink.timeline.slave.host.name = {{hostname}}
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2
index d63dfb7..e9fe024 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2
@@ -41,6 +41,8 @@
*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
@@ -51,4 +53,4 @@
llapdaemon.sink.timeline.port={{metric_collector_port}}
llapdaemon.sink.timeline.protocol={{metric_collector_protocol}}
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
index 4c9c981..bd7eb0c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
@@ -41,6 +41,8 @@
*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
@@ -51,4 +53,4 @@
llaptaskscheduler.sink.timeline.port={{metric_collector_port}}
llaptaskscheduler.sink.timeline.protocol={{metric_collector_protocol}}
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
index 7615ee7..69ef08e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
@@ -534,12 +534,20 @@ hive_site_config = dict(config['configurations']['hive-site'])
########################################################
############# AMS related params #####################
########################################################
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+set_instanceId = "false"
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
@@ -830,4 +838,4 @@ if enable_ranger_hive:
ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-properties/ranger.service.config.param.enable.hive.metastore.lookup', False)
if security_enabled:
- hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
\ No newline at end of file
+ hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
index 82f71c5..9328f9f 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2
@@ -42,6 +42,8 @@
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
*.sink.timeline.slave.host.name = {{hostname}}
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
index f1bfa03..9a7f9dc 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2
@@ -42,6 +42,8 @@
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
*.sink.timeline.slave.host.name = {{hostname}}
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2
index d63dfb7..e9fe024 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llapdaemon.j2
@@ -41,6 +41,8 @@
*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
@@ -51,4 +53,4 @@
llapdaemon.sink.timeline.port={{metric_collector_port}}
llapdaemon.sink.timeline.protocol={{metric_collector_protocol}}
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
index 4c9c981..bd7eb0c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2
@@ -41,6 +41,8 @@
*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
*.sink.timeline.period={{metrics_collection_period}}
*.sink.timeline.sendInterval={{metrics_report_interval}}000
+ *.sink.timeline.instanceId={{cluster_name}}
+ *.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
@@ -51,4 +53,4 @@
llaptaskscheduler.sink.timeline.port={{metric_collector_port}}
llaptaskscheduler.sink.timeline.protocol={{metric_collector_protocol}}
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
index f2b4a76..e01dacd 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml
@@ -407,6 +407,16 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>kafka.timeline.metrics.instanceId</name>
+ <value>{{cluster_name}}</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kafka.timeline.metrics.set.instanceId</name>
+ <value>{{set_instanceId}}</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>kafka.timeline.metrics.maxRowCacheSize</name>
<value>10000</value>
<description>Timeline metrics reporter send interval</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
index 32f18f2..5b0be54 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
@@ -130,13 +130,22 @@ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.trust
metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+set_instanceId = "false"
+cluster_name = config["clusterName"]
+
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
has_metric_collector = not len(ams_collector_hosts) == 0
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index 44b256e..d9fae76 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -174,13 +174,19 @@ if stack_supports_storm_kerberos:
else:
storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport']
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+set_instanceId = "false"
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_metric_collector = not len(ams_collector_hosts) == 0
metric_collector_port = None
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2
index f2a75cf..51162e8 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2
@@ -67,4 +67,7 @@ metrics_collector:
truststore.type : "{{metric_truststore_type}}"
truststore.password : "{{metric_truststore_password}}"
+ instanceId={{cluster_name}}
+ set.instanceId={{set_instanceId}}
+
{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2
index 001ff73..0501039 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2
@@ -23,6 +23,8 @@ zookeeper.quorum={{zookeeper_quorum}}
maxRowCacheSize=10000
sendInterval={{metrics_report_interval}}000
clusterReporterAppId=nimbus
+instanceId={{cluster_name}}
+set.instanceId={{set_instanceId}}
# HTTPS properties
truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
index 70ebfeb..7282bb5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
@@ -111,7 +111,14 @@ jtnode_host = default("/clusterHostInfo/jtnode_host", [])
namenode_host = default("/clusterHostInfo/namenode_host", [])
zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+cluster_name = config["clusterName"]
+set_instanceId = "false"
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_namenode = not len(namenode_host) == 0
has_resourcemanager = not len(rm_host) == 0
@@ -137,8 +144,8 @@ if has_ganglia_server:
metric_collector_port = None
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
index 2f3aab6..1b02a97 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -75,6 +75,8 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}
*.sink.timeline.protocol={{metric_collector_protocol}}
*.sink.timeline.port={{metric_collector_port}}
+*.sink.timeline.instanceId={{cluster_name}}
+*.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml
index 4aadb83..fae61d3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-metrics2.properties.xml
@@ -86,6 +86,8 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}
*.sink.timeline.protocol={{metric_collector_protocol}}
*.sink.timeline.port={{metric_collector_port}}
+*.sink.timeline.instanceId={{cluster_name}}
+*.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
index 5a5361c..678bbde 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
@@ -105,7 +105,15 @@ jtnode_host = default("/clusterHostInfo/jtnode_host", [])
namenode_host = default("/clusterHostInfo/namenode_host", [])
zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
-ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
+cluster_name = config["clusterName"]
+set_instanceId = "false"
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_external_hosts' in config['configurations']['cluster-env']:
+ ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts']
+ set_instanceId = "true"
+else:
+ ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
has_namenode = not len(namenode_host) == 0
has_resourcemanager = not len(rm_host) == 0
@@ -131,8 +139,8 @@ if has_ganglia_server:
metric_collector_port = None
if has_metric_collector:
if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ 'metrics_collector_external_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port']
else:
metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
if metric_collector_web_address.find(':') != -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
index 2f3aab6..1b02a97 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -75,6 +75,8 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}
*.sink.timeline.protocol={{metric_collector_protocol}}
*.sink.timeline.port={{metric_collector_port}}
+*.sink.timeline.instanceId={{cluster_name}}
+*.sink.timeline.set.instanceId={{set_instanceId}}
# HTTPS properties
*.sink.timeline.truststore.path = {{metric_truststore_path}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/362d6eca/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 94425e0..d267bc1 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -627,8 +627,8 @@
"user_group": "hadoop",
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
- "metrics_collector_vip_host": "c6402.ambari.apache.org",
- "metrics_collector_vip_port": "6189",
+ "metrics_collector_external_hosts": "c6402.ambari.apache.org",
+ "metrics_collector_external_port": "6189",
"override_uid" : "true",
"fetch_nonlocal_groups": "true",
"manage_dirs_on_root": "true",