You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2016/02/16 16:58:52 UTC
[4/4] ambari git commit: AMBARI-15050 Https Support for Metrics
System (dsen)
AMBARI-15050 Https Support for Metrics System (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7e75e52a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7e75e52a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7e75e52a
Branch: refs/heads/trunk
Commit: 7e75e52a91568d7ed0dd2bdaafbd60b37a3e7dc8
Parents: 0868a0f
Author: Dmytro Sen <ds...@apache.org>
Authored: Tue Feb 16 17:53:11 2016 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Tue Feb 16 17:53:11 2016 +0200
----------------------------------------------------------------------
.../timeline/AbstractTimelineMetricsSink.java | 92 +-
.../src/main/conf/flume-metrics2.properties.j2 | 3 +-
.../sink/flume/FlumeTimelineMetricsSink.java | 11 +-
.../conf/hadoop-metrics2-hbase.properties.j2 | 8 +-
.../src/main/conf/hadoop-metrics2.properties.j2 | 22 +-
.../timeline/HadoopTimelineMetricsSink.java | 13 +-
.../timeline/HadoopTimelineMetricsSinkTest.java | 6 +-
.../conf/unix/metric_monitor.ini | 1 +
.../src/main/python/core/config_reader.py | 6 +-
.../src/main/python/core/emitter.py | 18 +-
.../kafka/KafkaTimelineMetricsReporter.java | 17 +-
.../kafka/KafkaTimelineMetricsReporterTest.java | 2 +-
.../storm/StormTimelineMetricsReporter.java | 24 +-
.../sink/storm/StormTimelineMetricsSink.java | 8 +-
.../ApplicationHistoryServer.java | 11 +-
.../loadsimulator/net/RestMetricsSender.java | 6 +-
.../timeline/TimelineMetricConfiguration.java | 6 +-
.../ComponentSSLConfiguration.java | 14 +-
.../server/configuration/Configuration.java | 6 +-
.../ganglia/GangliaPropertyProvider.java | 2 +-
.../ganglia/GangliaReportPropertyProvider.java | 2 +-
.../metrics/timeline/AMSPropertyProvider.java | 8 +-
.../timeline/AMSReportPropertyProvider.java | 2 +-
.../ACCUMULO/1.6.1.2.2.0/metainfo.xml | 1 +
.../1.6.1.2.2.0/package/scripts/params.py | 7 +
.../hadoop-metrics2-accumulo.properties.j2 | 7 +-
.../0.1.0/configuration/ams-site.xml | 12 +-
.../0.1.0/configuration/ams-ssl-client.xml | 37 +
.../0.1.0/configuration/ams-ssl-server.xml | 64 +
.../AMBARI_METRICS/0.1.0/metainfo.xml | 2 +
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 8 +
.../package/scripts/metrics_grafana_util.py | 2 +-
.../0.1.0/package/scripts/params.py | 10 +
.../0.1.0/package/scripts/service_check.py | 17 +-
.../hadoop-metrics2-hbase.properties.j2 | 7 +-
.../package/templates/metric_monitor.ini.j2 | 1 +
.../FLUME/1.4.0.2.0/metainfo.xml | 1 +
.../FLUME/1.4.0.2.0/package/scripts/params.py | 7 +
.../templates/flume-metrics2.properties.j2 | 8 +-
.../common-services/HAWQ/2.0.0/metainfo.xml | 1 +
.../HBASE/0.96.0.2.0/metainfo.xml | 1 +
.../0.96.0.2.0/package/scripts/params_linux.py | 8 +
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 7 +-
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 7 +-
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 1 +
.../0.8.1.2.2/configuration/kafka-broker.xml | 21 +
.../KAFKA/0.8.1.2.2/metainfo.xml | 1 +
.../KAFKA/0.8.1.2.2/package/scripts/kafka.py | 4 +
.../KAFKA/0.8.1.2.2/package/scripts/params.py | 10 +-
.../STORM/0.9.1.2.1/metainfo.xml | 1 +
.../0.9.1.2.1/package/scripts/params_linux.py | 11 +-
.../0.9.1.2.1/package/templates/config.yaml.j2 | 8 +-
.../templates/storm-metrics2.properties.j2 | 10 +-
.../common-services/YARN/2.1.0.2.0/metainfo.xml | 2 +
.../2.0.6/hooks/before-START/scripts/params.py | 8 +
.../templates/hadoop-metrics2.properties.j2 | 25 +-
.../ComponentSSLConfigurationTest.java | 6 +-
.../ganglia/GangliaPropertyProviderTest.java | 26 +-
.../GangliaReportPropertyProviderTest.java | 2 +-
.../timeline/AMSPropertyProviderTest.java | 30 +-
.../timeline/AMSReportPropertyProviderTest.java | 4 +-
.../AMBARI_METRICS/test_metrics_collector.py | 8 +
.../python/stacks/2.0.6/configs/default.json | 1146 +++++++++---------
.../2.0.6/configs/default_ams_embedded.json | 7 +
64 files changed, 1113 insertions(+), 721 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 9173889..2854898 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -24,21 +24,35 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.security.KeyStore;
public abstract class AbstractTimelineMetricsSink {
public static final String TAGS_FOR_PREFIX_PROPERTY_PREFIX = "tagsForPrefix.";
public static final String MAX_METRIC_ROW_CACHE_SIZE = "maxRowCacheSize";
public static final String METRICS_SEND_INTERVAL = "sendInterval";
public static final String METRICS_POST_TIMEOUT_SECONDS = "timeout";
- public static final String COLLECTOR_HOST_PROPERTY = "collector";
- public static final String COLLECTOR_PORT_PROPERTY = "port";
+ public static final String COLLECTOR_PROPERTY = "collector";
public static final int DEFAULT_POST_TIMEOUT_SECONDS = 10;
public static final String SKIP_COUNTER_TRANSFROMATION = "skipCounterDerivative";
+ public static final String WS_V1_TIMELINE_METRICS = "/ws/v1/timeline/metrics";
+
+ public static final String SSL_KEYSTORE_PATH_PROPERTY = "truststore.path";
+ public static final String SSL_KEYSTORE_TYPE_PROPERTY = "truststore.type";
+ public static final String SSL_KEYSTORE_PASSWORD_PROPERTY = "truststore.password";
+
+ private SSLSocketFactory sslSocketFactory;
+
protected final Log LOG;
protected static ObjectMapper mapper;
@@ -48,7 +62,7 @@ public abstract class AbstractTimelineMetricsSink {
AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
mapper.setAnnotationIntrospector(introspector);
mapper.getSerializationConfig()
- .setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
+ .withSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
}
public AbstractTimelineMetricsSink() {
@@ -59,9 +73,13 @@ public abstract class AbstractTimelineMetricsSink {
String connectUrl = getCollectorUri();
int timeout = getTimeoutSeconds() * 1000;
try {
+ if (connectUrl == null) {
+ throw new IOException("Unknown URL. " +
+ "Unable to connect to metrics collector.");
+ }
String jsonData = mapper.writeValueAsString(metrics);
-
- HttpURLConnection connection = (HttpURLConnection) new URL(connectUrl).openConnection();
+ HttpURLConnection connection = connectUrl.startsWith("https") ?
+ getSSLConnection(connectUrl) : getConnection(connectUrl);
connection.setRequestMethod("POST");
connection.setRequestProperty("Content-Type", "application/json");
@@ -81,13 +99,75 @@ public abstract class AbstractTimelineMetricsSink {
LOG.info("Unable to POST metrics to collector, " + connectUrl + ", " +
"statusCode = " + statusCode);
} else {
- LOG.debug("Metrics posted to Collector " + connectUrl);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Metrics posted to Collector " + connectUrl);
+ }
}
} catch (IOException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to connect to collector, " + connectUrl, e);
+ } else {
+ LOG.info("Unable to connect to collector, " + connectUrl);
+ }
throw new UnableToConnectException(e).setConnectUrl(connectUrl);
}
}
+ // Get a connection
+ protected HttpURLConnection getConnection(String spec) throws IOException {
+ return (HttpURLConnection) new URL(spec).openConnection();
+ }
+
+ // Get an ssl connection
+ protected HttpsURLConnection getSSLConnection(String spec)
+ throws IOException, IllegalStateException {
+
+ HttpsURLConnection connection = (HttpsURLConnection) (new URL(spec)
+ .openConnection());
+
+ connection.setSSLSocketFactory(sslSocketFactory);
+
+ return connection;
+ }
+
+ protected void loadTruststore(String trustStorePath, String trustStoreType,
+ String trustStorePassword) {
+ if (sslSocketFactory == null) {
+ if (trustStorePath == null || trustStorePassword == null) {
+
+ String msg =
+ String.format("Can't load TrustStore. " +
+ "Truststore path or password is not set.");
+
+ LOG.error(msg);
+ throw new IllegalStateException(msg);
+ }
+ FileInputStream in = null;
+ try {
+ in = new FileInputStream(new File(trustStorePath));
+ KeyStore store = KeyStore.getInstance(trustStoreType == null ?
+ KeyStore.getDefaultType() : trustStoreType);
+ store.load(in, trustStorePassword.toCharArray());
+ TrustManagerFactory tmf = TrustManagerFactory
+ .getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(store);
+ SSLContext context = SSLContext.getInstance("TLS");
+ context.init(null, tmf.getTrustManagers(), null);
+ sslSocketFactory = context.getSocketFactory();
+ } catch (Exception e) {
+ LOG.error("Unable to load TrustStore", e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.error("Unable to load TrustStore", e);
+ }
+ }
+ }
+ }
+ }
+
abstract protected String getCollectorUri();
abstract protected int getTimeoutSeconds();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-flume-sink/src/main/conf/flume-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/main/conf/flume-metrics2.properties.j2 b/ambari-metrics/ambari-metrics-flume-sink/src/main/conf/flume-metrics2.properties.j2
index bc18043..f9b303e 100644
--- a/ambari-metrics/ambari-metrics-flume-sink/src/main/conf/flume-metrics2.properties.j2
+++ b/ambari-metrics/ambari-metrics-flume-sink/src/main/conf/flume-metrics2.properties.j2
@@ -16,8 +16,7 @@
# limitations under the License.
#}
-collector={{metric_collector_host}}
-port={{metric_collector_port}}
+collector=http://localhost:6188
collectionFrequency=60000
maxRowCacheSize=10000
sendInterval=59000
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
index cf2b4ae..3040c48 100644
--- a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.metrics2.sink.flume;
-import org.apache.commons.lang.ClassUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.flume.Context;
import org.apache.flume.FlumeException;
@@ -96,9 +95,13 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem
metricsSendInterval = Integer.parseInt(configuration.getProperty(METRICS_SEND_INTERVAL,
String.valueOf(TimelineMetricsCache.MAX_EVICTION_TIME_MILLIS)));
metricsCaches = new HashMap<String, TimelineMetricsCache>();
- String collectorHostname = configuration.getProperty(COLLECTOR_HOST_PROPERTY);
- String port = configuration.getProperty(COLLECTOR_PORT_PROPERTY);
- collectorUri = "http://" + collectorHostname + ":" + port + "/ws/v1/timeline/metrics";
+ collectorUri = configuration.getProperty(COLLECTOR_PROPERTY) + WS_V1_TIMELINE_METRICS;
+ if (collectorUri.toLowerCase().startsWith("https://")) {
+ String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim();
+ String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim();
+ String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim();
+ loadTruststore(trustStorePath, trustStoreType, trustStorePwd);
+ }
pollFrequency = Long.parseLong(configuration.getProperty("collectionFrequency"));
String[] metrics = configuration.getProperty(COUNTER_METRICS_PROPERTY).trim().split(",");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2 b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2
index 0d13498..c819301 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2-hbase.properties.j2
@@ -31,19 +31,19 @@ hbase.extendedperiod = 3600
# Configuration of the "hbase" context for timeline metrics service
hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.period=10
-hbase.collector={{timeline_server_hosts}}:8188
+hbase.collector={{timeline_server_hosts}}:6188
# Configuration of the "jvm" context for timeline metrics service
jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
jvm.period=10
-jvm.collector={{timeline_server_hosts}}:8188
+jvm.collector={{timeline_server_hosts}}:6188
# Configuration of the "rpc" context for timeline metrics service
rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
rpc.period=10
-rpc.collector={{timeline_server_hosts}}:8188
+rpc.collector={{timeline_server_hosts}}:6188
# Following hadoop example
hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period=10
-hbase.sink.timeline.collector={{timeline_server_hosts}}:8188
\ No newline at end of file
+hbase.sink.timeline.collector=http://{{timeline_server_hosts}}:6188
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2 b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2
index 76a00d1..ec1377d 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/conf/hadoop-metrics2.properties.j2
@@ -42,17 +42,17 @@
# Hook up to the server
-datanode.sink.timeline.collector={{timeline_server_hosts}}:8188
-namenode.sink.timeline.collector={{timeline_server_hosts}}:8188
-resourcemanager.sink.timeline.collector={{timeline_server_hosts}}:8188
-nodemanager.sink.timeline.collector={{timeline_server_hosts}}:8188
-historyserver.sink.timeline.collector={{timeline_server_hosts}}:8188
-journalnode.sink.timeline.collector={{timeline_server_hosts}}:8188
-nimbus.sink.timeline.collector={{timeline_server_hosts}}:8188
-supervisor.sink.timeline.collector={{timeline_server_hosts}}:8188
-maptask.sink.timeline.collector={{timeline_server_hosts}}:8188
-reducetask.sink.timeline.collector={{timeline_server_hosts}}:8188
+datanode.sink.timeline.collector=http://localhost:6188
+namenode.sink.timeline.collector=http://localhost:6188
+resourcemanager.sink.timeline.collector=http://localhost:6188
+nodemanager.sink.timeline.collector=http://localhost:6188
+historyserver.sink.timeline.collector=http://localhost:6188
+journalnode.sink.timeline.collector=http://localhost:6188
+nimbus.sink.timeline.collector=http://localhost:6188
+supervisor.sink.timeline.collector=http://localhost:6188
+maptask.sink.timeline.collector=http://localhost:6188
+reducetask.sink.timeline.collector=http://localhost:6188
resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
-{% endif %}
\ No newline at end of file
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
index 000b82e..6da9257 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.configuration.SubsetConfiguration;
-import org.apache.commons.lang.ClassUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -79,13 +78,19 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple
LOG.info("Identified hostname = " + hostName + ", serviceName = " + serviceName);
// Load collector configs
- metricsServers = Servers.parse(conf.getString(COLLECTOR_HOST_PROPERTY), 6188);
+ metricsServers = Servers.parse(conf.getString(COLLECTOR_PROPERTY), 6188);
if (metricsServers == null || metricsServers.isEmpty()) {
LOG.error("No Metric collector configured.");
} else {
- collectorUri = "http://" + conf.getString(COLLECTOR_HOST_PROPERTY).trim()
- + "/ws/v1/timeline/metrics";
+ collectorUri = conf.getString(COLLECTOR_PROPERTY).trim()
+ + WS_V1_TIMELINE_METRICS;
+ if (collectorUri.toLowerCase().startsWith("https://")) {
+ String trustStorePath = conf.getString(SSL_KEYSTORE_PATH_PROPERTY).trim();
+ String trustStoreType = conf.getString(SSL_KEYSTORE_TYPE_PROPERTY).trim();
+ String trustStorePwd = conf.getString(SSL_KEYSTORE_PASSWORD_PROPERTY).trim();
+ loadTruststore(trustStorePath, trustStoreType, trustStorePwd);
+ }
}
LOG.info("Collector Uri: " + collectorUri);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
index 6b23f36..528384e 100644
--- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
+++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.metrics2.sink.timeline;
-import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_HOST_PROPERTY;
+import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PROPERTY;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.MAX_METRIC_ROW_CACHE_SIZE;
import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.METRICS_SEND_INTERVAL;
import static org.easymock.EasyMock.anyInt;
@@ -62,7 +62,7 @@ public class HadoopTimelineMetricsSinkTest {
expect(conf.getString(eq("slave.host.name"))).andReturn("testhost").anyTimes();
expect(conf.getParent()).andReturn(null).anyTimes();
expect(conf.getPrefix()).andReturn("service").anyTimes();
- expect(conf.getString(eq(COLLECTOR_HOST_PROPERTY))).andReturn("localhost:63188").anyTimes();
+ expect(conf.getString(eq(COLLECTOR_PROPERTY))).andReturn("localhost:63188").anyTimes();
expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes();
expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes();
@@ -130,7 +130,7 @@ public class HadoopTimelineMetricsSinkTest {
expect(conf.getString(eq("slave.host.name"))).andReturn("testhost").anyTimes();
expect(conf.getParent()).andReturn(null).anyTimes();
expect(conf.getPrefix()).andReturn("service").anyTimes();
- expect(conf.getString(eq(COLLECTOR_HOST_PROPERTY))).andReturn("localhost:63188").anyTimes();
+ expect(conf.getString(eq(COLLECTOR_PROPERTY))).andReturn("localhost:63188").anyTimes();
expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes();
expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
index d302518..5952982 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
+++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
@@ -22,6 +22,7 @@ metrics_server = localhost:{{ams_collector_port}}
hostname = {{hostname}}
enable_time_threshold = false
enable_value_threshold = false
+https_enabled = false
[emitter]
send_interval = 60
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
index 48f5af2..d533537 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
@@ -91,6 +91,7 @@ debug_level = INFO
metrics_server = host:port
enable_time_threshold = false
enable_value_threshold = false
+https_enabled = false
[emitter]
send_interval = 60
@@ -181,7 +182,7 @@ class Configuration:
def get(self, section, key, default=None):
try:
- value = self.config.get(section, key)
+ value = str(self.config.get(section, key)).strip()
except:
return default
return value
@@ -209,3 +210,6 @@ class Configuration:
def get_max_queue_size(self):
return int(self.get("collector", "max_queue_size", 5000))
+
+ def get_server_https_enabled(self):
+ return "true" == str(self.get("default", "https_enabled")).lower()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
index 6f66093..4e39ab5 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py
@@ -20,13 +20,12 @@ limitations under the License.
import logging
import threading
-import time
import urllib2
logger = logging.getLogger()
class Emitter(threading.Thread):
- COLLECTOR_URL = "http://{0}/ws/v1/timeline/metrics"
+ COLLECTOR_URL = "{0}://{1}/ws/v1/timeline/metrics"
RETRY_SLEEP_INTERVAL = 5
MAX_RETRY_COUNT = 3
"""
@@ -36,10 +35,12 @@ class Emitter(threading.Thread):
threading.Thread.__init__(self)
logger.debug('Initializing Emitter thread.')
self.lock = threading.Lock()
- self.collector_address = config.get_server_address()
self.send_interval = config.get_send_interval()
self._stop_handler = stop_handler
self.application_metric_map = application_metric_map
+ # TODO verify certificate
+ protocol = 'https' if config.get_server_https_enabled() else 'http'
+ self.collector_url = self.COLLECTOR_URL.format(protocol, config.get_server_address())
def run(self):
logger.info('Running Emitter thread: %s' % threading.currentThread().getName())
@@ -54,7 +55,7 @@ class Emitter(threading.Thread):
logger.info('Shutting down Emitter thread')
return
pass
-
+
def submit_metrics(self):
retry_count = 0
# This call will acquire lock on the map and clear contents before returning
@@ -73,7 +74,7 @@ class Emitter(threading.Thread):
except Exception, e:
logger.warn('Error sending metrics to server. %s' % str(e))
pass
-
+
if response and response.getcode() == 200:
retry_count = self.MAX_RETRY_COUNT
else:
@@ -84,13 +85,12 @@ class Emitter(threading.Thread):
return
pass
pass
-
+ # TODO verify certificate
def push_metrics(self, data):
headers = {"Content-Type" : "application/json", "Accept" : "*/*"}
- server = self.COLLECTOR_URL.format(self.collector_address.strip())
- logger.info("server: %s" % server)
+ logger.info("server: %s" % self.collector_url)
logger.debug("message to sent: %s" % data)
- req = urllib2.Request(server, data, headers)
+ req = urllib2.Request(self.collector_url, data, headers)
response = urllib2.urlopen(req, timeout=int(self.send_interval - 10))
if response:
logger.debug("POST response from server: retcode = {0}".format(response.getcode()))
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
index 4915435..d6d251c 100644
--- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java
@@ -63,11 +63,13 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
private static final String TIMELINE_METRICS_MAX_ROW_CACHE_SIZE_PROPERTY = "kafka.timeline.metrics.maxRowCacheSize";
private static final String TIMELINE_HOST_PROPERTY = "kafka.timeline.metrics.host";
private static final String TIMELINE_PORT_PROPERTY = "kafka.timeline.metrics.port";
+ private static final String TIMELINE_PROTOCOL_PROPERTY = "kafka.timeline.metrics.protocol";
private static final String TIMELINE_REPORTER_ENABLED_PROPERTY = "kafka.timeline.metrics.reporter.enabled";
private static final String EXCLUDED_METRICS_PROPERTY = "external.kafka.metrics.exclude.prefix";
private static final String INCLUDED_METRICS_PROPERTY = "external.kafka.metrics.include.prefix";
private static final String TIMELINE_DEFAULT_HOST = "localhost";
- private static final String TIMELINE_DEFAULT_PORT = "8188";
+ private static final String TIMELINE_DEFAULT_PORT = "6188";
+ private static final String TIMELINE_DEFAULT_PROTOCOL = "http";
private boolean initialized = false;
private boolean running = false;
@@ -117,8 +119,19 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink
int maxRowCacheSize = props.getInt(TIMELINE_METRICS_MAX_ROW_CACHE_SIZE_PROPERTY, MAX_RECS_PER_NAME_DEFAULT);
String metricCollectorHost = props.getString(TIMELINE_HOST_PROPERTY, TIMELINE_DEFAULT_HOST);
String metricCollectorPort = props.getString(TIMELINE_PORT_PROPERTY, TIMELINE_DEFAULT_PORT);
+ String metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL);
setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval));
- collectorUri = "http://" + metricCollectorHost + ":" + metricCollectorPort + "/ws/v1/timeline/metrics";
+
+ collectorUri = metricCollectorProtocol + "://" + metricCollectorHost +
+ ":" + metricCollectorPort + WS_V1_TIMELINE_METRICS;
+
+ if (collectorUri.toLowerCase().startsWith("https://")) {
+ String trustStorePath = props.getString(SSL_KEYSTORE_PATH_PROPERTY).trim();
+ String trustStoreType = props.getString(SSL_KEYSTORE_TYPE_PROPERTY).trim();
+ String trustStorePwd = props.getString(SSL_KEYSTORE_PASSWORD_PROPERTY).trim();
+ loadTruststore(trustStorePath, trustStoreType, trustStorePwd);
+ }
+
// Exclusion policy
String excludedMetricsStr = props.getString(EXCLUDED_METRICS_PROPERTY, "");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
index 6f2fc27..e0adb4b 100644
--- a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
+++ b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java
@@ -79,7 +79,7 @@ public class KafkaTimelineMetricsReporterTest {
properties.setProperty("kafka.timeline.metrics.sendInterval", "5900");
properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000");
properties.setProperty("kafka.timeline.metrics.host", "localhost");
- properties.setProperty("kafka.timeline.metrics.port", "8188");
+ properties.setProperty("kafka.timeline.metrics.port", "6188");
properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
index f054f16..ab5f1e4 100644
--- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
+++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java
@@ -24,7 +24,6 @@ import backtype.storm.generated.TopologySummary;
import backtype.storm.metric.IClusterReporter;
import backtype.storm.utils.NimbusClient;
import backtype.storm.utils.Utils;
-import org.apache.commons.lang3.ClassUtils;
import org.apache.commons.lang3.Validate;
import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
@@ -40,9 +39,7 @@ import java.util.Map;
public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
implements IClusterReporter {
- public static final String COLLECTOR_HOST = "host";
- public static final String COLLECTOR_PORT = "port";
- public static final String METRICS_COLLECTOR = "metrics_collector";
+ public static final String METRICS_COLLECTOR_CATEGORY = "metrics_collector";
public static final String APP_ID = "appId";
private String hostname;
@@ -79,20 +76,25 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink
LOG.error("Could not identify hostname.");
throw new RuntimeException("Could not identify hostname.", e);
}
- Validate.notNull(conf.get(METRICS_COLLECTOR), METRICS_COLLECTOR + " can not be null");
- Map cf = (Map) conf.get(METRICS_COLLECTOR);
+ Validate.notNull(conf.get(METRICS_COLLECTOR_CATEGORY), METRICS_COLLECTOR_CATEGORY + " can not be null");
+ Map cf = (Map) conf.get(METRICS_COLLECTOR_CATEGORY);
Map stormConf = Utils.readStormConfig();
this.nimbusClient = NimbusClient.getConfiguredClient(stormConf);
- String collectorHostname = cf.get(COLLECTOR_HOST).toString();
- String port = cf.get(COLLECTOR_PORT).toString();
+ String collector = cf.get(COLLECTOR_PROPERTY).toString();
timeoutSeconds = cf.get(METRICS_POST_TIMEOUT_SECONDS) != null ?
Integer.parseInt(cf.get(METRICS_POST_TIMEOUT_SECONDS).toString()) :
DEFAULT_POST_TIMEOUT_SECONDS;
applicationId = cf.get(APP_ID).toString();
- collectorUri = "http://" + collectorHostname + ":" + port + "/ws/v1/timeline/metrics";
+ collectorUri = collector + WS_V1_TIMELINE_METRICS;
+ if (collectorUri.toLowerCase().startsWith("https://")) {
+ String trustStorePath = cf.get(SSL_KEYSTORE_PATH_PROPERTY).toString().trim();
+ String trustStoreType = cf.get(SSL_KEYSTORE_TYPE_PROPERTY).toString().trim();
+ String trustStorePwd = cf.get(SSL_KEYSTORE_PASSWORD_PROPERTY).toString().trim();
+ loadTruststore(trustStorePath, trustStoreType, trustStorePwd);
+ }
} catch (Exception e) {
- LOG.warn("Could not initialize metrics collector, please specify host, " +
- "port under $STORM_HOME/conf/config.yaml ", e);
+ LOG.warn("Could not initialize metrics collector, please specify " +
+ "protocol, host, port under $STORM_HOME/conf/config.yaml ", e);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
index 4208287..91c63b9 100644
--- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java
@@ -77,7 +77,13 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem
int metricsSendInterval = Integer.parseInt(configuration.getProperty(METRICS_SEND_INTERVAL,
String.valueOf(MAX_EVICTION_TIME_MILLIS)));
metricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval);
- collectorUri = "http://" + configuration.getProperty(COLLECTOR_HOST_PROPERTY) + ":" + configuration.getProperty(COLLECTOR_PORT_PROPERTY) + "/ws/v1/timeline/metrics";
+ collectorUri = configuration.getProperty(COLLECTOR_PROPERTY) + WS_V1_TIMELINE_METRICS;
+ if (collectorUri.toLowerCase().startsWith("https://")) {
+ String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim();
+ String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim();
+ String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim();
+ loadTruststore(trustStorePath, trustStoreType, trustStorePwd);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 11c162a..62a8cc3 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.service.CompositeService;
@@ -170,15 +171,15 @@ public class ApplicationHistoryServer extends CompositeService {
String bindAddress = metricConfiguration.getWebappAddress();
LOG.info("Instantiating AHSWebApp at " + bindAddress);
try {
+ Configuration conf = metricConfiguration.getMetricsConf();
+ HttpConfig.Policy policy = HttpConfig.Policy.valueOf(
+ conf.get(TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY,
+ HttpConfig.Policy.HTTP_ONLY.name()));
webApp =
WebApps
.$for("applicationhistory", ApplicationHistoryClientService.class,
ahsClientService, "ws")
- .with(getConfig())
- .withHttpSpnegoPrincipalKey(
- YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL)
- .withHttpSpnegoKeytabKey(
- YarnConfiguration.TIMELINE_SERVICE_KEYTAB)
+ .withHttpPolicy(conf, policy)
.at(bindAddress)
.start(new AHSWebApp(timelineStore, timelineMetricStore,
ahsClientService));
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
index c6d84c2..0a9a513 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java
@@ -37,7 +37,7 @@ public class RestMetricsSender implements MetricsSender {
/**
* Creates unconnected RestMetricsSender with endpoint configured as
- * http://${metricsHost}:8188/ws/v1/timeline/metrics,
+ * http://${metricsHost}:6188/ws/v1/timeline/metrics,
* where ${metricsHost} is specified by metricHost param.
*
* @param metricsHost the hostname that will be used to access application metrics history service.
@@ -70,10 +70,6 @@ public class RestMetricsSender implements MetricsSender {
if (responseString.length() > 0) {
LOG.debug("POST response from server: " + responseString);
}
- } catch (MalformedURLException e) {
- LOG.error("", e);
- } catch (ProtocolException e) {
- LOG.error("", e);
} catch (IOException e) {
LOG.error("", e);
} finally {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index cc9a2f3..3431ac8 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -205,6 +204,9 @@ public class TimelineMetricConfiguration {
public static final String AGGREGATORS_SKIP_BLOCK_CACHE =
"timeline.metrics.aggregators.skip.blockcache.enabled";
+ public static final String TIMELINE_SERVICE_HTTP_POLICY =
+ "timeline.metrics.service.http.policy";
+
public static final String DISABLE_METRIC_METADATA_MGMT =
"timeline.metrics.service.metadata.management.disabled";
@@ -262,7 +264,7 @@ public class TimelineMetricConfiguration {
}
public String getWebappAddress() {
- String defaultHttpAddress = "0.0.0.0:8188";
+ String defaultHttpAddress = "0.0.0.0:6188";
if (metricsConf != null) {
return metricsConf.get(WEBAPP_HTTP_ADDRESS, defaultHttpAddress);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/java/org/apache/ambari/server/configuration/ComponentSSLConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/ComponentSSLConfiguration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/ComponentSSLConfiguration.java
index a51b2ee..cb9651e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/ComponentSSLConfiguration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/ComponentSSLConfiguration.java
@@ -21,7 +21,7 @@ package org.apache.ambari.server.configuration;
* Configuration for SSL communication between Ambari and 3rd party services.
* Currently, the following services are supported with SSL communication:
* <ul>
- * <li>Ganglia</li>
+ * <li>Ambari metrics</li>
* </ul>
*/
public class ComponentSSLConfiguration {
@@ -32,7 +32,7 @@ public class ComponentSSLConfiguration {
private String truststorePath;
private String truststorePassword;
private String truststoreType;
- private boolean gangliaSSL;
+ private boolean httpsEnabled;
/**
* The singleton.
@@ -60,7 +60,7 @@ public class ComponentSSLConfiguration {
truststorePath = configuration.getProperty(Configuration.SSL_TRUSTSTORE_PATH_KEY);
truststorePassword = getPassword(configuration);
truststoreType = configuration.getProperty(Configuration.SSL_TRUSTSTORE_TYPE_KEY);
- gangliaSSL = Boolean.parseBoolean(configuration.getProperty(Configuration.GANGLIA_HTTPS_KEY));
+ httpsEnabled = Boolean.parseBoolean(configuration.getProperty(Configuration.AMRABI_METRICS_HTTPS_ENABLED_KEY));
}
@@ -94,12 +94,12 @@ public class ComponentSSLConfiguration {
}
/**
- * Indicates whether or not Ganglia is setup for SSL.
+ * Indicates whether or not Ambari Metrics is setup for SSL.
*
- * @return true if Ganglia is setup for SSL
+ * @return true if AMS is setup for SSL
*/
- public boolean isGangliaSSL() {
- return gangliaSSL;
+ public boolean isHttpsEnabled() {
+ return httpsEnabled;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 35e9b6f..cfc2f51 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -242,7 +242,6 @@ public class Configuration {
public static final String JAVAX_SSL_TRUSTSTORE = "javax.net.ssl.trustStore";
public static final String JAVAX_SSL_TRUSTSTORE_PASSWORD = "javax.net.ssl.trustStorePassword";
public static final String JAVAX_SSL_TRUSTSTORE_TYPE = "javax.net.ssl.trustStoreType";
- public static final String GANGLIA_HTTPS_KEY = "ganglia.https";
public static final String SRVR_TWO_WAY_SSL_PORT_DEFAULT = "8441";
public static final String SRVR_ONE_WAY_SSL_PORT_DEFAULT = "8440";
public static final String SRVR_CRT_NAME_DEFAULT = "ca.crt";
@@ -483,6 +482,9 @@ public class Configuration {
private static final String DEFAULT_TIMELINE_METRICS_CACHE_HEAP_PERCENT = "15%";
private static final String TIMELINE_METRICS_CACHE_USE_CUSTOM_SIZING_ENGINE = "server.timeline.metrics.cache.use.custom.sizing.engine";
+ // Timeline Metrics SSL settings
+ public static final String AMRABI_METRICS_HTTPS_ENABLED_KEY = "server.timeline.metrics.https.enabled";
+
/**
* Governs the use of {@link Parallel} to process {@link StageEntity}
* instances into {@link Stage}.
@@ -921,8 +923,6 @@ public class Configuration {
jsonObject = jsonElement.getAsJsonObject();
} catch (FileNotFoundException e) {
throw new IllegalArgumentException("No file " + file, e);
- } catch (IOException ioe){
- throw new IllegalArgumentException("Can't read file " + file, ioe);
}
return jsonObject;
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProvider.java
index c62b00c..16e2cd4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaPropertyProvider.java
@@ -253,7 +253,7 @@ public abstract class GangliaPropertyProvider extends MetricsPropertyProvider {
URIBuilder uriBuilder = new URIBuilder();
- if (configuration.isGangliaSSL()) {
+ if (configuration.isHttpsEnabled()) {
uriBuilder.setScheme("https");
} else {
uriBuilder.setScheme("http");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProvider.java
index e8dc7d1..80ecae6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/ganglia/GangliaReportPropertyProvider.java
@@ -209,7 +209,7 @@ public class GangliaReportPropertyProvider extends MetricsReportPropertyProvider
StringBuilder sb = new StringBuilder();
- if (configuration.isGangliaSSL()) {
+ if (configuration.isHttpsEnabled()) {
sb.append("https://");
} else {
sb.append("http://");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
index 3ecb520..a674371 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java
@@ -34,7 +34,6 @@ import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.spi.TemporalInfo;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
-import org.apache.ambari.server.controller.utilities.StreamProvider;
import org.apache.ambari.server.state.StackId;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
@@ -625,7 +624,8 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider {
if (metricsRequest == null) {
metricsRequest = new MetricsRequest(temporalInfo,
getAMSUriBuilder(collectorHostName,
- collectorPort != null ? Integer.parseInt(collectorPort) : COLLECTOR_DEFAULT_PORT),
+ collectorPort != null ? Integer.parseInt(collectorPort) : COLLECTOR_DEFAULT_PORT,
+ configuration.isHttpsEnabled()),
(String) resource.getPropertyValue(clusterNamePropertyId));
requests.put(temporalInfo, metricsRequest);
}
@@ -643,9 +643,9 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider {
return requestMap;
}
- static URIBuilder getAMSUriBuilder(String hostname, int port) {
+ static URIBuilder getAMSUriBuilder(String hostname, int port, boolean httpsEnabled) {
URIBuilder uriBuilder = new URIBuilder();
- uriBuilder.setScheme("http");
+ uriBuilder.setScheme(httpsEnabled ? "https" : "http");
uriBuilder.setHost(hostname);
uriBuilder.setPort(port);
uriBuilder.setPath("/ws/v1/timeline/metrics");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
index e40a5a5..306390c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSReportPropertyProvider.java
@@ -174,7 +174,7 @@ public class AMSReportPropertyProvider extends MetricsReportPropertyProvider {
String host = hostProvider.getCollectorHostName(clusterName, TIMELINE_METRICS);
String port = hostProvider.getCollectorPort(clusterName, TIMELINE_METRICS);
URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder(host,
- port != null ? Integer.parseInt(port) : 8188);
+ port != null ? Integer.parseInt(port) : 6188, configuration.isHttpsEnabled());
for (Map.Entry<String, MetricReportRequest> entry : reportRequestMap.entrySet()) {
MetricReportRequest reportRequest = entry.getValue();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/metainfo.xml
index f40da62..bd9f357 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/metainfo.xml
@@ -172,6 +172,7 @@
<configuration-dependencies>
<config-type>accumulo-env</config-type>
<config-type>accumulo-site</config-type>
+ <config-type>ams-ssl-client</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
index a9626b6..2bd2626 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
@@ -131,6 +131,13 @@ if has_metric_collector:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+ metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
pass
metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
index 399e0c9..c2c12a5 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2
@@ -35,7 +35,12 @@ rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
accumulo.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
accumulo.sink.timeline.period={{metrics_collection_period}}
accumulo.sink.timeline.sendInterval={{metrics_report_interval}}000
-accumulo.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+accumulo.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+
+# HTTPS properties
+accumulo.sink.timeline.truststore.path = {{metric_truststore_path}}
+accumulo.sink.timeline.truststore.type = {{metric_truststore_type}}
+accumulo.sink.timeline.truststore.password = {{metric_truststore_password}}
{% else %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index ab9593d..08bf47c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -484,7 +484,6 @@
</property>
</depends-on>
</property>
-
<property>
<name>timeline.metrics.sink.report.interval</name>
<value>60</value>
@@ -553,5 +552,16 @@
utilization only for user queries.
</description>
</property>
+ <property>
+ <name>timeline.metrics.service.http.policy</name>
+ <value>HTTP_ONLY</value>
+ <description>
+ This configures the HTTP endpoint for Yarn Application History Server for
+ Ambari Metrics System.
+ The following values are supported:
+ - HTTP_ONLY : Service is provided only on http
+ - HTTPS_ONLY : Service is provided only on https
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
new file mode 100644
index 0000000..31586ee
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-client.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>ssl.client.truststore.location</name>
+ <value>/etc/security/clientKeys/all.jks</value>
+ <description>Location of the trust store file.</description>
+ </property>
+ <property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. Default value is "jks".</description>
+ </property>
+ <property>
+ <name>ssl.client.truststore.password</name>
+ <value>bigdata</value>
+ <property-type>PASSWORD</property-type>
+ <description>Password to open the trust store file.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-server.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-server.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-server.xml
new file mode 100644
index 0000000..f95793e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-ssl-server.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>ssl.server.truststore.location</name>
+ <value>/etc/security/serverKeys/all.jks</value>
+ <description>Location of the trust store file.</description>
+ </property>
+ <property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. Default value is "jks".</description>
+ </property>
+ <property>
+ <name>ssl.server.truststore.password</name>
+ <value>bigdata</value>
+ <property-type>PASSWORD</property-type>
+ <description>Password to open the trust store file.</description>
+ </property>
+ <property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload interval, in milliseconds.</description>
+ </property>
+ <property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. Default value is "jks".</description>
+ </property>
+ <property>
+ <name>ssl.server.keystore.location</name>
+ <value>/etc/security/serverKeys/keystore.jks</value>
+ <description>Location of the keystore file.</description>
+ </property>
+ <property>
+ <name>ssl.server.keystore.password</name>
+ <value>bigdata</value>
+ <property-type>PASSWORD</property-type>
+ <description>Password to open the keystore file.</description>
+ </property>
+ <property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value>bigdata</value>
+ <property-type>PASSWORD</property-type>
+ <description>Password for private key in keystore file.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index 67a93a5..2f2f116 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -155,6 +155,8 @@
<config-type>ams-hbase-log4j</config-type>
<config-type>ams-grafana-env</config-type>
<config-type>ams-grafana-ini</config-type>
+ <config-type>ams-ssl-server</config-type>
+ <config-type>ams-ssl-client</config-type>
</configuration-dependencies>
<excluded-config-types>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index 40188c2..417574b 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -204,6 +204,14 @@ def ams(name=None):
group=params.user_group
)
+ XmlConfig("ssl-server.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['ams-ssl-server'],
+ configuration_attributes=params.config['configuration_attributes']['ams-ssl-server'],
+ owner=params.ams_user,
+ group=params.user_group
+ )
+
merged_ams_hbase_site = {}
merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
if params.security_enabled:
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
index 5ab40b0..02caa11 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -40,7 +40,7 @@ def create_ams_datasource():
Logger.info("Connecting (GET) to %s:%s%s" % (params.hostname,
params.ams_grafana_port,
GRAFANA_URL))
-
+# TODO add https support
conn = httplib.HTTPConnection(params.hostname,
int(params.ams_grafana_port))
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 89a60f7..f3a97fc 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -50,6 +50,16 @@ ams_pid_dir = status_params.ams_collector_pid_dir
ams_collector_script = "/usr/sbin/ambari-metrics-collector"
ams_collector_pid_dir = status_params.ams_collector_pid_dir
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_https_enabled = True
+ metric_collector_protocol = 'https'
+else:
+ metric_collector_https_enabled = False
+ metric_collector_protocol = 'http'
+metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
+
if 'cluster-env' in config['configurations'] and \
'metrics_collector_vip_host' in config['configurations']['cluster-env']:
metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
index f19c823..4346f0f 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
@@ -79,8 +79,9 @@ class AMSServiceCheck(Script):
Logger.info("Connecting (POST) to %s:%s%s" % (params.metric_collector_host,
params.metric_collector_port,
self.AMS_METRICS_POST_URL))
- conn = httplib.HTTPConnection(params.metric_collector_host,
- int(params.metric_collector_port))
+ conn = self.get_http_connection(params.metric_collector_host,
+ int(params.metric_collector_port),
+ params.metric_collector_https_enabled)
conn.request("POST", self.AMS_METRICS_POST_URL, metric_json, headers)
response = conn.getresponse()
@@ -127,8 +128,9 @@ class AMSServiceCheck(Script):
params.metric_collector_port,
self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters))
- conn = httplib.HTTPConnection(params.metric_collector_host,
- int(params.metric_collector_port))
+ conn = self.get_http_connection(params.metric_collector_host,
+ int(params.metric_collector_port),
+ params.metric_collector_https_enabled)
conn.request("GET", self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
response = conn.getresponse()
Logger.info("Http response: %s %s" % (response.status, response.reason))
@@ -161,6 +163,13 @@ class AMSServiceCheck(Script):
Logger.info("Ambari Metrics service check is finished.")
+ def get_http_connection(self, host, port, https_enabled=False):
+ if https_enabled:
+ # TODO verify certificate
+ return httplib.HTTPSConnection(host, port)
+ else:
+ return httplib.HTTPConnection(host, port)
+
if __name__ == "__main__":
AMSServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
index a65ea88..1591d82 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2
@@ -55,9 +55,14 @@ rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period={{metrics_collection_period}}
hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
-hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
hbase.sink.timeline.serviceName-prefix=ams
+# HTTPS properties
+hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
+hbase.sink.timeline.truststore.type = {{metric_truststore_type}}
+hbase.sink.timeline.truststore.password = {{metric_truststore_password}}
+
# Switch off metrics generation on a per region basis
*.source.filter.class=org.apache.hadoop.metrics2.filter.GlobFilter
hbase.*.source.filter.exclude=*Regions*
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
index fc86a58..4e2d0f5 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
@@ -22,6 +22,7 @@ metrics_server = {{metric_collector_host}}:{{metric_collector_port}}
hostname = {{hostname}}
enable_time_threshold = false
enable_value_threshold = false
+https_enabled = {{metric_collector_https_enabled}}
[emitter]
send_interval = {{metrics_report_interval}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
index 5e5e4a7..0396c28 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
@@ -58,6 +58,7 @@
<configuration-dependencies>
<config-type>flume-env</config-type>
<config-type>flume-conf</config-type>
+ <config-type>ams-ssl-client</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
index 28ee36b..5ec879c 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py
@@ -101,6 +101,13 @@ if has_metric_collector:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+ metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
pass
metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
index 753aa3f..df68242 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2
@@ -16,11 +16,15 @@
# limitations under the License.
#}
-collector={{metric_collector_host}}
-port={{metric_collector_port}}
+collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
collectionFrequency={{metrics_collection_period}}000
maxRowCacheSize=10000
sendInterval={{metrics_report_interval}}000
+# HTTPS properties
+truststore.path = {{metric_truststore_path}}
+truststore.type = {{metric_truststore_type}}
+truststore.password = {{metric_truststore_password}}
+
# Metric names having type COUNTER
counters=EventTakeSuccessCount,EventPutSuccessCount,EventTakeAttemptCount,EventPutAttemptCount
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
index ce625eb..7a71604 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
@@ -149,6 +149,7 @@
<config-type>yarn-client</config-type>
<config-type>hawq-limits-env</config-type>
<config-type>hawq-sysctl-env</config-type>
+ <config-type>ams-ssl-client</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/metainfo.xml
index 78aca3d..057e126 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/metainfo.xml
@@ -147,6 +147,7 @@
<config-type>ranger-hbase-audit</config-type>
<config-type>ranger-hbase-policymgr-ssl</config-type>
<config-type>ranger-hbase-security</config-type>
+ <config-type>ams-ssl-client</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index 6bbf379..6837bf1 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -149,6 +149,14 @@ if has_metric_collector:
metric_collector_port = metric_collector_web_address.split(':')[1]
else:
metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+ metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
+
pass
metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
index f74ea4c..3202605 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
@@ -66,7 +66,12 @@ rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period={{metrics_collection_period}}
hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
-hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+
+# HTTPS properties
+hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
+hbase.sink.timeline.truststore.type = {{metric_truststore_type}}
+hbase.sink.timeline.truststore.password = {{metric_truststore_password}}
{% else %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
index 2df141f..5a91866 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
@@ -65,7 +65,12 @@ rpc.collector={{metric_collector_host}}:{{metric_collector_port}}
hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
hbase.sink.timeline.period={{metrics_collection_period}}
hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
-hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}}
+hbase.sink.timeline.collector={{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}
+
+# HTTPS properties
+hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
+hbase.sink.timeline.truststore.type = {{metric_truststore_type}}
+hbase.sink.timeline.truststore.password = {{metric_truststore_password}}
{% else %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 73616ae..75d3bea 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -245,6 +245,7 @@
<config-type>ranger-hdfs-audit</config-type>
<config-type>ranger-hdfs-policymgr-ssl</config-type>
<config-type>ranger-hdfs-security</config-type>
+ <config-type>ams-ssl-client</config-type>
</configuration-dependencies>
<restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e75e52a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml
index 6d0c8c6..704d73f 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml
@@ -325,6 +325,27 @@
<description>Timeline port</description>
</property>
<property>
+ <property>
+ <name>kafka.timeline.metrics.protocol</name>
+ <value>{{metric_collector_protocol}}</value>
+ <description>Timeline protocol(http or https)</description>
+ </property>
+ <property>
+ <name>kafka.timeline.metrics.truststore.path</name>
+ <value>{{metric_truststore_path}}</value>
+ <description>Location of the trust store file.</description>
+ </property>
+ <property>
+ <name>kafka.timeline.metrics.truststore.type</name>
+ <value>{{metric_truststore_type}}</value>
+ <description>Optional. Default value is "jks".</description>
+ </property>
+ <property>
+ <name>kafka.timeline.metrics.truststore.password</name>
+ <value>{{metric_truststore_password}}</value>
+ <description>Password to open the trust store file.</description>
+ </property>
+ <property>
<name>kafka.timeline.metrics.reporter.sendInterval</name>
<value>5900</value>
<description>Timeline metrics reporter send interval</description>