You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2017/01/18 23:36:57 UTC
ambari git commit: AMBARI-19587 Log Feeder should be able to handle
metrics with https (mgergely)
Repository: ambari
Updated Branches:
refs/heads/trunk c27fdbc22 -> 498220505
AMBARI-19587 Log Feeder should be able to handle metrics with https (mgergely)
Change-Id: Id66b77e673be8520c05c2c1c0ca314919844dde8
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49822050
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49822050
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49822050
Branch: refs/heads/trunk
Commit: 4982205059e3ee2b8e4423c9ee889bd81ac7ed61
Parents: c27fdbc
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Thu Jan 19 00:36:48 2017 +0100
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Thu Jan 19 00:36:48 2017 +0100
----------------------------------------------------------------------
.../logfeeder/metrics/LogFeederAMSClient.java | 40 ++++++++++++++------
.../apache/ambari/logfeeder/util/SSLUtil.java | 26 +++++++++++++
.../server/upgrade/UpgradeCatalog250.java | 8 +---
.../LOGSEARCH/0.5.0/package/scripts/params.py | 27 +++++++++++--
.../0.5.0/properties/logfeeder-env.sh.j2 | 2 +-
.../0.5.0/properties/logsearch-env.sh.j2 | 2 +-
.../server/upgrade/UpgradeCatalog250Test.java | 6 +--
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 3 ++
8 files changed, 88 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
index 21ca165..2d1bf40 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
@@ -20,33 +20,51 @@
package org.apache.ambari.logfeeder.metrics;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.ambari.logfeeder.util.SSLUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
import org.apache.log4j.Logger;
+import com.google.common.base.Splitter;
+
import java.util.Collection;
+import java.util.List;
// TODO: Refactor for failover
public class LogFeederAMSClient extends AbstractTimelineMetricsSink {
private static final Logger LOG = Logger.getLogger(LogFeederAMSClient.class);
- private String collectorHosts = null;
+ private final List<String> collectorHosts;
+ private final String collectorProtocol;
+ private final String collectorPort;
+ private final String collectorPath;
public LogFeederAMSClient() {
- collectorHosts = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.hosts");
- if (StringUtils.isBlank(collectorHosts)) {
+ String collectorHostsString = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.hosts");
+ if (!StringUtils.isBlank(collectorHostsString)) {
+ collectorHostsString = collectorHostsString.trim();
+ LOG.info("AMS collector Hosts=" + collectorHostsString);
+
+ collectorHosts = Splitter.on(",").splitToList(collectorHostsString);
+ collectorProtocol = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.protocol");
+ collectorPort = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.port");
+ collectorPath = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.path");
+ } else {
collectorHosts = null;
+ collectorProtocol = null;
+ collectorPort = null;
+ collectorPath = null;
}
- if (collectorHosts != null) {
- collectorHosts = collectorHosts.trim();
+
+ if (StringUtils.isNotBlank(SSLUtil.getTrustStoreLocation())) {
+ loadTruststore(SSLUtil.getTrustStoreLocation(), SSLUtil.getTrustStoreType(), SSLUtil.getTrustStorePassword());
}
- LOG.info("AMS collector URL=" + collectorHosts);
}
@Override
public String getCollectorUri(String host) {
- return collectorHosts;
+ return String.format("%s://%s:%s%s", collectorProtocol, host, collectorPort, collectorPath);
}
@Override
@@ -62,7 +80,7 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink {
@Override
protected Collection<String> getConfiguredCollectorHosts() {
- return null;
+ return collectorHosts;
}
@Override
@@ -77,12 +95,12 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink {
@Override
protected String getCollectorProtocol() {
- return null;
+ return collectorProtocol;
}
@Override
protected String getCollectorPort() {
- return null;
+ return collectorPort;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
index 317f5ae..ea9f45d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java
@@ -27,6 +27,8 @@ import java.io.File;
public class SSLUtil {
private static final String KEYSTORE_LOCATION_ARG = "javax.net.ssl.keyStore";
private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore";
+ private static final String KEYSTORE_TYPE_ARG = "javax.net.ssl.keyStoreType";
+ private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType";
private static final String KEYSTORE_PASSWORD_ARG = "javax.net.ssl.keyStorePassword";
private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword";
private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt";
@@ -39,6 +41,30 @@ public class SSLUtil {
throw new UnsupportedOperationException();
}
+ public static String getKeyStoreLocation() {
+ return System.getProperty(KEYSTORE_LOCATION_ARG);
+ }
+
+ public static String getKeyStoreType() {
+ return System.getProperty(KEYSTORE_TYPE_ARG);
+ }
+
+ public static String getKeyStorePassword() {
+ return System.getProperty(KEYSTORE_PASSWORD_ARG);
+ }
+
+ public static String getTrustStoreLocation() {
+ return System.getProperty(TRUSTSTORE_LOCATION_ARG);
+ }
+
+ public static String getTrustStoreType() {
+ return System.getProperty(TRUSTSTORE_TYPE_ARG);
+ }
+
+ public static String getTrustStorePassword() {
+ return System.getProperty(TRUSTSTORE_PASSWORD_ARG);
+ }
+
public static void ensureStorePasswords() {
ensureStorePassword(KEYSTORE_LOCATION_ARG, KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_FILE);
ensureStorePassword(TRUSTSTORE_LOCATION_ARG, TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_FILE);
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 4e465c9..45d2874 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -694,7 +694,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
if (logfeederEnvProperties != null) {
String content = logfeederEnvProperties.getProperties().get("content");
if (content.contains("infra_solr_ssl_enabled")) {
- content = content.replace("infra_solr_ssl_enabled", "logsearch_solr_ssl_enabled");
+ content = content.replace("infra_solr_ssl_enabled", "logfeeder_use_ssl");
updateConfigurationPropertiesForCluster(cluster, "logfeeder-env", Collections.singletonMap("content", content), true, true);
}
}
@@ -704,11 +704,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
Map<String, String> newProperties = new HashMap<>();
String content = logsearchEnvProperties.getProperties().get("content");
if (content.contains("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'")) {
- content = content.replace("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'",
- "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl");
- }
- if (content.contains("infra_solr_ssl_enabled")) {
- content = content.replace("infra_solr_ssl_enabled", "logsearch_solr_ssl_enabled");
+ content = content.replace("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'", "logsearch_use_ssl");
}
if (!content.equals(logsearchEnvProperties.getProperties().get("content"))) {
newProperties.put("content", content);
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 25e947d..e9ab98f 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -69,12 +69,19 @@ logfeeder_metadata = get_logfeeder_metadata(logserch_meta_configs)
# for now just pick first collector
if 'metrics_collector_hosts' in config['clusterHostInfo']:
- metrics_collector_hosts_list = ",".join(config['clusterHostInfo']['metrics_collector_hosts'])
- metrics_collector_port = str(
- get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address']))
- metrics_collector_hosts = format('http://{metrics_collector_hosts_list}:{metrics_collector_port}/ws/v1/timeline/metrics')
+ metrics_http_policy = config['configurations']['ams-site']['timeline.metrics.service.http.policy']
+ metrics_collector_protocol = 'http'
+ if metrics_http_policy == 'HTTPS_ONLY':
+ metrics_collector_protocol = 'https'
+
+ metrics_collector_hosts = ",".join(config['clusterHostInfo']['metrics_collector_hosts'])
+ metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address']))
+ metrics_collector_path = '/ws/v1/timeline/metrics'
else:
+ metrics_collector_protocol = ''
metrics_collector_hosts = ''
+ metrics_collector_port = ''
+ metrics_collector_path = ''
#####################################
# Infra Solr configs
@@ -250,6 +257,10 @@ logsearch_solr_collection_audit_logs = logsearch_properties['logsearch.solr.coll
logsearch_audit_logs_split_interval_mins = logsearch_properties['logsearch.audit.logs.split.interval.mins']
logsearch_collection_audit_logs_numshards = logsearch_properties['logsearch.collection.audit.logs.numshards']
+# check if logsearch uses ssl in any way
+
+logsearch_use_ssl = logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl
+
#####################################
# Logfeeder configs
#####################################
@@ -328,6 +339,10 @@ logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_pro
logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files'])
logfeeder_properties['logfeeder.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode
+logfeeder_properties['logfeeder.metrics.collector.protocol'] = metrics_collector_protocol
+logfeeder_properties['logfeeder.metrics.collector.port'] = metrics_collector_port
+logfeeder_properties['logfeeder.metrics.collector.path'] = '/ws/v1/timeline/metrics'
+
if logsearch_solr_kerberos_enabled:
if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties:
logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
@@ -336,6 +351,10 @@ if logsearch_solr_kerberos_enabled:
logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder']
+# check if logfeeder uses ssl in any way
+
+logfeeder_use_ssl = logsearch_solr_ssl_enabled or metrics_collector_protocol == 'https'
+
#####################################
# Smoke command
#####################################
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
index 6d1c445..f2dd06b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
@@ -32,7 +32,7 @@ if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}}
fi
-{% if logsearch_solr_ssl_enabled %}
+{% if logfeeder_use_ssl %}
export LOGFEEDER_SSL="true"
export LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}}
export LOGFEEDER_KEYSTORE_TYPE={{logfeeder_keystore_type}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
index 338c7f7..f21b7be 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
@@ -38,7 +38,7 @@ export LOGSEARCH_DEBUG={{logsearch_debug_enabled}}
export LOGSEARCH_DEBUG_PORT={{logsearch_debug_port}}
-{% if logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl %}
+{% if logsearch_use_ssl %}
export LOGSEARCH_SSL="true"
export LOGSEARCH_KEYSTORE_LOCATION={{logsearch_keystore_location}}
export LOGSEARCH_KEYSTORE_TYPE={{logsearch_keystore_type}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 9e8da83..725928b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -1026,7 +1026,7 @@ public class UpgradeCatalog250Test {
"content", "infra_solr_ssl_enabled");
Map<String, String> expectedLogFeederEnv = ImmutableMap.of(
- "content", "logsearch_solr_ssl_enabled");
+ "content", "logfeeder_use_ssl");
Config mockLogFeederEnv = easyMockSupport.createNiceMock(Config.class);
expect(cluster.getDesiredConfigByType("logfeeder-env")).andReturn(mockLogFeederEnv).atLeastOnce();
@@ -1042,7 +1042,7 @@ public class UpgradeCatalog250Test {
"content", "infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'");
Map<String, String> expectedLogSearchEnv = ImmutableMap.of(
- "content", "logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl");
+ "content", "logsearch_use_ssl");
Config mockLogSearchEnv = easyMockSupport.createNiceMock(Config.class);
expect(cluster.getDesiredConfigByType("logsearch-env")).andReturn(mockLogSearchEnv).atLeastOnce();
@@ -1102,7 +1102,7 @@ public class UpgradeCatalog250Test {
expect(mockLogFeederLog4j.getProperties()).andReturn(oldLogFeederLog4j).anyTimes();
Capture<Map<String, String>> logFeederLog4jCapture = EasyMock.newCapture();
expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(logFeederLog4jCapture), anyString(),
- anyObject(Map.class))).andReturn(config).once();
+ EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
Map<String, String> oldLogSearchLog4j = ImmutableMap.of(
"content",
http://git-wip-us.apache.org/repos/asf/ambari/blob/49822050/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index b172f64..1c79c5c 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -84,6 +84,9 @@ class TestLogFeeder(RMFTestCase):
properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
'logfeeder.config.files': 'output.config.json,input.config-ambari.json,global.config.json,input.config-logsearch.json,input.config-zookeeper.json',
'logfeeder.metrics.collector.hosts': '',
+ 'logfeeder.metrics.collector.path': '/ws/v1/timeline/metrics',
+ 'logfeeder.metrics.collector.port': '',
+ 'logfeeder.metrics.collector.protocol': '',
'logfeeder.solr.core.config.name': 'history',
'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr'
}