You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2016/12/08 09:21:28 UTC
[3/3] ambari git commit: AMBARI-19012 Abillity to use external Solr
for Log Search instead of AMBARI_INFRA_SOLR
AMBARI-19012 Abillity to use external Solr for Log Search instead of AMBARI_INFRA_SOLR
Change-Id: I9501d854005e52153ed0bc38ba51d6b9113120c8
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a85000b8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a85000b8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a85000b8
Branch: refs/heads/branch-2.5
Commit: a85000b8a1e9012f95b14894a44baf861792d4bf
Parents: aedf2c0
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Thu Dec 8 10:20:51 2016 +0100
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Thu Dec 8 10:20:51 2016 +0100
----------------------------------------------------------------------
.../libraries/functions/package_conditions.py | 2 +-
.../0.5.0/configuration/logfeeder-env.xml | 14 +
.../0.5.0/configuration/logsearch-env.xml | 94 ++++++-
.../LOGSEARCH/0.5.0/metainfo.xml | 34 +--
.../LOGSEARCH/0.5.0/package/scripts/params.py | 79 +++---
.../0.5.0/package/scripts/setup_logfeeder.py | 2 +-
.../0.5.0/package/scripts/setup_logsearch.py | 10 +-
.../0.5.0/properties/logfeeder-env.sh.j2 | 2 +-
.../0.5.0/properties/logsearch-env.sh.j2 | 2 +-
.../0.5.0/properties/output.config.json.j2 | 6 +-
.../LOGSEARCH/0.5.0/themes/theme.json | 253 +++++++++++++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 55 ++--
.../test/python/stacks/2.4/configs/default.json | 2 +-
ambari-web/app/data/HDP2/site_properties.js | 36 +--
14 files changed, 472 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index 8257022..5a16061 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -70,7 +70,7 @@ def should_install_infra_solr():
def should_install_infra_solr_client():
config = Script.get_config()
- return _has_applicable_local_component(config, ['INFRA_SOLR_CLIENT', 'ATLAS_SERVER', 'RANGER_ADMIN'])
+ return _has_applicable_local_component(config, ['INFRA_SOLR_CLIENT', 'ATLAS_SERVER', 'RANGER_ADMIN', 'LOGSEARCH_SERVER'])
def should_install_logsearch_portal():
config = Script.get_config()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
index ee885e3..aba638a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
@@ -119,6 +119,20 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>logfeeder_external_solr_kerberos_keytab</name>
+ <value></value>
+ <display-name>Logfeeder External Solr keytab</display-name>
+ <description>The path to the Kerberos Keytab file containing service principal of Logfeeder e.g. /etc/security/keytabs/logfeeder.service.keytab</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logfeeder_external_solr_kerberos_principal</name>
+ <value></value>
+ <display-name>Logfeeder External Solr principal</display-name>
+ <description>The service principal for Logfeeder which will be used to access SolrClient e.g. logfeeder/_HOST@REALM</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>logfeeder_kerberos_keytab</name>
<value>/etc/security/keytabs/logfeeder.service.keytab</value>
<display-name>Logfeeder Solr keytab</display-name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
index c5b9b4e..2f13710 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
@@ -99,20 +99,100 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>logsearch_solr_audit_logs_zk_node</name>
- <value>{infra_solr_znode}</value>
- <display-name>Solr Audit Logs Znode</display-name>
- <description>Only needed if using custom solr cloud. E.g. /audit_logs</description>
+ <name>logsearch_use_external_solr</name>
+ <value>false</value>
+ <display-name>Use External Solr</display-name>
+ <description>Use External Solr to store logs</description>
+ <value-attributes>
+ <type>value-list</type>
+ <overridable>false</overridable>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>ON</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>OFF</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logsearch_external_solr_zk_znode</name>
+ <value></value>
+ <display-name>External Solr Logs Znode</display-name>
+ <description>Only needed if using custom solr cloud. E.g. /logsearch_solr</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>logsearch_solr_audit_logs_zk_quorum</name>
- <value>{zookeeper_quorum}</value>
- <display-name>Solr Audit Logs ZK Quorum</display-name>
+ <name>logsearch_external_solr_zk_quorum</name>
+ <value></value>
+ <display-name>External Solr Logs ZK Quorum</display-name>
<description>Only needed if using custom solr cloud. E.g. zk1:2181,zk2:2182</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>logsearch_external_solr_ssl_enabled</name>
+ <value>false</value>
+ <display-name>Log Search External Solr SSL Enabled</display-name>
+ <description>Use SSL to connect to the external solr</description>
+ <value-attributes>
+ <type>value-list</type>
+ <overridable>false</overridable>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>ON</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>OFF</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logsearch_external_solr_kerberos_enabled</name>
+ <value>false</value>
+ <display-name>Log Search External Solr Kerberos Enabled</display-name>
+ <description>Use Kerberos to connect to the external solr</description>
+ <value-attributes>
+ <type>value-list</type>
+ <overridable>false</overridable>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>ON</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>OFF</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logsearch_external_solr_kerberos_keytab</name>
+ <value></value>
+ <display-name>Log Search External Solr keytab</display-name>
+ <description>The path to the Kerberos Keytab file containing service principal of Log Search e.g. /etc/security/keytabs/logsearch.service.keytab</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logsearch_external_solr_kerberos_principal</name>
+ <value></value>
+ <display-name>Log Search External Solr principal</display-name>
+ <description>The service principal for Log Search which will be used to access SolrClient e.g. logsearch/_HOST@REALM</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>logsearch_truststore_location</name>
<value>/etc/security/serverKeys/logsearch.trustStore.jks</value>
<display-name>Log Search trust store location</display-name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index ed74bdf..6921a8e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -55,15 +55,6 @@
<enabled>true</enabled>
</auto-deploy>
</dependency>
-
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
-
</dependencies>
<configuration-dependencies>
<config-type>infra-solr-env</config-type>
@@ -95,20 +86,9 @@
</log>
</logs>
- <dependencies>
-
- <dependency>
- <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
- <scope>cluster</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
-
- </dependencies>
-
<configuration-dependencies>
<config-type>infra-solr-env</config-type>
+ <config-type>logsearch-env</config-type>
<config-type>logfeeder-properties</config-type>
<config-type>logfeeder-env</config-type>
<config-type>logfeeder-grok</config-type>
@@ -135,6 +115,10 @@
<skipUpgrade>true</skipUpgrade>
<condition>should_install_logsearch_portal</condition>
</package>
+ <package>
+ <name>ambari-infra-solr-client</name>
+ <condition>should_install_infra_solr_client</condition>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -149,6 +133,10 @@
<skipUpgrade>true</skipUpgrade>
<condition>should_install_logsearch_portal</condition>
</package>
+ <package>
+ <name>ambari-infra-solr-client</name>
+ <condition>should_install_infra_solr_client</condition>
+ </package>
</packages>
</osSpecific>
</osSpecifics>
@@ -159,10 +147,6 @@
<timeout>300</timeout>
</commandScript>
- <requiredServices>
- <service>AMBARI_INFRA</service>
- </requiredServices>
-
<quickLinksConfigurations>
<quickLinksConfiguration>
<fileName>quicklinks.json</fileName>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index d086f9f..c20d92d 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -38,7 +38,6 @@ def get_port_from_url(address):
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
-stack_version = default("/commandParams/version", None)
sudo = AMBARI_SUDO_BINARY
security_enabled = status_params.security_enabled
@@ -56,16 +55,10 @@ logfeeder_pid_dir = status_params.logfeeder_pid_dir
logfeeder_pid_file = status_params.logfeeder_pid_file
user_group = config['configurations']['cluster-env']['user_group']
-fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"]
# shared configs
java64_home = config['hostLevelParams']['java_home']
-zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts']
-zookeeper_hosts_list.sort()
-# get comma separated list of zookeeper hosts from clusterHostInfo
-zookeeper_hosts = ",".join(zookeeper_hosts_list)
cluster_name = str(config['clusterName'])
-availableServices = config['availableServices']
configurations = config['configurations'] # need reference inside logfeeder jinja templates
logserch_meta_configs = get_logsearch_meta_configs(configurations)
@@ -84,30 +77,31 @@ else:
#####################################
# Infra Solr configs
#####################################
-infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr')
-infra_solr_instance_count = len(config['clusterHostInfo']['infra_solr_hosts'])
-infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False)
-infra_solr_jmx_port = config['configurations']['infra-solr-env']['infra_solr_jmx_port']
-
-zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
-index = 0
-zookeeper_quorum = ""
-for host in config['clusterHostInfo']['zookeeper_hosts']:
- zookeeper_quorum += host + ":" + str(zookeeper_port)
- index += 1
- if index < len(config['clusterHostInfo']['zookeeper_hosts']):
- zookeeper_quorum += ","
+infra_solr_znode = '/infra-solr'
+infra_solr_ssl_enabled = False
+infra_solr_jmx_port = ''
+if 'infra-solr-env' in config['configurations']:
+ infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr')
+ infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False)
+ infra_solr_jmx_port = config['configurations']['infra-solr-env']['infra_solr_jmx_port']
if security_enabled:
kinit_path_local = status_params.kinit_path_local
_hostname_lowercase = config['hostname'].lower()
logsearch_jaas_file = logsearch_server_conf + '/logsearch_jaas.conf'
logfeeder_jaas_file = logsearch_logfeeder_conf + '/logfeeder_jaas.conf'
- logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_kerberos_keytab']
- logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_kerberos_principal'].replace('_HOST',_hostname_lowercase)
- logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_kerberos_keytab']
- logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_kerberos_principal'].replace('_HOST',_hostname_lowercase)
+ use_external_solr_with_kerberos = default('configurations/logsearch-env/logsearch_external_solr_kerberos_enabled', False)
+ if use_external_solr_with_kerberos:
+ logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_external_solr_kerberos_keytab']
+ logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_external_solr_kerberos_principal'].replace('_HOST',_hostname_lowercase)
+ logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_external_solr_kerberos_keytab']
+ logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_external_solr_kerberos_principal'].replace('_HOST',_hostname_lowercase)
+ else:
+ logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_kerberos_keytab']
+ logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_kerberos_principal'].replace('_HOST',_hostname_lowercase)
+ logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_kerberos_keytab']
+ logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_kerberos_principal'].replace('_HOST',_hostname_lowercase)
#####################################
# Logsearch configs
@@ -120,10 +114,29 @@ logsearch_service_logs_merge_factor = config['configurations']['logsearch-servic
logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention']
logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor']
-logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', infra_solr_znode)
-logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', zookeeper_quorum)
-logsearch_solr_audit_logs_zk_node = format(logsearch_solr_audit_logs_zk_node)
-logsearch_solr_audit_logs_zk_quorum = format(logsearch_solr_audit_logs_zk_quorum)
+logsearch_use_external_solr = default('/configurations/logsearch-env/logsearch_use_external_solr', False)
+
+if logsearch_use_external_solr:
+ logsearch_solr_zk_znode = config['configurations']['logsearch-env']['logsearch_external_solr_zk_znode']
+ logsearch_solr_zk_quorum = config['configurations']['logsearch-env']['logsearch_external_solr_zk_quorum']
+ logsearch_solr_ssl_enabled = default('configurations/logsearch-env/logsearch_external_solr_ssl_enabled', False)
+ logsearch_solr_kerberos_enabled = security_enabled and default('configurations/logsearch-env/logsearch_external_solr_kerberos_enabled', False)
+else:
+ logsearch_solr_zk_znode = infra_solr_znode
+
+ logsearch_solr_zk_quorum = ""
+ zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
+ if 'zookeeper_hosts' in config['clusterHostInfo']:
+ for host in config['clusterHostInfo']['zookeeper_hosts']:
+ if logsearch_solr_zk_quorum:
+ logsearch_solr_zk_quorum += ','
+ logsearch_solr_zk_quorum += host + ":" + str(zookeeper_port)
+
+ logsearch_solr_ssl_enabled = infra_solr_ssl_enabled
+ logsearch_solr_kerberos_enabled = security_enabled
+
+zookeeper_quorum = logsearch_solr_zk_quorum
+
# logsearch-env configs
@@ -179,8 +192,8 @@ logsearch_properties = {}
# default values
-logsearch_properties['logsearch.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
-logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_audit_logs_zk_quorum + logsearch_solr_audit_logs_zk_node
+logsearch_properties['logsearch.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode
+logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode
logsearch_properties['logsearch.solr.collection.history'] = 'history'
logsearch_properties['logsearch.solr.history.config.name'] = 'history'
@@ -210,7 +223,7 @@ del logsearch_properties['logsearch.solr.audit.logs.use.ranger']
logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts'])
logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url'])
-if security_enabled:
+if logsearch_solr_kerberos_enabled:
logsearch_properties['logsearch.solr.kerberos.enable'] = 'true'
logsearch_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file
@@ -293,9 +306,9 @@ logfeeder_properties = dict(logfeeder_properties.items() + dict(config['configur
logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts'])
logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files'])
-logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
+logfeeder_properties['logfeeder.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode
-if security_enabled:
+if logsearch_solr_kerberos_enabled:
if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties:
logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
if 'logfeeder.solr.jaas.file' not in logfeeder_properties:
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index 9582334..14f8d20 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -92,7 +92,7 @@ def setup_logfeeder():
)
- if params.security_enabled:
+ if params.logsearch_solr_kerberos_enabled:
File(format("{logfeeder_jaas_file}"),
content=Template("logfeeder_jaas.conf.j2")
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 351126b..874b90b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -118,10 +118,10 @@ def setup_logsearch():
def upload_conf_set(config_set, solrconfig_content = None):
import params
- jaas_file = params.logsearch_jaas_file if params.security_enabled else None
+ jaas_file = params.logsearch_jaas_file if params.logsearch_solr_kerberos_enabled else None
solr_cloud_util.upload_configuration_to_zk(
- zookeeper_quorum=params.zookeeper_quorum,
- solr_znode=params.infra_solr_znode,
+ zookeeper_quorum=params.logsearch_solr_zk_quorum,
+ solr_znode=params.logsearch_solr_zk_znode,
config_set_dir=format("{logsearch_server_conf}/solr_configsets/{config_set}/conf"),
config_set=config_set,
tmp_dir=params.tmp_dir,
@@ -134,7 +134,7 @@ def upload_conf_set(config_set, solrconfig_content = None):
def check_znode():
import params
solr_cloud_util.check_znode(
- zookeeper_quorum=params.zookeeper_quorum,
- solr_znode=params.infra_solr_znode,
+ zookeeper_quorum=params.logsearch_solr_zk_quorum,
+ solr_znode=params.logsearch_solr_zk_znode,
java64_home=params.java64_home,
retry=30, interval=5)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
index 2818708..6795dab 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2
@@ -32,7 +32,7 @@ if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then
export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}}
fi
-{% if infra_solr_ssl_enabled %}
+{% if logsearch_solr_ssl_enabled %}
export LOGFEEDER_SSL="true"
export LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}}
export LOGFEEDER_KEYSTORE_PASSWORD={{logfeeder_keystore_password}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
index eb7306c..a179983 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2
@@ -38,7 +38,7 @@ export LOGSEARCH_DEBUG={{logsearch_debug_enabled}}
export LOGSEARCH_DEBUG_PORT={{logsearch_debug_port}}
-{% if infra_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl %}
+{% if logsearch_solr_ssl_enabled or logsearch_ui_protocol == 'https' or ambari_server_use_ssl %}
export LOGSEARCH_SSL="true"
export LOGSEARCH_KEYSTORE_LOCATION={{logsearch_keystore_location}}
export LOGSEARCH_KEYSTORE_PASSWORD={{logsearch_keystore_password}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
index 062d636..214e5ba 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2
@@ -21,7 +21,7 @@
"comment":"Output to solr for service logs",
"is_enabled":"{{solr_service_logs_enable}}",
"destination":"solr",
- "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}",
+ "zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}",
"collection":"{{logsearch_solr_collection_service_logs}}",
"number_of_shards": "{{logsearch_collection_service_logs_numshards}}",
"splits_interval_mins": "{{logsearch_service_logs_split_interval_mins}}",
@@ -40,7 +40,7 @@
"comment":"Output to solr for audit records",
"is_enabled":"{{solr_audit_logs_enable}}",
"destination":"solr",
- "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}",
+ "zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}",
"collection":"{{logsearch_solr_collection_audit_logs}}",
"number_of_shards": "{{logsearch_collection_audit_logs_numshards}}",
"splits_interval_mins": "{{logsearch_audit_logs_split_interval_mins}}",
@@ -57,5 +57,5 @@
}
]
-
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json
index 0193689..2858825 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/themes/theme.json
@@ -57,6 +57,25 @@
"column-span": "1"
}
]
+ },
+ {
+ "name": "section-logsearch-solr-connection",
+ "display-name": "Solr Connection",
+ "row-index": "4",
+ "column-index": "1",
+ "row-span": "3",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "4",
+ "subsections": [
+ {
+ "name": "subsection-logsearch-solr-connection-col1",
+ "row-index": "4",
+ "column-index": "0",
+ "row-span": "3",
+ "column-span": "1"
+ }
+ ]
}
]
}
@@ -138,6 +157,186 @@
}
}
]
+ },
+ {
+ "config": "logsearch-env/logsearch_use_external_solr",
+ "subsection-name": "subsection-logsearch-solr-connection-col1"
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_zk_znode",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_use_external_solr"
+ ],
+ "if": "${logsearch-env/logsearch_use_external_solr}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_zk_quorum",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_use_external_solr"
+ ],
+ "if": "${logsearch-env/logsearch_use_external_solr}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_ssl_enabled",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_use_external_solr"
+ ],
+ "if": "${logsearch-env/logsearch_use_external_solr}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_kerberos_enabled",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_use_external_solr"
+ ],
+ "if": "${logsearch-env/logsearch_use_external_solr}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_kerberos_keytab",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_external_solr_kerberos_enabled"
+ ],
+ "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_kerberos_principal",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_external_solr_kerberos_enabled"
+ ],
+ "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logfeeder-env/logfeeder_external_solr_kerberos_keytab",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_external_solr_kerberos_enabled"
+ ],
+ "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "logfeeder-env/logfeeder_external_solr_kerberos_principal",
+ "subsection-name": "subsection-logsearch-solr-connection-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "logsearch-env/logsearch_external_solr_kerberos_enabled"
+ ],
+ "if": "${logsearch-env/logsearch_external_solr_kerberos_enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
}
]
},
@@ -220,6 +419,60 @@
"widget": {
"type": "directories"
}
+ },
+ {
+ "config": "logsearch-env/logsearch_use_external_solr",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_zk_znode",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_zk_quorum",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_ssl_enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_kerberos_enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_kerberos_keytab",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "logsearch-env/logsearch_external_solr_kerberos_principal",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "logfeeder-env/logfeeder_external_solr_kerberos_keytab",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "logfeeder-env/logfeeder_external_solr_kerberos_principal",
+ "widget": {
+ "type": "text-field"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85000b8/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 4f0a9d0..4802193 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -1015,27 +1015,50 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def recommendLogsearchConfigurations(self, configurations, clusterData, services, hosts):
putLogsearchProperty = self.putProperty(configurations, "logsearch-properties", services)
+ putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties")
+ putLogsearchEnvProperty = self.putProperty(configurations, "logsearch-env", services)
+ putLogsearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env")
+ putLogfeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env")
+
infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA", "INFRA_SOLR")
- if infraSolrHosts is not None and len(infraSolrHosts) > 0 \
- and "logsearch-properties" in services["configurations"]:
+ if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]:
+ replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
+ recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
+
recommendedMinShards = len(infraSolrHosts)
recommendedShards = 2 * len(infraSolrHosts)
recommendedMaxShards = 3 * len(infraSolrHosts)
- # recommend number of shard
- putLogsearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties")
- putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
- putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
- putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
-
- putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
- putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
- putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
- # recommend replication factor
- replicationReccomendFloat = math.log(len(infraSolrHosts), 5)
- recommendedReplicationFactor = int(1 + math.floor(replicationReccomendFloat))
- putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
- putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
+ else:
+ recommendedReplicationFactor = 2
+
+ recommendedMinShards = 1
+ recommendedShards = 1
+ recommendedMaxShards = 100
+
+ putLogsearchEnvProperty('logsearch_use_external_solr', 'true')
+ putLogsearchEnvAttribute('logsearch_use_external_solr', 'visible', 'false')
+
+ # recommend number of shard
+ putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards)
+ putLogsearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogsearchProperty("logsearch.collection.service.logs.numshards", recommendedShards)
+
+ putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards)
+ putLogsearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards)
+ putLogsearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards)
+ # recommend replication factor
+ putLogsearchProperty("logsearch.collection.service.logs.replication.factor", recommendedReplicationFactor)
+ putLogsearchProperty("logsearch.collection.audit.logs.replication.factor", recommendedReplicationFactor)
+
+ kerberos_authentication_enabled = self.isSecurityEnabled(services)
+ if not kerberos_authentication_enabled:
+ putLogsearchEnvProperty('logsearch_external_solr_kerberos_enabled', 'false')
+ putLogsearchEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false')
+ putLogsearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false')
+ putLogsearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false')
+ putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false')
+ putLogfeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false')
def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},