You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/01/19 13:13:27 UTC
[1/3] ambari git commit: AMBARI-19468. Run ZK operations in logsearch
internally and enable ACL security (oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 e382df243 -> 548a0b6a3
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties b/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
index 1bd8c70..39c0dac 100644
--- a/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
+++ b/ambari-logsearch/docker/test-config/logsearch/logsearch-https.properties
@@ -36,6 +36,9 @@ logsearch.solr.collection.history=history
logsearch.solr.history.config.name=history
logsearch.collection.history.replication.factor=1
+logsearch.solr.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/solr_configsets
+logsearch.solr.audit.logs.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/solr_configsets
+
# Metrics
logsearch.solr.metrics.collector.hosts=
logsearch.solr.jmx.port=18886
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
index 71c1a1c..6a97efc 100644
--- a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
+++ b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties
@@ -31,6 +31,9 @@ logsearch.audit.logs.split.interval.mins=15
logsearch.collection.audit.logs.numshards=3
logsearch.collection.audit.logs.replication.factor=2
+logsearch.solr.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/solr_configsets
+logsearch.solr.audit.logs.config_set.folder=/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/solr_configsets
+
# History logs
logsearch.solr.collection.history=history
logsearch.solr.history.config.name=history
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/infra_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/infra_solr.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/infra_solr.py
index 532be47..4a8b9c0 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/infra_solr.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/infra_solr.py
@@ -18,6 +18,7 @@ limitations under the License.
"""
from resource_management.core.resources.system import Execute, File
+from resource_management.core.resources.zkmigrator import ZkMigrator
from resource_management.core.logger import Logger
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.format import format
@@ -105,6 +106,18 @@ class InfraSolr(Script):
action="delete"
)
+ def disable_security(self, env):
+ import params
+ if not params.infra_solr_znode:
+ Logger.info("Skipping reverting ACL")
+ return
+ zkmigrator = ZkMigrator(
+ zk_host=params.zookeeper_quorum,
+ java_exec=params.java_exec,
+ java_home=params.java64_home,
+ jaas_file=params.infra_solr_jaas_file,
+ user=params.infra_solr_user)
+ zkmigrator.set_acls(params.infra_solr_znode, 'world:anyone:crdwa')
if __name__ == "__main__":
InfraSolr().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py
index 946cf15..c2b8a44 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py
@@ -31,6 +31,16 @@ def get_port_from_url(address):
else:
return address
+def get_name_from_principal(principal):
+ if not principal: # return if empty
+ return principal
+ slash_split = principal.split('/')
+ if len(slash_split) == 2:
+ return slash_split[0]
+ else:
+ at_split = principal.split('@')
+ return at_split[0]
+
# config object that holds the configurations declared in the -site.xml file
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
@@ -50,6 +60,7 @@ fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_
# shared configs
java64_home = config['hostLevelParams']['java_home']
+java_exec = format("{java64_home}/bin/java")
zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts']
zookeeper_hosts_list.sort()
# get comma separated list of zookeeper hosts from clusterHostInfo
@@ -107,6 +118,7 @@ if security_enabled:
infra_solr_web_kerberos_keytab = config['configurations']['infra-solr-env']['infra_solr_web_kerberos_keytab']
infra_solr_web_kerberos_principal = config['configurations']['infra-solr-env']['infra_solr_web_kerberos_principal'].replace('_HOST',_hostname_lowercase)
infra_solr_kerberos_name_rules = config['configurations']['infra-solr-env']['infra_solr_kerberos_name_rules'].replace('$', '\$')
+ infra_solr_sasl_user = get_name_from_principal(infra_solr_kerberos_principal)
#Solr log4j
infra_log_maxfilesize = default('configurations/infra-solr-log4j/infra_log_maxfilesize',10)
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
index 740c560..46535cd 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
@@ -99,6 +99,15 @@ def setup_infra_solr(name = None):
secure=params.security_enabled
)
+ if params.security_enabled:
+ solr_cloud_util.secure_solr_znode(
+ zookeeper_quorum=params.zookeeper_quorum,
+ solr_znode=params.infra_solr_znode,
+ jaas_file=jaas_file,
+ java64_home=params.java64_home,
+ sasl_users_str=params.infra_solr_sasl_user
+ )
+
elif name == 'client':
solr_cloud_util.setup_solr_client(params.config)
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index e9ab98f..7f502d7 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -33,6 +33,16 @@ def get_port_from_url(address):
else:
return address
+def get_name_from_principal(principal):
+ if not principal: # return if empty
+ return principal
+ slash_split = principal.split('/')
+ if len(slash_split) == 2:
+ return slash_split[0]
+ else:
+ at_split = principal.split('@')
+ return at_split[0]
+
# config object that holds the configurations declared in the -site.xml file
config = Script.get_config()
@@ -148,8 +158,6 @@ else:
zookeeper_quorum = logsearch_solr_zk_quorum
-
-
# logsearch-env configs
logsearch_user = config['configurations']['logsearch-env']['logsearch_user']
logsearch_log_dir = config['configurations']['logsearch-env']['logsearch_log_dir']
@@ -229,6 +237,14 @@ logsearch_properties['logsearch.auth.simple.enabled'] = 'false'
logsearch_properties['logsearch.protocol'] = logsearch_ui_protocol
+logsearch_acls = ''
+if 'infra-solr-env' in config['configurations'] and security_enabled and not logsearch_use_external_solr:
+ acl_infra_solr_principal = get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal'])
+ acl_logsearch_principal = get_name_from_principal(config['configurations']['logsearch-env']['logsearch_kerberos_principal'])
+ logsearch_acls = format('world:anyone:r,sasl:{acl_infra_solr_principal}:cdrwa,sasl:{acl_logsearch_principal}:cdrwa')
+ logsearch_properties['logsearch.solr.zk.acls'] = logsearch_acls
+ logsearch_properties['logsearch.solr.audit.logs.zk.acls'] = logsearch_acls
+
# load config values
logsearch_properties = dict(logsearch_properties.items() + dict(config['configurations']['logsearch-properties']).items())
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py
index 30a1cda..c81268c 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/service_check.py
@@ -28,7 +28,7 @@ class LogSearchServiceCheck(Script):
try:
Execute(params.smoke_logsearch_cmd, user=params.logsearch_user,
- tries=10, try_sleep=5, timeout=10)
+ tries=15, try_sleep=5, timeout=10)
Logger.info('Log Search Server up and running')
except:
Logger.error('Log Search Server not running')
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 2690a3a..08d3d9d 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -17,11 +17,8 @@ limitations under the License.
"""
-from resource_management.core.exceptions import Fail
-from resource_management.libraries.functions import solr_cloud_util
from resource_management.core.resources.system import Directory, Execute, File
from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.decorator import retry
from resource_management.core.source import InlineTemplate, Template
from resource_management.libraries.resources.properties_file import PropertiesFile
@@ -119,42 +116,7 @@ def setup_logsearch():
content=Template("logsearch_jaas.conf.j2"),
owner=params.logsearch_user
)
- check_znode()
-
- upload_conf_set(format('{logsearch_solr_collection_service_logs}'),
- solrconfig_content=InlineTemplate(params.logsearch_service_logs_solrconfig_content)
- )
-
- upload_conf_set('history')
-
- upload_conf_set(format('{logsearch_solr_collection_audit_logs}'),
- solrconfig_content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content)
- )
Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")),
sudo=True
)
-
-
-def upload_conf_set(config_set, solrconfig_content = None):
- import params
- jaas_file = params.logsearch_jaas_file if params.logsearch_solr_kerberos_enabled else None
- solr_cloud_util.upload_configuration_to_zk(
- zookeeper_quorum=params.logsearch_solr_zk_quorum,
- solr_znode=params.logsearch_solr_zk_znode,
- config_set_dir=format("{logsearch_server_conf}/solr_configsets/{config_set}/conf"),
- config_set=config_set,
- tmp_dir=params.tmp_dir,
- java64_home=params.java64_home,
- solrconfig_content= solrconfig_content,
- jaas_file=jaas_file,
- retry=30, interval=5)
-
-@retry(times=30, sleep_time=5, err_class=Fail)
-def check_znode():
- import params
- solr_cloud_util.check_znode(
- zookeeper_quorum=params.logsearch_solr_zk_quorum,
- solr_znode=params.logsearch_solr_zk_znode,
- java64_home=params.java64_home,
- retry=30, interval=5)
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 7d501a7..e97759b 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -142,8 +142,7 @@ class TestMetadataServer(RMFTestCase):
mode=0755,
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content=InlineTemplate(self.getConfig()['configurations'][
- 'infra-solr-client-log4j']['content']),
+ content=self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode=0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
@@ -285,8 +284,7 @@ class TestMetadataServer(RMFTestCase):
mode=0755,
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content=InlineTemplate(self.getConfig()['configurations'][
- 'infra-solr-client-log4j']['content']),
+ content=self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode=0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index 33ac715..f63cd42 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -149,39 +149,6 @@ class TestLogSearch(RMFTestCase):
group='hadoop',
content = InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content'])
)
-
- self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_hadoop_logs_0.[0-9]* --config-set hadoop_logs --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_hadoop_logs_0.[0-9]*',
- content=InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content']),
- only_if='test -d /tmp/solr_config_hadoop_logs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_hadoop_logs_0.[0-9]* --config-set hadoop_logs --retry 30 --interval 5',
- only_if='test -d /tmp/solr_config_hadoop_logs_0.[0-9]*')
-
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf --config-set hadoop_logs --retry 30 --interval 5',
- not_if='test -d /tmp/solr_config_hadoop_logs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_hadoop_logs_0.[0-9]*',
- action=['delete'],
- create_parents=True)
-
-
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_history_0.[0-9]* --config-set history --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf --config-set history --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_history_0.[0-9]*',
- action=['delete'],
- create_parents=True)
-
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_audit_logs_0.[0-9]* --config-set audit_logs --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_audit_logs_0.[0-9]*',
- content=InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content']),
- only_if='test -d /tmp/solr_config_audit_logs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_audit_logs_0.[0-9]* --config-set audit_logs --retry 30 --interval 5',
- only_if='test -d /tmp/solr_config_audit_logs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf --config-set audit_logs --retry 30 --interval 5',
- not_if='test -d /tmp/solr_config_audit_logs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_audit_logs_0.[0-9]*',
- action=['delete'],
- create_parents=True)
self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'),
sudo = True
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index 4db31ca..bbaef78 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -140,8 +140,7 @@ class TestAtlasServer(RMFTestCase):
mode=0755
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content=InlineTemplate(self.getConfig()['configurations'][
- 'infra-solr-client-log4j']['content']),
+ content=self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode=0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py
index 8e959ad..b01e7da 100644
--- a/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py
+++ b/ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py
@@ -67,7 +67,7 @@ class TestRangerAdmin(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content = InlineTemplate(self.getConfig()['configurations']['infra-solr-client-log4j']['content']),
+ content = self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode = 0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
@@ -148,7 +148,7 @@ class TestRangerAdmin(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content = InlineTemplate(self.getConfig()['configurations']['infra-solr-client-log4j']['content']),
+ content = self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode = 0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
index 4510f4b..a1d930c 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
@@ -672,7 +672,7 @@
},
"infra-solr-client-log4j": {
"infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
- "content": "# Copyright 2011 The Apache Software Foundation\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlog4j.rootLogger=INFO,file,stdout,stderr\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender
.file.File={{logsearch_solr_client_log}}\nlog4j.appender.file.MaxFileSize=80MB\nlog4j.appender.file.MaxBackupIndex=60\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.Threshold=INFO\nlog4j.appender.stdout.Target=System.out\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=%m%n\n\nlog4j.appender.stderr=org.apache.log4j.ConsoleAppender\nlog4j.appender.stderr.Threshold=ERROR\nlog4j.appender.stderr.Target=System.err\nlog4j.appender.stderr.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stderr.layout.ConversionPattern=%m%n"
+ "content": "content"
},
"cluster-env": {
"security_enabled": "false",
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
index 9144f12..f959b1f 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
@@ -739,7 +739,7 @@
},
"infra-solr-client-log4j": {
"infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
- "content": "# Copyright 2011 The Apache Software Foundation\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlog4j.rootLogger=INFO,file,stdout,stderr\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender
.file.File={{logsearch_solr_client_log}}\nlog4j.appender.file.MaxFileSize=80MB\nlog4j.appender.file.MaxBackupIndex=60\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.Threshold=INFO\nlog4j.appender.stdout.Target=System.out\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=%m%n\n\nlog4j.appender.stderr=org.apache.log4j.ConsoleAppender\nlog4j.appender.stderr.Threshold=ERROR\nlog4j.appender.stderr.Target=System.err\nlog4j.appender.stderr.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stderr.layout.ConversionPattern=%m%n"
+ "content": "content"
},
"cluster-env": {
"security_enabled": "true",
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
index 927ee0c..8dda363 100644
--- a/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
+++ b/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
@@ -55,7 +55,7 @@ class TestRangerAdmin(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content = InlineTemplate(self.getConfig()['configurations']['infra-solr-client-log4j']['content']),
+ content = self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode = 0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
@@ -122,7 +122,7 @@ class TestRangerAdmin(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
- content = InlineTemplate(self.getConfig()['configurations']['infra-solr-client-log4j']['content']),
+ content = self.getConfig()['configurations']['infra-solr-client-log4j']['content'],
mode = 0644,
)
self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
index e517e3a..2c4815b 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
@@ -701,7 +701,7 @@
},
"infra-solr-client-log4j": {
"infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
- "content": "# Copyright 2011 The Apache Software Foundation\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlog4j.rootLogger=INFO,file,stdout,stderr\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender
.file.File={{logsearch_solr_client_log}}\nlog4j.appender.file.MaxFileSize=80MB\nlog4j.appender.file.MaxBackupIndex=60\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.Threshold=INFO\nlog4j.appender.stdout.Target=System.out\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=%m%n\n\nlog4j.appender.stderr=org.apache.log4j.ConsoleAppender\nlog4j.appender.stderr.Threshold=ERROR\nlog4j.appender.stderr.Target=System.err\nlog4j.appender.stderr.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stderr.layout.ConversionPattern=%m%n"
+ "content": "content"
},
"cluster-env": {
"security_enabled": "false",
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
index a04f9f8..fa791c1 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
@@ -768,7 +768,7 @@
},
"infra-solr-client-log4j": {
"infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
- "content": "# Copyright 2011 The Apache Software Foundation\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlog4j.rootLogger=INFO,file,stdout,stderr\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender
.file.File={{logsearch_solr_client_log}}\nlog4j.appender.file.MaxFileSize=80MB\nlog4j.appender.file.MaxBackupIndex=60\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.Threshold=INFO\nlog4j.appender.stdout.Target=System.out\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=%m%n\n\nlog4j.appender.stderr=org.apache.log4j.ConsoleAppender\nlog4j.appender.stderr.Threshold=ERROR\nlog4j.appender.stderr.Target=System.err\nlog4j.appender.stderr.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stderr.layout.ConversionPattern=%m%n"
+ "content": "content"
},
"cluster-env": {
"security_enabled": "true",
[2/3] ambari git commit: AMBARI-19468. Run ZK operations in logsearch
internally and enable ACL security (oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
deleted file mode 100644
index da76924..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.dao;
-
-import org.apache.ambari.logsearch.conf.SolrPropsConfig;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
-
-import javax.inject.Named;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-
-@Named
-class SolrCollectionDao {
-
- private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
-
- private static final int SETUP_RETRY_SECOND = 30;
- private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
- private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
-
- /**
- * This will try to get the collections from the Solr. Ping doesn't work if
- * collection is not given
- */
- boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
- int waitDurationMS = 3 * 60 * 1000;
- boolean status = false;
- try {
- long beginTimeMS = System.currentTimeMillis();
- long waitIntervalMS = 2000;
- int pingCount = 0;
- while (true) {
- pingCount++;
- try {
- List<String> collectionList = getCollections(cloudSolrClient);
- if (collectionList != null) {
- LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList);
- status = true;
- break;
- }
- } catch (Exception ex) {
- LOG.error("Error while doing Solr check", ex);
- }
- if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
- LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " +
- "If you are using alias, then you might have to restart LogSearch after Solr is up and running.");
- break;
- } else {
- LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " +
- "Will sleep for " + waitIntervalMS + " ms and try again.");
- }
- Thread.sleep(waitIntervalMS);
-
- }
- } catch (Throwable t) {
- LOG.error("Seems Solr is not up.");
- }
- return status;
- }
-
- void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
- boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
- LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus);
- if (!setupStatus) {
- // Start a background thread to do setup
- Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) {
- @Override
- public void run() {
- LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection());
- int retryCount = 0;
- while (true) {
- try {
- Thread.sleep(SETUP_RETRY_SECOND * 1000);
- retryCount++;
- boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
- if (setupStatus) {
- LOG.info("Setup for collection " + solrPropsConfig.getCollection() + " is successful. Exiting setup retry thread");
- break;
- }
- } catch (InterruptedException sleepInterrupted) {
- LOG.info("Sleep interrupted while setting up collection " + solrPropsConfig.getCollection());
- break;
- } catch (Exception e) {
- LOG.error("Error setting up collection=" + solrPropsConfig.getCollection(), e);
- }
- LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
- }
- }
- };
- setupThread.setDaemon(true);
- setupThread.start();
- }
- }
-
- private boolean createCollectionsIfNeeded(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
- boolean result = false;
- try {
- List<String> allCollectionList = getCollections(solrClient);
- if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
- result = createCollection(solrClient, solrPropsConfig, allCollectionList);
- } else {
- result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, allCollectionList);
- }
- } catch (Exception ex) {
- LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection(), ex);
- }
- return result;
- }
-
- @SuppressWarnings("unchecked")
- List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
- IOException {
- try {
- CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
- CollectionAdminResponse response = colListReq.process(solrClient);
- if (response.getStatus() != 0) {
- LOG.error("Error getting collection list from solr. response=" + response);
- return null;
- }
- return (List<String>) response.getResponse().get("collections");
- } catch (SolrException e) {
- LOG.error("getCollections() operation failed", e);
- return new ArrayList<>();
- }
- }
-
- private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
- throws Exception {
- LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
- + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
-
- // Default is true, because if the collection and shard is already there, then it will return true
- boolean returnValue = true;
-
- List<String> shardsList = new ArrayList<String>();
- for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
- shardsList.add("shard" + i);
- }
- String shardsListStr = StringUtils.join(shardsList, ',');
-
- // Check if collection is already in zookeeper
- if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
- LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
- CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
- collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
- collectionCreateRequest.setRouterName("implicit");
- collectionCreateRequest.setShards(shardsListStr);
- collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
- collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
- collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
- collectionCreateRequest.setRouterField(ROUTER_FIELD);
- collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
-
- CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- returnValue = false;
- LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
- + ", shardsList=" + shardsList +", response=" + createResponse);
- } else {
- LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
- }
- } else {
- LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
- Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
- Collection<String> existingShards = getShards(slices, solrPropsConfig);
- if (existingShards.size() < shardsList.size()) {
- try {
- updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
- } catch (Throwable t) {
- returnValue = false;
- LOG.error(String.format("Exception during updating collection (%s)", t));
- }
- }
- for (String shard : shardsList) {
- if (!existingShards.contains(shard)) {
- try {
- LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
- CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
- createShardRequest.setCollectionName(solrPropsConfig.getCollection());
- createShardRequest.setShardName(shard);
- CollectionAdminResponse response = createShardRequest.process(solrClient);
- if (response.getStatus() != 0) {
- LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
- returnValue = false;
- break;
- } else {
- LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
- }
- } catch (Throwable t) {
- LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
- returnValue = false;
- break;
- }
- }
- }
- }
- return returnValue;
- }
-
- private String getRandomBaseUrl(Collection<Slice> slices) {
- String coreUrl = null;
- if (slices != null) {
- for (Slice slice : slices) {
- if (!slice.getReplicas().isEmpty()) {
- Replica replica = slice.getReplicas().iterator().next();
- coreUrl = replica.getStr("base_url");
- if (coreUrl != null) {
- break;
- }
- }
- }
- }
- return coreUrl;
- }
-
- private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
- String baseUrl = getRandomBaseUrl(slices);
- if (baseUrl != null) {
- CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
- HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
- solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
- HttpResponse response = httpClient.execute(request);
- if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
- throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
- }
- } else {
- throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
- }
- }
-
- private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
- ZkStateReader reader = solrClient.getZkStateReader();
- return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
- }
-
- private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
- Collection<String> list = new HashSet<>();
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
- ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
- list.add(slice.getName());
- }
- }
- return list;
- }
-
- private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
-
- if (allCollectionList.contains(solrPropsConfig.getCollection())) {
- LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
- return true;
- }
-
- LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
- ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
-
- CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
- collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
- collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
- collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
- collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
- collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
- CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
- return false;
- } else {
- LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
- ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
- return true;
- }
- }
-
- private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
- return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 39b65ae..0568fd7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,6 +22,9 @@ package org.apache.ambari.logsearch.dao;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.log4j.Logger;
@@ -36,6 +39,7 @@ import org.springframework.data.solr.core.SolrCallback;
import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.core.query.SolrDataQuery;
+import javax.inject.Inject;
import java.io.IOException;
public abstract class SolrDaoBase {
@@ -44,6 +48,9 @@ public abstract class SolrDaoBase {
private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance");
private LogType logType;
+
+ @Inject
+ private SolrKerberosConfig solrKerberosConfig;
protected SolrDaoBase(LogType logType) {
this.logType = logType;
@@ -108,5 +115,14 @@ public abstract class SolrDaoBase {
public abstract SolrTemplate getSolrTemplate();
- public abstract SolrSchemaFieldDao getSolrSchemaFieldDao();
+ public abstract void setSolrTemplate(SolrTemplate solrTemplate);
+
+ public abstract SolrCollectionState getSolrCollectionState();
+
+ public abstract SolrPropsConfig getSolrPropsConfig();
+
+ public SolrKerberosConfig getSolrKerberosConfig() {
+ return this.solrKerberosConfig;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index e6a562e..01378c1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -43,7 +43,6 @@ import org.apache.solr.common.util.NamedList;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import java.io.IOException;
@@ -54,54 +53,30 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
-import javax.inject.Named;
public class SolrSchemaFieldDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
private static final int RETRY_SECOND = 30;
-
+
@Inject
- @Named("serviceSolrTemplate")
- private SolrTemplate serviceSolrTemplate;
+ private ServiceLogsSolrDao serviceLogsSolrDao;
@Inject
- @Named("auditSolrTemplate")
- private SolrTemplate auditSolrTemplate;
+ private AuditSolrDao auditSolrDao;
@Inject
private SolrUserPropsConfig solrUserConfigPropsConfig;
- private CloudSolrClient serviceSolrClient;
- private CloudSolrClient auditSolrClient;
-
private int retryCount;
private int skipCount;
-
- private boolean serviceCollectionSetUp = false;
- private boolean auditCollectionSetUp = false;
private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
-
- @PostConstruct
- public void init() {
- this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient();
- this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient();
- }
-
- void serviceCollectionSetUp() {
- this.serviceCollectionSetUp = true;
- }
-
- void auditCollectionSetUp() {
- this.auditCollectionSetUp = true;
- }
@Scheduled(fixedDelay = RETRY_SECOND * 1000)
public void populateAllSchemaFields() {
@@ -109,11 +84,12 @@ public class SolrSchemaFieldDao {
skipCount--;
return;
}
-
- if (serviceCollectionSetUp) {
+ if (serviceLogsSolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+ CloudSolrClient serviceSolrClient = (CloudSolrClient) serviceLogsSolrDao.getSolrTemplate().getSolrClient();
populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap);
}
- if (auditCollectionSetUp) {
+ if (auditSolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+ CloudSolrClient auditSolrClient = (CloudSolrClient) auditSolrDao.getSolrTemplate().getSolrClient();
populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
index 106bedc..13d7e03 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
@@ -32,7 +32,11 @@ import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.LogType;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.configurer.LogfeederFilterConfigurer;
+import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer;
import org.apache.ambari.logsearch.model.common.LogFeederDataMap;
import org.apache.ambari.logsearch.model.common.LogfeederFilterData;
import org.apache.solr.client.solrj.SolrQuery;
@@ -66,12 +70,13 @@ public class UserConfigSolrDao extends SolrDaoBase {
private SolrUserPropsConfig solrUserConfig;
@Inject
- private SolrCollectionDao solrCollectionDao;
-
- @Inject
@Named("userConfigSolrTemplate")
private SolrTemplate userConfigSolrTemplate;
+ @Inject
+ @Named("solrUserConfigState")
+ private SolrCollectionState solrUserConfigState;
+
public UserConfigSolrDao() {
super(LogType.SERVICE);
}
@@ -81,6 +86,11 @@ public class UserConfigSolrDao extends SolrDaoBase {
return userConfigSolrTemplate;
}
+ @Override
+ public void setSolrTemplate(SolrTemplate solrTemplate) {
+ this.userConfigSolrTemplate = solrTemplate;
+ }
+
@PostConstruct
public void postConstructor() {
String solrUrl = solrUserConfig.getSolrUrl();
@@ -88,24 +98,14 @@ public class UserConfigSolrDao extends SolrDaoBase {
String collection = solrUserConfig.getCollection();
try {
- solrCollectionDao.checkSolrStatus(getSolrClient());
- solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
- intializeLogFeederFilter();
-
+ new SolrCollectionConfigurer(this).start();
+ new LogfeederFilterConfigurer(this).start();
} catch (Exception e) {
LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString +
", collection=" + collection, e);
}
}
- private void intializeLogFeederFilter() {
- try {
- getUserFilter();
- } catch (SolrServerException | IOException e) {
- LOG.error("not able to save logfeeder filter while initialization", e);
- }
- }
-
public void saveUserFilter(LogFeederDataMap logfeederFilterWrapper) throws SolrException, SolrServerException, IOException {
String filterName = LogSearchConstants.LOGFEEDER_FILTER_NAME;
String json = JSONUtil.objToJson(logfeederFilterWrapper);
@@ -179,7 +179,12 @@ public class UserConfigSolrDao extends SolrDaoBase {
}
@Override
- public SolrSchemaFieldDao getSolrSchemaFieldDao() {
- throw new UnsupportedOperationException();
+ public SolrCollectionState getSolrCollectionState() {
+ return solrUserConfigState;
+ }
+
+ @Override
+ public SolrPropsConfig getSolrPropsConfig() {
+ return solrUserConfig;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 0fc9852..caf0636 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -115,4 +115,11 @@ public class DocConstants {
public static final String GET_USER_FILTER_BY_ID_OD = "Get user filter by id";
public static final String GET_ALL_USER_NAMES_OD = "Get all user names";
}
+
+ public class StatusOperationDescriptions {
+ public static final String STATUS_OD = "Get statuses for collections (not health state - show true if something already done)";
+ public static final String SERVICE_LOGS_STATUS_OD = "Get statuses for service log collection (not health state - show true if something already done)";
+ public static final String AUDIT_LOGS_STATUS_OD = "Get statuses for collections (not health state - show true if something already done)";
+ public static final String USER_CONFIG_STATUS_OD = "Get statuses for userconfig collection (not health state - show true if something already done)";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
new file mode 100644
index 0000000..fde176f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class ACLHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ACLHandler.class);
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ List<ACL> aclsToSetList = solrPropsConfig.getZkAcls();
+ if (CollectionUtils.isNotEmpty(aclsToSetList)) {
+ LOG.info("Setting acls for '{}' collection...", solrPropsConfig.getCollection());
+ SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
+ SolrZooKeeper solrZooKeeper = zkClient.getSolrZooKeeper();
+ String collectionPath = String.format("/collections/%s", solrPropsConfig.getCollection());
+ String configsPath = String.format("/configs/%s", solrPropsConfig.getConfigName());
+ List<ACL> collectionAcls = solrZooKeeper.getACL(collectionPath, new Stat());
+ if (isRefreshAclsNeeded(aclsToSetList, collectionAcls)) {
+ LOG.info("Acls differs for {}, update acls.", collectionPath);
+ setRecursivelyOn(solrZooKeeper, collectionPath, aclsToSetList);
+ }
+ List<ACL> configsAcls = solrZooKeeper.getACL(configsPath, new Stat());
+ if (isRefreshAclsNeeded(aclsToSetList, configsAcls)) {
+ LOG.info("Acls differs for {}, update acls.", configsPath);
+ setRecursivelyOn(solrZooKeeper, configsPath, aclsToSetList);
+ }
+ }
+ return true;
+ }
+
+ private boolean isRefreshAclsNeeded(List<ACL> acls, List<ACL> newAcls) {
+ boolean result = false;
+ if (acls != null) {
+ if (acls.size() != newAcls.size()) {
+ return true;
+ }
+ result = aclDiffers(acls, newAcls);
+ if (!result) {
+ result = aclDiffers(newAcls, acls);
+ }
+ }
+ return result;
+ }
+
+ private boolean aclDiffers(List<ACL> aclList1, List<ACL> aclList2) {
+ for (ACL acl : aclList1) {
+ for (ACL newAcl : aclList2) {
+ if (acl.getId() != null && acl.getId().getId().equals(newAcl.getId().getId())
+ && acl.getPerms() != newAcl.getPerms()) {
+ LOG.info("ACL for '{}' differs: '{}' on znode, should be '{}'",
+ acl.getId().getId(), acl.getPerms(), newAcl.getPerms());
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void setRecursivelyOn(SolrZooKeeper solrZooKeeper, String node, List<ACL> acls)
+ throws KeeperException, InterruptedException {
+ solrZooKeeper.setACL(node, acls, -1);
+ for (String child : solrZooKeeper.getChildren(node, null)) {
+ String path = node.endsWith("/") ? node + child : node + "/" + child;
+ setRecursivelyOn(solrZooKeeper, path, acls);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
new file mode 100644
index 0000000..752a1e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
+
+public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CreateCollectionHandler.class);
+
+ private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
+ private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
+
+ private List<String> allCollectionList;
+
+ public CreateCollectionHandler(List<String> allCollectionList) {
+ this.allCollectionList = allCollectionList;
+ }
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean result;
+ if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
+ result = createCollection(solrClient, solrPropsConfig, this.allCollectionList);
+ } else {
+ result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, this.allCollectionList);
+ }
+ return result;
+ }
+
+ private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
+ throws Exception {
+ LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
+ + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
+
+ // Default is true, because if the collection and shard is already there, then it will return true
+ boolean returnValue = true;
+
+ List<String> shardsList = new ArrayList<String>();
+ for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
+ shardsList.add("shard" + i);
+ }
+ String shardsListStr = StringUtils.join(shardsList, ',');
+
+ // Check if collection is already in zookeeper
+ if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
+ LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+ CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+ collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+ collectionCreateRequest.setRouterName("implicit");
+ collectionCreateRequest.setShards(shardsListStr);
+ collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+ collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+ collectionCreateRequest.setRouterField(ROUTER_FIELD);
+ collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
+
+ CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ returnValue = false;
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
+ + ", shardsList=" + shardsList +", response=" + createResponse);
+ } else {
+ LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+ }
+ } else {
+ LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
+ Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
+ Collection<String> existingShards = getShards(slices, solrPropsConfig);
+ if (existingShards.size() < shardsList.size()) {
+ try {
+ updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
+ } catch (Throwable t) {
+ returnValue = false;
+ LOG.error(String.format("Exception during updating collection (%s)", t));
+ }
+ }
+ for (String shard : shardsList) {
+ if (!existingShards.contains(shard)) {
+ try {
+ LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
+ CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
+ createShardRequest.setCollectionName(solrPropsConfig.getCollection());
+ createShardRequest.setShardName(shard);
+ CollectionAdminResponse response = createShardRequest.process(solrClient);
+ if (response.getStatus() != 0) {
+ LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
+ returnValue = false;
+ break;
+ } else {
+ LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
+ }
+ } catch (Throwable t) {
+ LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
+ returnValue = false;
+ break;
+ }
+ }
+ }
+ }
+ return returnValue;
+ }
+
+ private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
+
+ if (allCollectionList.contains(solrPropsConfig.getCollection())) {
+ LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
+ return true;
+ }
+
+ LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+ ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+
+ CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+ collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+ collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+ collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+ collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
+ CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
+ return false;
+ } else {
+ LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+ ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+ return true;
+ }
+ }
+
+ private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
+ String baseUrl = getRandomBaseUrl(slices);
+ if (baseUrl != null) {
+ CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
+ HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
+ solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
+ HttpResponse response = httpClient.execute(request);
+ if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
+ throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
+ }
+ } else {
+ throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
+ }
+ }
+
+ private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
+ ZkStateReader reader = solrClient.getZkStateReader();
+ return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
+ }
+
+ private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
+ Collection<String> list = new HashSet<>();
+ for (Slice slice : slices) {
+ for (Replica replica : slice.getReplicas()) {
+ LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
+ ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
+ list.add(slice.getName());
+ }
+ }
+ return list;
+ }
+
+ private String getRandomBaseUrl(Collection<Slice> slices) {
+ String coreUrl = null;
+ if (slices != null) {
+ for (Slice slice : slices) {
+ if (!slice.getReplicas().isEmpty()) {
+ Replica replica = slice.getReplicas().iterator().next();
+ coreUrl = replica.getStr("base_url");
+ if (coreUrl != null) {
+ break;
+ }
+ }
+ }
+ }
+ return coreUrl;
+ }
+
+ private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
+ return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
new file mode 100644
index 0000000..124ce40
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.SolrException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ListCollectionHandler implements SolrZkRequestHandler<List<String>> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ListCollectionHandler.class);
+
+ @Override
+ public List<String> handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ try {
+ CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
+ CollectionAdminResponse response = colListReq.process(solrClient);
+ if (response.getStatus() != 0) {
+ LOG.error("Error getting collection list from solr. response=" + response);
+ return null;
+ }
+ return (List<String>) response.getResponse().get("collections");
+ } catch (SolrException e) {
+ LOG.error("getCollections() operation failed", e);
+ return new ArrayList<>();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
new file mode 100644
index 0000000..52f3366
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReloadCollectionHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ReloadCollectionHandler.class);
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean result = false;
+ try {
+ LOG.info("Reload collection - '{}'", solrPropsConfig.getCollection());
+ CollectionAdminRequest.Reload reloadCollectionRequest = new CollectionAdminRequest.Reload();
+ reloadCollectionRequest.setCollectionName(solrPropsConfig.getCollection());
+ reloadCollectionRequest.process(solrClient);
+ result = true;
+ } catch (Exception e) {
+ LOG.error(String.format("Reload collection ('%s') failed.", solrPropsConfig.getCollection()), e);
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
new file mode 100644
index 0000000..85ae6cb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+
+interface SolrZkRequestHandler<T> {
+ T handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
new file mode 100644
index 0000000..23defea
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+public class UploadConfigurationHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(UploadConfigurationHandler.class);
+
+ private static final String SCHEMA_FILE = "managed-schema";
+ private static final String SOLR_CONFIG_FILE = "solrconfig.xml";
+
+ private File configSetFolder;
+
+ public UploadConfigurationHandler(File configSetFolder) {
+ this.configSetFolder = configSetFolder;
+ }
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean reloadCollectionNeeded = false;
+ String separator = FileSystems.getDefault().getSeparator();
+ String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator,
+ UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName());
+ solrClient.connect();
+ SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
+ File tmpDir = new File(downloadFolderLocation);
+ try {
+ ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
+ boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName());
+ if (configExists) {
+ LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection());
+ if (!tmpDir.mkdirs()) {
+ LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath());
+ }
+ zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation));
+ File[] listOfFiles = configSetFolder.listFiles();
+ if (listOfFiles != null) {
+ for (File file : listOfFiles) {
+ if (file.getName().equals(SOLR_CONFIG_FILE) || file.getName().equals(SCHEMA_FILE)) { // TODO: try to find an another solution to reload schema
+ if (!FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))){
+ LOG.info("One of the local solr config file differs ('{}'), upload config set to zookeeper", file.getName());
+ zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName());
+ reloadCollectionNeeded = true;
+ break;
+ }
+ }
+ }
+ }
+ } else {
+ LOG.info("Config set does not exist for '{}' collection. Uploading it to zookeeper...", solrPropsConfig.getCollection());
+ File[] listOfFiles = configSetFolder.listFiles();
+ if (listOfFiles != null) {
+ zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)",
+ solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e);
+ } finally {
+ if (tmpDir.exists()) {
+ try {
+ FileUtils.deleteDirectory(tmpDir);
+ } catch (IOException e){
+ LOG.error("Cannot delete temp directory.", e);
+ }
+ }
+ }
+ return reloadCollectionNeeded;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
new file mode 100644
index 0000000..8cc6b94
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.rest;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.springframework.context.annotation.Scope;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.AUDIT_LOGS_STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.SERVICE_LOGS_STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.USER_CONFIG_STATUS_OD;
+
+@Api(value = "status", description = "Status Operations")
+@Path("status")
+@Named
+@Scope("request")
+public class StatusResource {
+
+ @Inject
+ @Named("solrServiceLogsState")
+ private SolrCollectionState solrServiceLogsState;
+
+ @Inject
+ @Named("solrAuditLogsState")
+ private SolrCollectionState solrAuditLogsState;
+
+ @Inject
+ @Named("solrUserConfigState")
+ private SolrCollectionState solrUserConfigState;
+
+ @GET
+ @Produces({"application/json"})
+ @ApiOperation(STATUS_OD)
+ public Map<String, SolrCollectionState> getStatus() {
+ Map<String, SolrCollectionState> response = new HashMap<>();
+ response.put("serviceLogs", solrServiceLogsState);
+ response.put("auditLogs", solrAuditLogsState);
+ response.put("userConfig", solrUserConfigState);
+ return response;
+ }
+
+ @GET
+ @Path("/servicelogs")
+ @Produces({"application/json"})
+ @ApiOperation(SERVICE_LOGS_STATUS_OD)
+ public SolrCollectionState getServiceLogStatus() {
+ return solrServiceLogsState;
+ }
+
+ @GET
+ @Path("/auditlogs")
+ @Produces({"application/json"})
+ @ApiOperation(AUDIT_LOGS_STATUS_OD)
+ public SolrCollectionState getSolrAuditLogsStatus() {
+ return solrAuditLogsState;
+ }
+
+ @GET
+ @Path("/userconfig")
+ @Produces({"application/json"})
+ @ApiOperation(USER_CONFIG_STATUS_OD)
+ public SolrCollectionState getSolrUserConfigStatus() {
+ return solrUserConfigState;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
index 532428b..b1a53f2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
@@ -41,14 +41,18 @@ public class RESTErrorUtil {
return createRESTException(response, HttpServletResponse.SC_BAD_REQUEST);
}
- public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) {
- List<MessageData> messageList = new ArrayList<MessageData>();
+ public static VResponse createMessageResponse(String errorMessage, MessageEnums messageEnum) {
+ List<MessageData> messageList = new ArrayList<>();
messageList.add(messageEnum.getMessage());
-
VResponse response = new VResponse();
response.setStatusCode(VResponse.STATUS_ERROR);
response.setMsgDesc(errorMessage);
response.setMessageList(messageList);
+ return response;
+ }
+
+ public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) {
+ VResponse response = createMessageResponse(errorMessage, messageEnum);
WebApplicationException webAppEx = createRESTException(response);
logger.error("Operation error. response=" + response, webAppEx);
return webAppEx;
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
new file mode 100644
index 0000000..4d529fb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+abstract public class AbstractLogsearchGlobalStateFilter implements Filter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractLogsearchGlobalStateFilter.class);
+
+ protected static final String ZNODE_NOT_READY_MSG = "ZNode is not available for %s. (connection string: %s, endpoint: %s)";
+ protected static final String ZK_CONFIG_NOT_READY_MSG = "Collection configuration has not uploaded yet for %s. (configuration name: %s, collection name: %s, endpoint: %s)";
+ protected static final String SOLR_COLLECTION_NOT_READY_MSG = "Solr has not accessible yet for %s collection. (endpoint: %s)";
+
+ private SolrCollectionState solrCollectionState;
+ private RequestMatcher requestMatcher;
+ private SolrPropsConfig solrPropsConfig;
+
+ public AbstractLogsearchGlobalStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ this.requestMatcher = requestMatcher;
+ this.solrCollectionState = state;
+ this.solrPropsConfig = solrPropsConfig;
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+ HttpServletRequest request = (HttpServletRequest) servletRequest;
+ if (requestMatcher.matches(request)) {
+ VResponse errorResponse = getErrorResponse(solrCollectionState, solrPropsConfig, request);
+ if (errorResponse != null) {
+ LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc());
+ HttpServletResponse resp = (HttpServletResponse) servletResponse;
+ resp.setStatus(500);
+ resp.setContentType("application/json");
+ resp.getWriter().print(createStringFromErrorMessageObject(errorResponse));
+ return;
+ }
+ }
+ filterChain.doFilter(servletRequest, servletResponse);
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ /**
+ * Fill the error message with data in case of any condition based on collection state.
+ * If return value is null, that means there were no any errors
+ * @param solrCollectionState object to store solr state (e.g. : zookeeper/solr collection availability)
+ * @param solrPropsConfig object to store details for solr collection (e.g. : collection name, connection string)
+ */
+ public abstract VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request);
+
+ private String createStringFromErrorMessageObject(VResponse responseObject) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(responseObject);
+ } catch (Exception e) {
+ throw RESTErrorUtil.createRESTException("Cannot parse response object on backend", MessageEnums.ERROR_CREATING_OBJECT);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
new file mode 100644
index 0000000..117fdd4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchAuditLogsStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+ public LogsearchAuditLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ super(requestMatcher, state, solrPropsConfig);
+ }
+
+ @Override
+ public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (!solrCollectionState.isZnodeReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+ "audit logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+ } else if (!solrCollectionState.isConfigurationUploaded()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "audit logs",
+ solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+ } else if (!solrCollectionState.isSolrCollectionReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+ solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
new file mode 100644
index 0000000..74b30e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchServiceLogsStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+ public LogsearchServiceLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ super(requestMatcher, state, solrPropsConfig);
+ }
+
+ @Override
+ public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (!solrCollectionState.isZnodeReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+ "service logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+ } else if (!solrCollectionState.isConfigurationUploaded()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "service logs",
+ solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+ } else if (!solrCollectionState.isSolrCollectionReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+ solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
new file mode 100644
index 0000000..037bed0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchUserConfigStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+
+ public LogsearchUserConfigStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ super(requestMatcher, state, solrPropsConfig);
+ }
+
+ @Override
+ public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (!solrCollectionState.isZnodeReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+ "userconfig", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+ } else if (!solrCollectionState.isConfigurationUploaded()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "userconfig",
+ solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+ } else if (!solrCollectionState.isSolrCollectionReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+ solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
index 85c8245..15322ef 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
@@ -411,8 +411,15 @@ define(['require',
try {
if (!errorShown) {
errorShown = true;
+ var errorMessage = "Some issues on server, Please try again later."
+ if (error != null && error.responseText != null) {
+ var errorObj = JSON.parse(error.responseText);
+ if (errorObj.hasOwnProperty('msgDesc')) {
+ errorMessage = errorObj.msgDesc;
+ }
+ }
Utils.notifyError({
- content: "Some issue on server, Please try again later."
+ content: errorMessage
});
setTimeout(function() {
errorShown = false;
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
index fa6162a..6bb5a77 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
@@ -20,7 +20,6 @@ package org.apache.ambari.logsearch.solr;
import org.apache.ambari.logsearch.solr.commands.CheckConfigZkCommand;
import org.apache.ambari.logsearch.solr.commands.CreateCollectionCommand;
-import org.apache.ambari.logsearch.solr.commands.CreateSaslUsersZkCommand;
import org.apache.ambari.logsearch.solr.commands.CreateShardCommand;
import org.apache.ambari.logsearch.solr.commands.CreateSolrZnodeZkCommand;
import org.apache.ambari.logsearch.solr.commands.DownloadConfigZkCommand;
@@ -34,7 +33,6 @@ import org.apache.ambari.logsearch.solr.commands.SetClusterPropertyZkCommand;
import org.apache.ambari.logsearch.solr.commands.UploadConfigZkCommand;
import org.apache.ambari.logsearch.solr.commands.CheckZnodeZkCommand;
import org.apache.ambari.logsearch.solr.util.ShardUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -129,18 +127,6 @@ public class AmbariSolrCloudClient {
}
/**
- * Add sasl user (only if not exist on the znode).
- */
- public String addSaslUsers() throws Exception {
- LOG.info("Add sasl user to znode: {}", this.saslUsers);
- String newUsers = new CreateSaslUsersZkCommand(getRetryTimes(), getInterval()).run(this);
- if (StringUtils.isNotEmpty(newUsers)) {
- LOG.info("New sasl users added to znode: {}", newUsers);
- }
- return newUsers;
- }
-
- /**
* Create a znode only if it does not exist. Return 0 code if it exists.
*/
public void createZnode() throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
deleted file mode 100644
index b144b41..0000000
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.solr.commands;
-
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.util.AclUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.SolrZooKeeper;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
-import org.apache.zookeeper.data.Stat;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-public class CreateSaslUsersZkCommand extends AbstractZookeeperRetryCommand<String> {
-
- public CreateSaslUsersZkCommand(int maxRetries, int interval) {
- super(maxRetries, interval);
- }
-
- @Override
- protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception {
- List<ACL> acls = solrZooKeeper.getACL(client.getZnode(), new Stat());
- List<String> newUsers = new ArrayList<>();
- Set<String> existingUsers = AclUtils.getUsersFromAclData(acls);
- String saslUsers = client.getSaslUsers();
- if (StringUtils.isNotEmpty(saslUsers)) {
- String[] saslUserNames = saslUsers.split(",");
- for (String saslUser : saslUserNames) {
- if (!existingUsers.contains(saslUser)) {
- acls.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", saslUser)));
- newUsers.add(saslUser);
- }
- }
- }
- acls = AclUtils.updatePermissionForScheme(acls, "world", ZooDefs.Perms.READ);
- solrZooKeeper.setACL(client.getZnode(), acls, -1);
- return StringUtils.join(newUsers, ",");
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
index faa21d4..31ad5d3 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
@@ -23,6 +23,8 @@ import org.apache.ambari.logsearch.solr.util.AclUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
@@ -52,9 +54,13 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool
String configsPath = String.format("%s/%s", zNode, "configs");
String collectionsPath = String.format("%s/%s", zNode, "collections");
- List<String> exlustePaths = Arrays.asList(configsPath, collectionsPath);
+ String aliasesPath = String.format("%s/%s", zNode, "aliases.json"); // TODO: protect this later somehow
+ List<String> excludePaths = Arrays.asList(configsPath, collectionsPath, aliasesPath);
- AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, exlustePaths);
+ createZnodeIfNeeded(configsPath, client.getSolrZkClient());
+ createZnodeIfNeeded(collectionsPath, client.getSolrZkClient());
+
+ AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, excludePaths);
List<ACL> commonConfigAcls = new ArrayList<>();
commonConfigAcls.addAll(saslUserList);
@@ -71,4 +77,11 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool
return true;
}
+
+ private void createZnodeIfNeeded(String configsPath, SolrZkClient zkClient) throws KeeperException, InterruptedException {
+ if (!zkClient.exists(configsPath, true)) {
+ LOG.info("'{}' does not exist. Creating it ...", configsPath);
+ zkClient.makePath(configsPath, true);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
index 8e0132d..fc25c49 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
@@ -34,36 +34,6 @@ import java.util.Set;
public class AclUtils {
- public static Set<String> getUsersFromAclData(List<ACL> acls) {
- Set<String> result = new HashSet<>();
- if (!acls.isEmpty()) {
- for (ACL acl : acls) {
- String username = "";
- String id = acl.getId().getId();
- String[] splitted = id.split(":");
- if (splitted.length > 1) {
- username = splitted[0];
- } else {
- username = id;
- }
- result.add(username);
- }
- }
- return result;
- }
-
- public static List<ACL> updatePermissionForScheme(List<ACL> acls, String scheme, int permission) {
- List<ACL> aclResult = new ArrayList<>();
- if (!acls.isEmpty()) {
- for (ACL acl : acls) {
- int permissionToAdd = scheme.equals(acl.getId().getScheme()) ? permission : acl.getPerms();
- acl.setPerms(permissionToAdd);
- aclResult.add(acl);
- }
- }
- return aclResult;
- }
-
public static List<ACL> mergeAcls(List<ACL> originalAcls, List<ACL> updateAcls) {
Map<String, ACL> aclMap = new HashMap<>();
List<ACL> acls = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/docker/bin/start.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh
index 81c9e66..4c60981 100644
--- a/ambari-logsearch/docker/bin/start.sh
+++ b/ambari-logsearch/docker/bin/start.sh
@@ -81,12 +81,6 @@ function start_solr() {
}
function start_logsearch() {
- echo "Upload configuration sets ..."
-
- $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/audit_logs/conf -confname audit_logs
- $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/hadoop_logs/conf -confname hadoop_logs
- $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/history/conf -confname history
-
$LOGSEARCH_SERVER_PATH/run.sh
touch /var/log/ambari-logsearch-portal/logsearch-app.log
}
[3/3] ambari git commit: AMBARI-19468. Run ZK operations in logsearch
internally and enable ACL security (oleewere)
Posted by ol...@apache.org.
AMBARI-19468. Run ZK operations in logsearch internally and enable ACL security (oleewere)
Change-Id: Ida845767ee34c4f58e9b791cd6b131e22e0439f6
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/548a0b6a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/548a0b6a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/548a0b6a
Branch: refs/heads/branch-2.5
Commit: 548a0b6a31c486a6f24c0d431be0d1a247b72250
Parents: e382df2
Author: oleewere <ol...@gmail.com>
Authored: Wed Jan 11 16:20:44 2017 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Thu Jan 19 14:13:04 2017 +0100
----------------------------------------------------------------------
.../libraries/functions/solr_cloud_util.py | 22 +-
.../ambari-logsearch-portal/pom.xml | 13 +-
.../configsets/audit_logs/conf/managed-schema | 100 +++---
.../logsearch/common/ACLPropertiesSplitter.java | 70 +++++
.../ambari/logsearch/common/MessageEnums.java | 3 +
.../ambari/logsearch/conf/SecurityConfig.java | 57 +++-
.../logsearch/conf/SolrAuditLogPropsConfig.java | 27 ++
.../ambari/logsearch/conf/SolrConfig.java | 71 ++---
.../conf/SolrConnectionPropsConfig.java | 29 ++
.../ambari/logsearch/conf/SolrPropsConfig.java | 12 +
.../conf/global/SolrAuditLogsState.java | 68 ++++
.../conf/global/SolrCollectionState.java | 33 ++
.../conf/global/SolrServiceLogsState.java | 59 ++++
.../conf/global/SolrUserConfigState.java | 60 ++++
.../configurer/LogfeederFilterConfigurer.java | 66 ++++
.../configurer/SolrAuditAliasConfigurer.java | 136 ++++++++
.../configurer/SolrCollectionConfigurer.java | 230 ++++++++++++++
.../logsearch/configurer/SolrConfigurer.java | 23 ++
.../ambari/logsearch/dao/AuditSolrDao.java | 38 ++-
.../logsearch/dao/ServiceLogsSolrDao.java | 30 +-
.../ambari/logsearch/dao/SolrAliasDao.java | 121 -------
.../ambari/logsearch/dao/SolrCollectionDao.java | 313 -------------------
.../ambari/logsearch/dao/SolrDaoBase.java | 18 +-
.../logsearch/dao/SolrSchemaFieldDao.java | 38 +--
.../ambari/logsearch/dao/UserConfigSolrDao.java | 39 ++-
.../ambari/logsearch/doc/DocConstants.java | 7 +
.../ambari/logsearch/handler/ACLHandler.java | 97 ++++++
.../handler/CreateCollectionHandler.java | 222 +++++++++++++
.../handler/ListCollectionHandler.java | 51 +++
.../handler/ReloadCollectionHandler.java | 45 +++
.../logsearch/handler/SolrZkRequestHandler.java | 26 ++
.../handler/UploadConfigurationHandler.java | 100 ++++++
.../ambari/logsearch/rest/StatusResource.java | 91 ++++++
.../ambari/logsearch/util/RESTErrorUtil.java | 10 +-
.../AbstractLogsearchGlobalStateFilter.java | 100 ++++++
.../filters/LogsearchAuditLogsStateFilter.java | 51 +++
.../LogsearchServiceLogsStateFilter.java | 51 +++
.../filters/LogsearchUserConfigStateFilter.java | 52 +++
.../src/main/webapp/scripts/utils/Utils.js | 9 +-
.../logsearch/solr/AmbariSolrCloudClient.java | 14 -
.../solr/commands/CreateSaslUsersZkCommand.java | 60 ----
.../solr/commands/SecureSolrZNodeZkCommand.java | 17 +-
.../ambari/logsearch/solr/util/AclUtils.java | 30 --
ambari-logsearch/docker/bin/start.sh | 6 -
.../logsearch/logsearch-https.properties | 3 +
.../test-config/logsearch/logsearch.properties | 3 +
.../0.1.0/package/scripts/infra_solr.py | 13 +
.../0.1.0/package/scripts/params.py | 12 +
.../0.1.0/package/scripts/setup_infra_solr.py | 9 +
.../LOGSEARCH/0.5.0/package/scripts/params.py | 20 +-
.../0.5.0/package/scripts/service_check.py | 2 +-
.../0.5.0/package/scripts/setup_logsearch.py | 38 ---
.../stacks/2.3/ATLAS/test_metadata_server.py | 6 +-
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 33 --
.../stacks/2.5/ATLAS/test_atlas_server.py | 3 +-
.../stacks/2.5/RANGER/test_ranger_admin.py | 4 +-
.../2.5/configs/ranger-admin-default.json | 2 +-
.../2.5/configs/ranger-admin-secured.json | 2 +-
.../stacks/2.6/RANGER/test_ranger_admin.py | 4 +-
.../2.6/configs/ranger-admin-default.json | 2 +-
.../2.6/configs/ranger-admin-secured.json | 2 +-
61 files changed, 2052 insertions(+), 821 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
index 2e48c6a..3e076b8 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
@@ -18,13 +18,14 @@ limitations under the License.
"""
import random
from ambari_commons.constants import AMBARI_SUDO_BINARY
+from ambari_jinja2 import Environment as JinjaEnvironment
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.core.resources.system import Directory, Execute, File
-from resource_management.core.source import StaticFile, InlineTemplate
+from resource_management.core.source import StaticFile
__all__ = ["upload_configuration_to_zk", "create_collection", "setup_kerberos", "set_cluster_prop",
- "setup_kerberos_plugin", "create_znode", "check_znode", "create_sasl_users"]
+ "setup_kerberos_plugin", "create_znode", "check_znode", "secure_solr_znode", "secure_znode"]
def __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, separated_znode=False):
sudo = AMBARI_SUDO_BINARY
@@ -172,12 +173,12 @@ def secure_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_user
Execute(secure_znode_cmd)
-def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users=[]):
+def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users_str=''):
"""
Secure solr znode - setup acls to 'cdrwa' for solr user, set 'r' only for the world, skipping /znode/configs and znode/collections (set those to 'cr' for the world)
+ sasl_users_str: comma separated sasl users
"""
solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
- sasl_users_str = ",".join(str(x) for x in sasl_users)
secure_solr_znode_cmd = format('{solr_cli_prefix} --secure-solr-znode --jaas-file {jaas_file} --sasl-users {sasl_users_str}')
Execute(secure_solr_znode_cmd)
@@ -218,8 +219,19 @@ def setup_solr_client(config, custom_log4j = True, custom_log_location = None, l
if custom_log4j:
# use custom log4j content only, when infra is not installed on the cluster
solr_client_log4j_content = config['configurations']['infra-solr-client-log4j']['content'] if log4jcontent is None else log4jcontent
+ context = {
+ 'solr_client_log': solr_client_log,
+ 'solr_client_log_maxfilesize': solr_client_log_maxfilesize,
+ 'solr_client_log_maxbackupindex': solr_client_log_maxbackupindex
+ }
+ template = JinjaEnvironment(
+ line_statement_prefix='%',
+ variable_start_string="{{",
+ variable_end_string="}}")\
+ .from_string(solr_client_log4j_content)
+
File(format("{solr_client_dir}/log4j.properties"),
- content=InlineTemplate(solr_client_log4j_content),
+ content=template.render(context),
mode=0644
)
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 9d17fe4..4bba464 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -642,11 +642,6 @@
</dependency>
<dependency>
<groupId>org.apache.ambari</groupId>
- <artifactId>ambari-logsearch-solr-client</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.ambari</groupId>
<artifactId>ambari-metrics-common</artifactId>
<version>${project.version}</version>
</dependency>
@@ -656,6 +651,14 @@
<version>1.3.1</version>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.springframework.security.kerberos</groupId>
<artifactId>spring-security-kerberos-core</artifactId>
<version>1.0.1.RELEASE</version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
index 4cd412b..3f7fe3d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema
@@ -15,30 +15,37 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+<!-- Solr managed schema - automatically generated - DO NOT EDIT -->
<schema name="audit-logs-schema" version="1.5">
<uniqueKey>id</uniqueKey>
<fieldType name="binary" class="solr.BinaryField"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
- <fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/>
- <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/>
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="key_lower_case" class="solr.TextField" multiValued="false" sortMissingLast="true" omitNorms="true">
+ <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
+ <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/>
+ <fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="key_lower_case" class="solr.TextField" omitNorms="true" sortMissingLast="true" multiValued="false">
<analyzer>
<tokenizer class="solr.KeywordTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="n_gram" class="solr.TextField" omitNorms="true" sortMissingLast="true">
+ <analyzer>
+ <tokenizer class="solr.NGramTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
<fieldType name="random" class="solr.RandomSortField" indexed="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
- <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
- <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" multiValued="true" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
- <fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true" positionIncrementGap="100">
+ <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/>
+ <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
@@ -50,45 +57,54 @@
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
- <fieldType name="n_gram" class="solr.TextField" sortMissingLast="true" omitNorms="true">
- <analyzer>
- <tokenizer class="solr.NGramTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- </analyzer>
- </fieldType>
-
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<field name="_expire_at_" type="tdate" multiValued="false" stored="true"/>
+ <field name="_router_field_" type="int" multiValued="false" indexed="false" stored="false"/>
<field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/>
<field name="_version_" type="long" indexed="true" stored="true"/>
- <field name="_router_field_" type="int" indexed="false" stored="false" multiValued="false"/>
-
<field name="access" type="key_lower_case" multiValued="false"/>
<field name="action" type="key_lower_case" multiValued="false"/>
<field name="agent" type="key_lower_case" multiValued="false"/>
<field name="agentHost" type="key_lower_case" multiValued="false"/>
+ <field name="authType" type="key_lower_case"/>
+ <field name="bundle_id" type="key_lower_case" multiValued="false"/>
+ <field name="case_id" type="key_lower_case" multiValued="false"/>
<field name="cliIP" type="key_lower_case" multiValued="false"/>
<field name="cliType" type="key_lower_case" multiValued="false"/>
<field name="cluster" type="key_lower_case" multiValued="false"/>
- <field name="reqContext" type="key_lower_case" multiValued="true"/>
+ <field name="dst" type="key_lower_case"/>
<field name="enforcer" type="key_lower_case" multiValued="false"/>
- <field name="event_count" type="tlong" multiValued="false" docValues="true" default="1"/>
+ <field name="event_count" type="tlong" default="1" docValues="true" multiValued="false"/>
<field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
- <field name="evtTime" type="tdate" docValues="true"/>
+ <field name="event_md5" type="string" multiValued="false"/>
+ <field name="evtTime" type="tdate" docValues="true"/>
+ <field name="file" type="key_lower_case" multiValued="false"/>
+ <field name="host" type="key_lower_case"/>
<field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
+ <field name="ip" type="key_lower_case"/>
+ <field name="level" type="key_lower_case"/>
<field name="logType" type="key_lower_case" multiValued="false"/>
+ <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/>
+ <field name="logfile_line_number" type="tint" omitNorms="false"/>
+ <field name="logger_name" type="key_lower_case"/>
+ <field name="message_md5" type="string" multiValued="false"/>
+ <field name="path" type="key_lower_case"/>
+ <field name="perm" type="key_lower_case"/>
<field name="policy" type="tlong" docValues="true"/>
<field name="proxyUsers" type="key_lower_case" multiValued="true"/>
<field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
<field name="repo" type="key_lower_case" multiValued="false"/>
<field name="repoType" type="tint" multiValued="false" docValues="true"/>
+ <field name="reqContext" type="key_lower_case" multiValued="true"/>
<field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
<field name="reqUser" type="key_lower_case" multiValued="false"/>
+ <field name="req_caller_id" type="key_lower_case" multiValued="false"/>
+ <field name="req_self_id" type="key_lower_case" multiValued="false"/>
<field name="resType" type="key_lower_case" multiValued="false"/>
<field name="resource" type="key_lower_case" multiValued="false"/>
<field name="result" type="tint" multiValued="false"/>
@@ -96,22 +112,14 @@
<field name="sess" type="key_lower_case" multiValued="false"/>
<field name="tags" type="key_lower_case" multiValued="true"/>
<field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
+ <field name="task_id" type="tlong"/>
<field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>
-
- <field name="bundle_id" type="key_lower_case" multiValued="false"/>
- <field name="case_id" type="key_lower_case" multiValued="false"/>
- <field name="req_caller_id" type="key_lower_case" multiValued="false"/>
- <field name="req_self_id" type="key_lower_case" multiValued="false"/>
- <field name="event_md5" type="string" multiValued="false"/>
- <field name="file" type="key_lower_case" multiValued="false"/>
- <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/>
- <field name="logfile_line_number" type="tint" omitNorms="false"/>
- <field name="message_md5" type="string" multiValued="false"/>
<field name="type" type="key_lower_case" multiValued="false"/>
-
- <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="true"/>
- <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="true"/>
- <dynamicField name='key_*' type="key_lower_case" multiValued="false" stored="true"/>
- <dynamicField name="ws_*" type="text_ws" multiValued="false" omitNorms="false" stored="true"/>
+ <field name="ugi" type="key_lower_case"/>
+
+ <dynamicField name="ngram_*" type="n_gram" multiValued="false" stored="true"/>
+ <dynamicField name="std_*" type="text_std_token_lower_case" multiValued="false" stored="true"/>
+ <dynamicField name="key_*" type="key_lower_case" multiValued="false" stored="true"/>
+ <dynamicField name="ws_*" type="text_ws" omitNorms="false" multiValued="false" stored="true"/>
</schema>
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java
new file mode 100644
index 0000000..35a2b1b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ACLPropertiesSplitter.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.common;
+
+import com.google.common.base.Splitter;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
+import javax.inject.Named;
+import java.util.ArrayList;
+import java.util.List;
+
+@Named
+public class ACLPropertiesSplitter {
+
+ public List<ACL> parseAcls(String aclStr) {
+ List<ACL> acls = new ArrayList<>();
+ List<String> aclStrList = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(aclStr);
+ for (String unparcedAcl : aclStrList) {
+ String[] parts = unparcedAcl.split(":");
+ if (parts.length == 3) {
+ acls.add(new ACL(parsePermission(parts[2]), new Id(parts[0], parts[1])));
+ }
+ }
+ return acls;
+ }
+
+ private Integer parsePermission(String permission) {
+ int permissionCode = 0;
+ for (char each : permission.toLowerCase().toCharArray()) {
+ switch (each) {
+ case 'r':
+ permissionCode |= ZooDefs.Perms.READ;
+ break;
+ case 'w':
+ permissionCode |= ZooDefs.Perms.WRITE;
+ break;
+ case 'c':
+ permissionCode |= ZooDefs.Perms.CREATE;
+ break;
+ case 'd':
+ permissionCode |= ZooDefs.Perms.DELETE;
+ break;
+ case 'a':
+ permissionCode |= ZooDefs.Perms.ADMIN;
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported permission: " + permission);
+ }
+ }
+ return permissionCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
index c0a7f32..4f1725f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
@@ -30,6 +30,9 @@ public enum MessageEnums {
ERROR_DUPLICATE_OBJECT("logsearch.error.duplicate_object", "Error creating duplicate object"),
ERROR_SYSTEM("logsearch.error.system", "System Error. Please try later."),
SOLR_ERROR("logsearch.solr.error","Something went wrong, For more details check the logs or configuration."),
+ ZNODE_NOT_READY("logsearch.zk.znode.error", "ZNode is not available."),
+ ZK_CONFIG_NOT_READY("logsearch.zk.config.error", "Collection configuration has not uploaded yet"),
+ SOLR_COLLECTION_NOT_READY("logsearch.solr.collection.error", "Solr has not accessible yet for collection."),
// Common Validations
INVALID_PASSWORD("logsearch.validation.invalid_password", "Invalid password"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
index c99a738..115dcc3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
@@ -19,13 +19,17 @@
package org.apache.ambari.logsearch.conf;
import com.google.common.collect.Lists;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler;
import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler;
import org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler;
+import org.apache.ambari.logsearch.web.filters.LogsearchAuditLogsStateFilter;
import org.apache.ambari.logsearch.web.filters.LogsearchAuthenticationEntryPoint;
import org.apache.ambari.logsearch.web.filters.LogsearchKRBAuthenticationFilter;
import org.apache.ambari.logsearch.web.filters.LogsearchJWTFilter;
import org.apache.ambari.logsearch.web.filters.LogsearchSecurityContextFormationFilter;
+import org.apache.ambari.logsearch.web.filters.LogsearchServiceLogsStateFilter;
+import org.apache.ambari.logsearch.web.filters.LogsearchUserConfigStateFilter;
import org.apache.ambari.logsearch.web.filters.LogsearchUsernamePasswordAuthenticationFilter;
import org.apache.ambari.logsearch.web.security.LogsearchAuthenticationProvider;
import org.springframework.context.annotation.Bean;
@@ -37,11 +41,11 @@ import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.web.access.intercept.FilterSecurityInterceptor;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
-import org.springframework.security.web.util.matcher.NegatedRequestMatcher;
import org.springframework.security.web.util.matcher.OrRequestMatcher;
import org.springframework.security.web.util.matcher.RequestMatcher;
import javax.inject.Inject;
+import javax.inject.Named;
import java.util.List;
import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_SESSION_ID;
@@ -53,6 +57,27 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Inject
private AuthPropsConfig authPropsConfig;
+ @Inject
+ private SolrServiceLogPropsConfig solrServiceLogPropsConfig;
+
+ @Inject
+ private SolrAuditLogPropsConfig solrAuditLogPropsConfig;
+
+ @Inject
+ private SolrUserPropsConfig solrUserPropsConfig;
+
+ @Inject
+ @Named("solrServiceLogsState")
+ private SolrCollectionState solrServiceLogsState;
+
+ @Inject
+ @Named("solrAuditLogsState")
+ private SolrCollectionState solrAuditLogsState;
+
+ @Inject
+ @Named("solrUserConfigState")
+ private SolrCollectionState solrUserConfigState;
+
@Override
protected void configure(HttpSecurity http) throws Exception {
http
@@ -76,6 +101,9 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
.addFilterBefore(logsearchUsernamePasswordAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(logsearchKRBAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterAfter(securityContextFormationFilter(), FilterSecurityInterceptor.class)
+ .addFilterAfter(logsearchUserConfigFilter(), LogsearchSecurityContextFormationFilter.class)
+ .addFilterAfter(logsearchAuditLogFilter(), LogsearchSecurityContextFormationFilter.class)
+ .addFilterAfter(logsearchServiceLogFilter(), LogsearchSecurityContextFormationFilter.class)
.addFilterBefore(logsearchJwtFilter(), LogsearchSecurityContextFormationFilter.class)
.logout()
.logoutUrl("/logout.html")
@@ -124,6 +152,21 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
}
@Bean
+ public LogsearchServiceLogsStateFilter logsearchServiceLogFilter() {
+ return new LogsearchServiceLogsStateFilter(serviceLogsRequestMatcher(), solrServiceLogsState, solrServiceLogPropsConfig);
+ }
+
+ @Bean
+ public LogsearchAuditLogsStateFilter logsearchAuditLogFilter() {
+ return new LogsearchAuditLogsStateFilter(auditLogsRequestMatcher(), solrAuditLogsState, solrAuditLogPropsConfig);
+ }
+
+ @Bean
+ public LogsearchUserConfigStateFilter logsearchUserConfigFilter() {
+ return new LogsearchUserConfigStateFilter(userConfigRequestMatcher(), solrUserConfigState, solrUserPropsConfig);
+ }
+
+ @Bean
public RequestMatcher requestMatcher() {
List<RequestMatcher> matchers = Lists.newArrayList();
matchers.add(new AntPathRequestMatcher("/login.html"));
@@ -141,4 +184,16 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
return new OrRequestMatcher(matchers);
}
+ public RequestMatcher serviceLogsRequestMatcher() {
+ return new AntPathRequestMatcher("/api/v1/service/logs/**");
+ }
+
+ public RequestMatcher auditLogsRequestMatcher() {
+ return new AntPathRequestMatcher("/api/v1/audit/logs/**");
+ }
+
+ public RequestMatcher userConfigRequestMatcher() {
+ return new AntPathRequestMatcher("/api/v1/userconfig/**");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
index d270b13..ace278f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.conf;
+import org.apache.zookeeper.data.ACL;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@@ -54,6 +55,12 @@ public class SolrAuditLogPropsConfig implements SolrPropsConfig {
@Value("${logsearch.collection.audit.logs.replication.factor:1}")
private Integer replicationFactor;
+ @Value("#{ACLPropertiesSplitter.parseAcls('${logsearch.solr.audit.logs.zk.acls:}')}")
+ private List<ACL> zkAcls;
+
+ @Value("${logsearch.solr.audit.logs.config_set.folder:/etc/ambari-logsearch-portal/conf/solr_configsets}")
+ private String configSetFolder;
+
@Override
public String getSolrUrl() {
return solrUrl;
@@ -124,6 +131,26 @@ public class SolrAuditLogPropsConfig implements SolrPropsConfig {
this.splitInterval = splitInterval;
}
+ @Override
+ public List<ACL> getZkAcls() {
+ return zkAcls;
+ }
+
+ @Override
+ public void setZkAcls(List<ACL> zkAcls) {
+ this.zkAcls = zkAcls;
+ }
+
+ @Override
+ public String getConfigSetFolder() {
+ return configSetFolder;
+ }
+
+ @Override
+ public void setConfigSetFolder(String configSetFolder) {
+ this.configSetFolder = configSetFolder;
+ }
+
public String getRangerCollection() {
return rangerCollection;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
index fb25a63..f00e8c5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java
@@ -18,13 +18,11 @@
*/
package org.apache.ambari.logsearch.conf;
+import org.apache.ambari.logsearch.conf.global.SolrAuditLogsState;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.conf.global.SolrServiceLogsState;
+import org.apache.ambari.logsearch.conf.global.SolrUserConfigState;
import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
@@ -32,52 +30,26 @@ import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.repository.config.EnableSolrRepositories;
import org.springframework.scheduling.annotation.EnableScheduling;
-import javax.inject.Inject;
-
@Configuration
@EnableSolrRepositories
@EnableScheduling
public class SolrConfig {
- private static final Logger LOG = LoggerFactory.getLogger(SolrConfig.class);
-
- @Inject
- private SolrServiceLogPropsConfig solrServiceLogPropsConfig;
-
- @Inject
- private SolrAuditLogPropsConfig solrAuditLogPropsConfig;
-
- @Inject
- private SolrUserPropsConfig solrUserConfigPropsConfig;
-
- @Inject
- private SolrKerberosConfig solrKerberosConfig;
-
@Bean(name = "serviceSolrTemplate")
public SolrTemplate serviceSolrTemplate() {
- setupSecurity();
- return new SolrTemplate(createClient(
- solrServiceLogPropsConfig.getSolrUrl(),
- solrServiceLogPropsConfig.getZkConnectString(),
- solrServiceLogPropsConfig.getCollection()));
+ return null;
}
@Bean(name = "auditSolrTemplate")
@DependsOn("serviceSolrTemplate")
public SolrTemplate auditSolrTemplate() {
- return new SolrTemplate(createClient(
- solrAuditLogPropsConfig.getSolrUrl(),
- solrAuditLogPropsConfig.getZkConnectString(),
- solrAuditLogPropsConfig.getCollection()));
+ return null;
}
@Bean(name = "userConfigSolrTemplate")
@DependsOn("serviceSolrTemplate")
public SolrTemplate userConfigSolrTemplate() {
- return new SolrTemplate(createClient(
- solrUserConfigPropsConfig.getSolrUrl(),
- solrUserConfigPropsConfig.getZkConnectString(),
- solrUserConfigPropsConfig.getCollection()));
+ return null;
}
@Bean
@@ -86,26 +58,19 @@ public class SolrConfig {
return new SolrSchemaFieldDao();
}
- private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) {
- if (StringUtils.isNotEmpty(zookeeperConnectString)) {
- CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString);
- cloudSolrClient.setDefaultCollection(defaultCollection);
- return cloudSolrClient;
- } else if (StringUtils.isNotEmpty(solrUrl)) {
- throw new UnsupportedOperationException("Currently only cloud mode is supported. Set zookeeper connect string.");
- }
- throw new IllegalStateException(
- "Solr url or zookeeper connection string is missing. collection: " + defaultCollection);
+ @Bean(name = "solrServiceLogsState")
+ public SolrCollectionState solrServiceLogsState() {
+ return new SolrServiceLogsState();
+ }
+
+ @Bean(name = "solrAuditLogsState")
+ public SolrCollectionState solrAuditLogsState() {
+ return new SolrAuditLogsState();
}
- private void setupSecurity() {
- String jaasFile = solrKerberosConfig.getJaasFile();
- boolean securityEnabled = solrKerberosConfig.isEnabled();
- if (securityEnabled) {
- System.setProperty("java.security.auth.login.config", jaasFile);
- HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
- LOG.info("setupSecurity() called for kerberos configuration, jaas file: " + jaasFile);
- }
+ @Bean(name = "solrUserConfigState")
+ public SolrCollectionState solrUserConfigState() {
+ return new SolrUserConfigState();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
index 82dd7df..7d37efd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java
@@ -18,8 +18,11 @@
*/
package org.apache.ambari.logsearch.conf;
+import org.apache.zookeeper.data.ACL;
import org.springframework.beans.factory.annotation.Value;
+import java.util.List;
+
public abstract class SolrConnectionPropsConfig implements SolrPropsConfig {
@Value("${logsearch.solr.url:}")
private String solrUrl;
@@ -27,6 +30,12 @@ public abstract class SolrConnectionPropsConfig implements SolrPropsConfig {
@Value("${logsearch.solr.zk_connect_string:}")
private String zkConnectString;
+ @Value("#{ACLPropertiesSplitter.parseAcls('${logsearch.solr.zk.acls:}')}")
+ private List<ACL> zkAcls;
+
+ @Value("${logsearch.solr.config_set.folder:/etc/ambari-logsearch-portal/conf/solr_configsets}")
+ private String configSetFolder;
+
@Override
public String getSolrUrl() {
return solrUrl;
@@ -46,4 +55,24 @@ public abstract class SolrConnectionPropsConfig implements SolrPropsConfig {
public void setZkConnectString(String zkConnectString) {
this.zkConnectString = zkConnectString;
}
+
+ @Override
+ public List<ACL> getZkAcls() {
+ return zkAcls;
+ }
+
+ @Override
+ public void setZkAcls(List<ACL> zkAcls) {
+ this.zkAcls = zkAcls;
+ }
+
+ @Override
+ public String getConfigSetFolder() {
+ return configSetFolder;
+ }
+
+ @Override
+ public void setConfigSetFolder(String configSetFolder) {
+ this.configSetFolder = configSetFolder;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
index f80d6e5..ceddf7e 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java
@@ -18,6 +18,10 @@
*/
package org.apache.ambari.logsearch.conf;
+import org.apache.zookeeper.data.ACL;
+
+import java.util.List;
+
public interface SolrPropsConfig {
String getSolrUrl();
@@ -46,4 +50,12 @@ public interface SolrPropsConfig {
String getSplitInterval();
void setSplitInterval(String splitInterval);
+
+ List<ACL> getZkAcls();
+
+ void setZkAcls(List<ACL> zkAcls);
+
+ String getConfigSetFolder();
+
+ void setConfigSetFolder(String configSetFolder);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java
new file mode 100644
index 0000000..546a5dc
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrAuditLogsState.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.conf.global;
+
+import javax.inject.Named;
+
+@Named
+public class SolrAuditLogsState implements SolrCollectionState {
+
+ private volatile boolean znodeReady;
+ private volatile boolean solrCollectionReady;
+ private volatile boolean solrAliasReady;
+ private volatile boolean configurationUploaded;
+
+ @Override
+ public boolean isZnodeReady() {
+ return znodeReady;
+ }
+
+ @Override
+ public void setZnodeReady(boolean znodeAvailable) {
+ this.znodeReady = znodeAvailable;
+ }
+
+ @Override
+ public boolean isSolrCollectionReady() {
+ return solrCollectionReady;
+ }
+
+ @Override
+ public void setSolrCollectionReady(boolean solrCollectionReady) {
+ this.solrCollectionReady = solrCollectionReady;
+ }
+
+ @Override
+ public boolean isConfigurationUploaded() {
+ return configurationUploaded;
+ }
+
+ @Override
+ public void setConfigurationUploaded(boolean configurationUploaded) {
+ this.configurationUploaded = configurationUploaded;
+ }
+
+ public boolean isSolrAliasReady() {
+ return solrAliasReady;
+ }
+
+ public void setSolrAliasReady(boolean solrAliasReady) {
+ this.solrAliasReady = solrAliasReady;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java
new file mode 100644
index 0000000..5885611
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrCollectionState.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.conf.global;
+
+public interface SolrCollectionState {
+ boolean isZnodeReady();
+
+ void setZnodeReady(boolean znodeAvailable);
+
+ boolean isSolrCollectionReady();
+
+ void setSolrCollectionReady(boolean solrCollectionCreated);
+
+ boolean isConfigurationUploaded();
+
+ void setConfigurationUploaded(boolean configurationUploaded);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java
new file mode 100644
index 0000000..60eafc5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrServiceLogsState.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.conf.global;
+
+import javax.inject.Named;
+
+@Named
+public class SolrServiceLogsState implements SolrCollectionState {
+
+ private volatile boolean znodeReady;
+ private volatile boolean solrCollectionReady;
+ private volatile boolean configurationUploaded;
+
+ @Override
+ public boolean isZnodeReady() {
+ return znodeReady;
+ }
+
+ @Override
+ public void setZnodeReady(boolean znodeAvailable) {
+ this.znodeReady = znodeAvailable;
+ }
+
+ @Override
+ public boolean isSolrCollectionReady() {
+ return solrCollectionReady;
+ }
+
+ @Override
+ public void setSolrCollectionReady(boolean solrCollectionReady) {
+ this.solrCollectionReady = solrCollectionReady;
+ }
+
+ @Override
+ public boolean isConfigurationUploaded() {
+ return configurationUploaded;
+ }
+
+ @Override
+ public void setConfigurationUploaded(boolean configurationUploaded) {
+ this.configurationUploaded = configurationUploaded;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java
new file mode 100644
index 0000000..06af14c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/global/SolrUserConfigState.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.conf.global;
+
+import javax.inject.Named;
+
+@Named
+public class SolrUserConfigState implements SolrCollectionState {
+
+ private volatile boolean znodeReady;
+ private volatile boolean solrCollectionReady;
+ private volatile boolean configurationUploaded;
+
+ @Override
+ public boolean isZnodeReady() {
+ return znodeReady;
+ }
+
+ @Override
+ public void setZnodeReady(boolean znodeAvailable) {
+ this.znodeReady = znodeAvailable;
+ }
+
+ @Override
+ public boolean isSolrCollectionReady() {
+ return solrCollectionReady;
+ }
+
+ @Override
+ public void setSolrCollectionReady(boolean solrCollectionReady) {
+ this.solrCollectionReady = solrCollectionReady;
+ }
+
+ @Override
+ public boolean isConfigurationUploaded() {
+ return configurationUploaded;
+ }
+
+ @Override
+ public void setConfigurationUploaded(boolean configurationUploaded) {
+ this.configurationUploaded = configurationUploaded;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java
new file mode 100644
index 0000000..34e1bec
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/LogfeederFilterConfigurer.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.configurer;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.dao.UserConfigSolrDao;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogfeederFilterConfigurer implements SolrConfigurer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogfeederFilterConfigurer.class);
+
+ private static final int SETUP_RETRY_SECOND = 10;
+
+ private final UserConfigSolrDao userConfigSolrDao;
+
+ public LogfeederFilterConfigurer(final UserConfigSolrDao userConfigSolrDao) {
+ this.userConfigSolrDao = userConfigSolrDao;
+ }
+
+ @Override
+ public void start() {
+ final SolrPropsConfig solrPropsConfig = userConfigSolrDao.getSolrPropsConfig();
+ final SolrCollectionState state = userConfigSolrDao.getSolrCollectionState();
+ Thread setupFiltersThread = new Thread("logfeeder_filter_setup") {
+ @Override
+ public void run() {
+ LOG.info("logfeeder_filter_setup thread started (to upload logfeeder config)");
+ while (true) {
+ int retryCount = 0;
+ try {
+ retryCount++;
+ Thread.sleep(SETUP_RETRY_SECOND * 1000);
+ if (state.isSolrCollectionReady()) {
+ LOG.info("Tries to initialize logfeeder filters in '{}' collection", solrPropsConfig.getCollection());
+ userConfigSolrDao.getUserFilter();
+ break;
+ }
+ } catch (Exception e) {
+ LOG.error("Not able to save logfeeder filter while initialization, retryCount=" + retryCount, e);
+ }
+ }
+ }
+ };
+ setupFiltersThread.setDaemon(true);
+ setupFiltersThread.start();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java
new file mode 100644
index 0000000..c80a10d
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrAuditAliasConfigurer.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.configurer;
+
+import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrAuditLogsState;
+import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.handler.ListCollectionHandler;
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+public class SolrAuditAliasConfigurer implements SolrConfigurer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SolrAuditAliasConfigurer.class);
+
+ private static final int ALIAS_SETUP_RETRY_SECOND = 30 * 60;
+
+ private final AuditSolrDao auditSolrDao;
+
+ public SolrAuditAliasConfigurer(final AuditSolrDao auditSolrDao) {
+ this.auditSolrDao = auditSolrDao;
+ }
+
+ @Override
+ public void start() {
+ final SolrAuditLogPropsConfig solrPropsConfig = (SolrAuditLogPropsConfig) auditSolrDao.getSolrPropsConfig();
+ final SolrAuditLogsState state = (SolrAuditLogsState) auditSolrDao.getSolrCollectionState();
+ final Collection<String> collectionListIn =
+ Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
+
+ if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0) {
+ LOG.info("Will not create alias {} for {}", solrPropsConfig.getAliasNameIn(), collectionListIn.toString());
+ return;
+ }
+
+ LOG.info("setupAlias " + solrPropsConfig.getAliasNameIn() + " for " + collectionListIn.toString());
+ // Start a background thread to do setup
+ Thread setupThread = new Thread("setup_alias_" + solrPropsConfig.getAliasNameIn()) {
+ @Override
+ public void run() {
+ LOG.info("Started monitoring thread to check availability of Solr server. alias=" + solrPropsConfig.getAliasNameIn() +
+ ", collections=" + collectionListIn.toString());
+ int retryCount = 0;
+ while (true) {
+ if (state.isSolrCollectionReady()) {
+ try {
+ CloudSolrClient solrClient = auditSolrDao.getSolrClient();
+ int count = createAlias(solrClient, solrPropsConfig.getAliasNameIn(), collectionListIn);
+ if (count > 0) {
+ solrClient.setDefaultCollection(solrPropsConfig.getAliasNameIn());
+ if (count == collectionListIn.size()) {
+ LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " +
+ "Collections=" + collectionListIn);
+ state.setSolrAliasReady(true);
+ break;
+ }
+ } else {
+ LOG.warn("Not able to create alias=" + solrPropsConfig.getAliasNameIn() + ", retryCount=" + retryCount);
+ }
+ } catch (Exception e) {
+ LOG.error("Error setting up alias=" + solrPropsConfig.getAliasNameIn(), e);
+ }
+ }
+ try {
+ Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000);
+ } catch (InterruptedException sleepInterrupted) {
+ LOG.info("Sleep interrupted while setting up alias " + solrPropsConfig.getAliasNameIn());
+ break;
+ }
+ retryCount++;
+ }
+ }
+ };
+ setupThread.setDaemon(true);
+ setupThread.start();
+ }
+
+ private int createAlias(final CloudSolrClient solrClient, String aliasNameIn, Collection<String> collectionListIn)
+ throws SolrServerException, IOException {
+ List<String> collectionToAdd = new ArrayList<>();
+ try {
+ collectionToAdd = new ListCollectionHandler().handle(solrClient, null);
+ } catch (Exception e) {
+ LOG.error("Invalid state during getting collections for creating alias");
+ }
+ collectionToAdd.retainAll(collectionListIn);
+
+ String collectionsCSV = null;
+ if (!collectionToAdd.isEmpty()) {
+ collectionsCSV = StringUtils.join(collectionToAdd, ',');
+ CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias();
+ aliasCreateRequest.setAliasName(aliasNameIn);
+ aliasCreateRequest.setAliasedCollections(collectionsCSV);
+ CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ LOG.error("Error creating alias. alias=" + aliasNameIn + ", collectionList=" + collectionsCSV
+ + ", response=" + createResponse);
+ return 0;
+ }
+ }
+ if (collectionToAdd.size() == collectionListIn.size()) {
+ LOG.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV);
+ } else {
+ LOG.info("Created alias for " + collectionToAdd.size() + " out of " + collectionListIn.size() + " collections. " +
+ "alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV);
+ }
+ return collectionToAdd.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
new file mode 100644
index 0000000..7edc6aa
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.configurer;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
+import org.apache.ambari.logsearch.handler.ACLHandler;
+import org.apache.ambari.logsearch.handler.CreateCollectionHandler;
+import org.apache.ambari.logsearch.handler.ListCollectionHandler;
+import org.apache.ambari.logsearch.handler.ReloadCollectionHandler;
+import org.apache.ambari.logsearch.handler.UploadConfigurationHandler;
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.data.solr.core.SolrTemplate;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class SolrCollectionConfigurer implements SolrConfigurer {
+
+ private Logger LOG = LoggerFactory.getLogger(SolrCollectionConfigurer.class);
+
+ private static final int SETUP_RETRY_SECOND = 10;
+ private static final int SESSION_TIMEOUT = 15000;
+ private static final int CONNECTION_TIMEOUT = 30000;
+
+ private final SolrDaoBase solrDaoBase;
+
+ public SolrCollectionConfigurer(final SolrDaoBase solrDaoBase) {
+ this.solrDaoBase = solrDaoBase;
+ }
+
+ @Override
+ public void start() {
+ setupSecurity();
+ final SolrPropsConfig solrPropsConfig = solrDaoBase.getSolrPropsConfig();
+ final SolrCollectionState state = solrDaoBase.getSolrCollectionState();
+ final String separator = FileSystems.getDefault().getSeparator();
+ final String localConfigSetLocation = String.format("%s%s%s%sconf", solrPropsConfig.getConfigSetFolder(), separator,
+ solrPropsConfig.getConfigName(), separator);
+ final File configSetFolder = new File(localConfigSetLocation);
+ if (!configSetFolder.exists()) { // show exception only once during startup
+ throw new RuntimeException(String.format("Cannot load config set location: %s", localConfigSetLocation));
+ }
+ Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) {
+ @Override
+ public void run() {
+ LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection());
+ while (!stopSetupCondition(state)) {
+ int retryCount = 0;
+ try {
+ retryCount++;
+ Thread.sleep(SETUP_RETRY_SECOND * 1000);
+ openZkConnectionAndUpdateStatus(state, solrPropsConfig);
+ if (solrDaoBase.getSolrTemplate() == null) {
+ solrDaoBase.setSolrTemplate(createSolrTemplate(solrPropsConfig));
+ }
+ CloudSolrClient cloudSolrClient = (CloudSolrClient) solrDaoBase.getSolrTemplate().getSolrClient();
+ boolean reloadCollectionNeeded = uploadConfigurationsIfNeeded(cloudSolrClient, configSetFolder, state, solrPropsConfig);
+ checkSolrStatus(cloudSolrClient);
+ createCollectionsIfNeeded(cloudSolrClient, state, solrPropsConfig, reloadCollectionNeeded);
+ } catch (Exception e) {
+ retryCount++;
+ LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount, e);
+ }
+ }
+ }
+ };
+ setupThread.setDaemon(true);
+ setupThread.start();
+ }
+
+ private boolean uploadConfigurationsIfNeeded(CloudSolrClient cloudSolrClient, File configSetFolder, SolrCollectionState state, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean reloadCollectionNeeded = new UploadConfigurationHandler(configSetFolder).handle(cloudSolrClient, solrPropsConfig);
+ if (!state.isConfigurationUploaded()) {
+ state.setConfigurationUploaded(true);
+ }
+ return reloadCollectionNeeded;
+ }
+
+ public boolean stopSetupCondition(SolrCollectionState state) {
+ return state.isSolrCollectionReady();
+ }
+
+ public SolrTemplate createSolrTemplate(SolrPropsConfig solrPropsConfig) {
+ return new SolrTemplate(createClient(
+ solrPropsConfig.getSolrUrl(),
+ solrPropsConfig.getZkConnectString(),
+ solrPropsConfig.getCollection()));
+ }
+
+ private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) {
+ if (StringUtils.isNotEmpty(zookeeperConnectString)) {
+ CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString);
+ cloudSolrClient.setDefaultCollection(defaultCollection);
+ return cloudSolrClient;
+ } else if (StringUtils.isNotEmpty(solrUrl)) {
+ throw new UnsupportedOperationException("Currently only cloud mode is supported. Set zookeeper connect string.");
+ }
+ throw new IllegalStateException(
+ "Solr url or zookeeper connection string is missing. collection: " + defaultCollection);
+ }
+
+ private void setupSecurity() {
+ String jaasFile = solrDaoBase.getSolrKerberosConfig().getJaasFile();
+ boolean securityEnabled = solrDaoBase.getSolrKerberosConfig().isEnabled();
+ if (securityEnabled) {
+ System.setProperty("java.security.auth.login.config", jaasFile);
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ LOG.info("setupSecurity() called for kerberos configuration, jaas file: " + jaasFile);
+ }
+ }
+
+ private void openZkConnectionAndUpdateStatus(final SolrCollectionState state, final SolrPropsConfig solrPropsConfig) throws Exception {
+ ZooKeeper zkClient = null;
+ try {
+ LOG.info("Checking that Znode ('{}') is ready or not... ", solrPropsConfig.getZkConnectString());
+ zkClient = openZookeeperConnection(solrPropsConfig);
+ if (!state.isZnodeReady()) {
+ LOG.info("State change: Zookeeper ZNode is available for {}", solrPropsConfig.getZkConnectString());
+ state.setZnodeReady(true);
+ }
+ } catch (Exception e) {
+ LOG.error("Error occurred during the creation of zk client (connection string: {})", solrPropsConfig.getZkConnectString());
+ throw e;
+ } finally {
+ try {
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ } catch (Exception e) {
+ LOG.error("Could not close zk connection properly.", e);
+ }
+ }
+ }
+
+ private ZooKeeper openZookeeperConnection(final SolrPropsConfig solrPropsConfig) throws InterruptedException, IOException {
+ final CountDownLatch connSignal = new CountDownLatch(1);
+ ZooKeeper zooKeeper = new ZooKeeper(solrPropsConfig.getZkConnectString(), SESSION_TIMEOUT, new Watcher() {
+ public void process(WatchedEvent event) {
+ if (event.getState() == Event.KeeperState.SyncConnected) {
+ connSignal.countDown();
+ }
+ }
+ });
+ connSignal.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS);
+ return zooKeeper;
+ }
+
+ private boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
+ int waitDurationMS = 3 * 60 * 1000;
+ boolean status = false;
+ try {
+ long beginTimeMS = System.currentTimeMillis();
+ long waitIntervalMS = 2000;
+ int pingCount = 0;
+ while (true) {
+ pingCount++;
+ try {
+ List<String> collectionList = new ListCollectionHandler().handle(cloudSolrClient, null);
+ if (collectionList != null) {
+ LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList);
+ status = true;
+ break;
+ }
+ } catch (Exception ex) {
+ LOG.error("Error while doing Solr check", ex);
+ }
+ if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
+ LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " +
+ "If you are using alias, then you might have to restart LogSearch after Solr is up and running.");
+ break;
+ } else {
+ LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " +
+ "Will sleep for " + waitIntervalMS + " ms and try again.");
+ }
+ Thread.sleep(waitIntervalMS);
+
+ }
+ } catch (Throwable t) {
+ LOG.error("Seems Solr is not up.");
+ }
+ return status;
+ }
+
+ private void createCollectionsIfNeeded(CloudSolrClient solrClient, SolrCollectionState state, SolrPropsConfig solrPropsConfig, boolean reloadCollectionNeeded) {
+ try {
+ List<String> allCollectionList = new ListCollectionHandler().handle(solrClient, null);
+ boolean collectionCreated = new CreateCollectionHandler(allCollectionList).handle(solrClient, solrPropsConfig);
+ boolean collectionReloaded = true;
+ if (reloadCollectionNeeded) {
+ collectionReloaded = new ReloadCollectionHandler().handle(solrClient, solrPropsConfig);
+ }
+ boolean aclsUpdated = new ACLHandler().handle(solrClient, solrPropsConfig);
+ if (!state.isSolrCollectionReady() && collectionCreated && collectionReloaded && aclsUpdated) {
+ state.setSolrCollectionReady(true);
+ }
+ } catch (Exception ex) {
+ LOG.error("Error during creating/updating collection. collectionName=" + solrPropsConfig.getCollection(), ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java
new file mode 100644
index 0000000..67cb9d1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrConfigurer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.configurer;
+
+interface SolrConfigurer {
+ void start();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
index 309687d..d058383 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
@@ -25,6 +25,10 @@ import javax.inject.Named;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.configurer.SolrAuditAliasConfigurer;
+import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.data.solr.core.SolrTemplate;
@@ -42,13 +46,8 @@ public class AuditSolrDao extends SolrDaoBase {
private SolrTemplate auditSolrTemplate;
@Inject
- private SolrAliasDao solrAliasDao;
-
- @Inject
- private SolrCollectionDao solrCollectionDao;
-
- @Inject
- private SolrSchemaFieldDao solrSchemaFieldDao;
+ @Named("solrAuditLogsState")
+ private SolrCollectionState solrAuditLogsState;
public AuditSolrDao() {
super(LogType.AUDIT);
@@ -59,26 +58,37 @@ public class AuditSolrDao extends SolrDaoBase {
return auditSolrTemplate;
}
+ @Override
+ public void setSolrTemplate(SolrTemplate solrTemplate) {
+ this.auditSolrTemplate = solrTemplate;
+ }
+
@PostConstruct
public void postConstructor() {
String aliasNameIn = solrAuditLogPropsConfig.getAliasNameIn();
String rangerAuditCollection = solrAuditLogPropsConfig.getRangerCollection();
try {
- solrCollectionDao.checkSolrStatus(getSolrClient());
+ new SolrCollectionConfigurer(this).start();
boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection));
- solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig);
if (createAlias) {
- solrAliasDao.setupAlias(getSolrClient(), solrAuditLogPropsConfig);
+ new SolrAuditAliasConfigurer(this).start();
}
- solrSchemaFieldDao.auditCollectionSetUp();
} catch (Exception e) {
LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" +
- solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e);
+ solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e);
}
}
- public SolrSchemaFieldDao getSolrSchemaFieldDao() {
- return solrSchemaFieldDao;
+ @Override
+ public SolrCollectionState getSolrCollectionState() {
+ return this.solrAuditLogsState;
+ }
+
+ @Override
+ public SolrPropsConfig getSolrPropsConfig() {
+ return this.solrAuditLogPropsConfig;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
index cf61163..37375dc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java
@@ -24,7 +24,10 @@ import javax.inject.Inject;
import javax.inject.Named;
import org.apache.ambari.logsearch.common.LogType;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer;
import org.apache.log4j.Logger;
import org.springframework.data.solr.core.SolrTemplate;
@@ -34,17 +37,15 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
private static final Logger LOG = Logger.getLogger(ServiceLogsSolrDao.class);
@Inject
- private SolrCollectionDao solrCollectionDao;
-
- @Inject
private SolrServiceLogPropsConfig solrServiceLogPropsConfig;
@Inject
@Named("serviceSolrTemplate")
- private SolrTemplate serviceSolrTemplate;
+ private volatile SolrTemplate serviceSolrTemplate;
@Inject
- private SolrSchemaFieldDao solrSchemaFieldDao;
+ @Named("solrServiceLogsState")
+ private SolrCollectionState solrServiceLogsState;
public ServiceLogsSolrDao() {
super(LogType.SERVICE);
@@ -55,13 +56,16 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
return serviceSolrTemplate;
}
+ @Override
+ public void setSolrTemplate(SolrTemplate solrTemplate) {
+ this.serviceSolrTemplate = solrTemplate;
+ }
+
@PostConstruct
public void postConstructor() {
LOG.info("postConstructor() called.");
try {
- solrCollectionDao.checkSolrStatus(getSolrClient());
- solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig);
- solrSchemaFieldDao.serviceCollectionSetUp();
+ new SolrCollectionConfigurer(this).start();
} catch (Exception e) {
LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl()
+ ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString()
@@ -69,7 +73,13 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
}
}
- public SolrSchemaFieldDao getSolrSchemaFieldDao() {
- return solrSchemaFieldDao;
+ @Override
+ public SolrCollectionState getSolrCollectionState() {
+ return solrServiceLogsState;
+ }
+
+ @Override
+ public SolrPropsConfig getSolrPropsConfig() {
+ return solrServiceLogPropsConfig;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
deleted file mode 100644
index d78ff0da..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.dao;
-
-import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Named;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-@Named
-class SolrAliasDao {
-
- private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class);
-
- private static final int ALIAS_SETUP_RETRY_SECOND = 30*60;
-
- @Inject
- private SolrCollectionDao solrCollectionDao;
-
- void setupAlias(final CloudSolrClient solrClient, final SolrAuditLogPropsConfig solrPropsConfig) throws Exception {
- final Collection<String> collectionListIn =
- Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
-
- if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) {
- LOG.info("Will not create alias " + solrPropsConfig.getAliasNameIn() + " for " +
- collectionListIn.toString() + ", solrCloudClient=" + solrClient);
- return;
- }
-
- LOG.info("setupAlias " + solrPropsConfig.getAliasNameIn() + " for " + collectionListIn.toString());
- // Start a background thread to do setup
- Thread setupThread = new Thread("setup_alias_" + solrPropsConfig.getAliasNameIn()) {
- @Override
- public void run() {
- LOG.info("Started monitoring thread to check availability of Solr server. alias=" + solrPropsConfig.getAliasNameIn() +
- ", collections=" + collectionListIn.toString());
- int retryCount = 0;
- while (true) {
- try {
- int count = createAlias(solrClient, solrPropsConfig.getAliasNameIn(), collectionListIn);
- if (count > 0) {
- solrClient.setDefaultCollection(solrPropsConfig.getAliasNameIn());
- if (count == collectionListIn.size()) {
- LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " +
- "Collections=" + collectionListIn);
- break;
- }
- } else {
- LOG.warn("Not able to create alias=" + solrPropsConfig.getAliasNameIn() + ", retryCount=" + retryCount);
- }
- } catch (Exception e) {
- LOG.error("Error setting up alias=" + solrPropsConfig.getAliasNameIn(), e);
- }
- try {
- Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000);
- } catch (InterruptedException sleepInterrupted) {
- LOG.info("Sleep interrupted while setting up alias " + solrPropsConfig.getAliasNameIn());
- break;
- }
- retryCount++;
- }
- }
- };
- setupThread.setDaemon(true);
- setupThread.start();
- }
-
- private int createAlias(final CloudSolrClient solrClient, String aliasNameIn, Collection<String> collectionListIn)
- throws SolrServerException, IOException {
- List<String> collectionToAdd = solrCollectionDao.getCollections(solrClient);
- collectionToAdd.retainAll(collectionListIn);
-
- String collectionsCSV = null;
- if (!collectionToAdd.isEmpty()) {
- collectionsCSV = StringUtils.join(collectionToAdd, ',');
- CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias();
- aliasCreateRequest.setAliasName(aliasNameIn);
- aliasCreateRequest.setAliasedCollections(collectionsCSV);
- CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- LOG.error("Error creating alias. alias=" + aliasNameIn + ", collectionList=" + collectionsCSV
- + ", response=" + createResponse);
- return 0;
- }
- }
- if ( collectionToAdd.size() == collectionListIn.size()) {
- LOG.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV);
- } else {
- LOG.info("Created alias for " + collectionToAdd.size() + " out of " + collectionListIn.size() + " collections. " +
- "alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV);
- }
- return collectionToAdd.size();
- }
-}