You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2017/02/09 12:37:15 UTC
ambari git commit: AMBARI-19645 Log Search should use Credential
Store API to store keystore/truststore passwords - ambari side (mgergely)
Repository: ambari
Updated Branches:
refs/heads/trunk e9f07973f -> a1bd2987a
AMBARI-19645 Log Search should use Credential Store API to store keystore/truststore passwords - ambari side (mgergely)
Change-Id: I0d7cf0c85f2cb5e1cbfabd681a6f6aab2d66bcb9
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a1bd2987
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a1bd2987
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a1bd2987
Branch: refs/heads/trunk
Commit: a1bd2987aad13f84cc0e93ae7f87c10d11b034d0
Parents: e9f0797
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Thu Feb 9 13:36:03 2017 +0100
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Thu Feb 9 13:37:03 2017 +0100
----------------------------------------------------------------------
.../0.5.0/configuration/logfeeder-env.xml | 2 ++
.../0.5.0/configuration/logsearch-env.xml | 2 ++
.../LOGSEARCH/0.5.0/metainfo.xml | 6 +++++
.../LOGSEARCH/0.5.0/package/scripts/params.py | 22 ++++++++-------
.../0.5.0/package/scripts/setup_logfeeder.py | 28 ++++++--------------
.../0.5.0/package/scripts/setup_logsearch.py | 22 ++++++---------
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 23 ++--------------
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 19 +++----------
8 files changed, 45 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
index 508ef4e..e308479 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
@@ -90,6 +90,7 @@
<description>Password to open the trust store file.</description>
<value-attributes>
<type>password</type>
+ <keystore>true</keystore>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
@@ -115,6 +116,7 @@
<description>Password to open the key store file.</description>
<value-attributes>
<type>password</type>
+ <keystore>true</keystore>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
index 10b21be..f1e871d 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-env.xml
@@ -135,6 +135,7 @@
<description>Password to open the trust store file.</description>
<value-attributes>
<type>password</type>
+ <keystore>true</keystore>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
@@ -160,6 +161,7 @@
<description>Password to open the key store file.</description>
<value-attributes>
<type>password</type>
+ <keystore>true</keystore>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index 8a9105e..5f6ec51 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -24,6 +24,12 @@
<comment>Log aggregation, analysis, and visualization for Ambari managed services. This service is <b>Technical Preview</b>.</comment>
<version>0.5.0</version>
<selection>TECH_PREVIEW</selection>
+
+ <credential-store>
+ <supported>true</supported>
+ <enabled>true</enabled>
+ </credential-store>
+
<components>
<component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 08c0a7b..fecd802 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -18,6 +18,8 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+
+import os
from ambari_commons.constants import AMBARI_SUDO_BINARY
from logsearch_config_aggregator import get_logfeeder_metadata, get_logsearch_metadata, get_logsearch_meta_configs
from resource_management.libraries.functions.default import default
@@ -54,7 +56,6 @@ security_enabled = status_params.security_enabled
logsearch_server_conf = "/etc/ambari-logsearch-portal/conf"
logsearch_server_keys_folder = logsearch_server_conf + "/keys"
logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf"
-logsearch_logfeeder_keys_folder = logsearch_logfeeder_conf + "/keys"
logsearch_config_set_dir = format("{logsearch_server_conf}/solr_configsets")
@@ -168,6 +169,14 @@ logsearch_debug_enabled = str(config['configurations']['logsearch-env']["logsear
logsearch_debug_port = config['configurations']['logsearch-env']["logsearch_debug_port"]
logsearch_app_max_memory = config['configurations']['logsearch-env']['logsearch_app_max_memory']
+logsearch_keystore_location = config['configurations']['logsearch-env']['logsearch_keystore_location']
+logsearch_keystore_type = config['configurations']['logsearch-env']['logsearch_keystore_type']
+logsearch_truststore_location = config['configurations']['logsearch-env']['logsearch_truststore_location']
+logsearch_truststore_type = config['configurations']['logsearch-env']['logsearch_truststore_type']
+
+logsearch_env_config = dict(config['configurations']['logsearch-env'])
+logsearch_env_jceks_file = os.path.join(logsearch_server_conf, 'logsearch.jceks')
+
#Logsearch log4j properties
logsearch_log_maxfilesize = default('/configurations/logsearch-log4j/logsearch_log_maxfilesize',10)
logsearch_log_maxbackupindex = default('/configurations/logsearch-log4j/logsearch_log_maxbackupindex',10)
@@ -296,19 +305,14 @@ solr_audit_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_a
logfeeder_env_content = config['configurations']['logfeeder-env']['content']
logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content']
-logsearch_keystore_location = config['configurations']['logsearch-env']['logsearch_keystore_location']
-logsearch_keystore_password = config['configurations']['logsearch-env']['logsearch_keystore_password']
-logsearch_keystore_type = config['configurations']['logsearch-env']['logsearch_keystore_type']
-logsearch_truststore_location = config['configurations']['logsearch-env']['logsearch_truststore_location']
-logsearch_truststore_password = config['configurations']['logsearch-env']['logsearch_truststore_password']
-logsearch_truststore_type = config['configurations']['logsearch-env']['logsearch_truststore_type']
logfeeder_keystore_location = config['configurations']['logfeeder-env']['logfeeder_keystore_location']
-logfeeder_keystore_password = config['configurations']['logfeeder-env']['logfeeder_keystore_password']
logfeeder_keystore_type = config['configurations']['logfeeder-env']['logfeeder_keystore_type']
logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfeeder_truststore_location']
-logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password']
logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type']
+logfeeder_env_config = dict(config['configurations']['logfeeder-env'])
+logfeeder_env_jceks_file = os.path.join(logsearch_logfeeder_conf, 'logfeeder.jceks')
+
logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content']
logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content']
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index a04618f..6952c2c 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -22,6 +22,7 @@ from resource_management.core.resources.system import Directory, File
from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate, Template
from resource_management.libraries.resources.properties_file import PropertiesFile
+from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME
def setup_logfeeder():
import params
@@ -39,31 +40,18 @@ def setup_logfeeder():
recursive_ownership=True
)
- Directory(params.logsearch_logfeeder_keys_folder,
- cd_access='a',
- mode=0755,
- owner=params.logsearch_user,
- group=params.user_group)
-
- File(format("{logsearch_logfeeder_keys_folder}/ks_pass.txt"),
- content=params.logfeeder_keystore_password,
- mode=0600,
- owner=params.logsearch_user,
- group=params.user_group
- )
-
- File(format("{logsearch_logfeeder_keys_folder}/ts_pass.txt"),
- content=params.logfeeder_truststore_password,
- mode=0600,
- owner=params.logsearch_user,
- group=params.user_group
- )
-
File(params.logfeeder_log,
mode=0644,
content=''
)
+ params.logfeeder_env_config = update_credential_provider_path(params.logfeeder_env_config,
+ 'logfeeder-env',
+ params.logfeeder_env_jceks_file,
+ params.logsearch_user,
+ params.user_group
+ )
+ params.logfeeder_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logfeeder_env_jceks_file
PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
properties = params.logfeeder_properties
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 08d3d9d..ba91e20 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -21,6 +21,7 @@ from resource_management.core.resources.system import Directory, Execute, File
from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate, Template
from resource_management.libraries.resources.properties_file import PropertiesFile
+from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME
def setup_logsearch():
@@ -49,20 +50,6 @@ def setup_logsearch():
owner=params.logsearch_user,
group=params.user_group)
- File(format("{logsearch_server_keys_folder}/ks_pass.txt"),
- content=params.logsearch_keystore_password,
- mode=0600,
- owner=params.logsearch_user,
- group=params.user_group
- )
-
- File(format("{logsearch_server_keys_folder}/ts_pass.txt"),
- content=params.logsearch_truststore_password,
- mode=0600,
- owner=params.logsearch_user,
- group=params.user_group
- )
-
File(params.logsearch_log,
mode=0644,
owner=params.logsearch_user,
@@ -70,6 +57,13 @@ def setup_logsearch():
content=''
)
+ params.logsearch_env_config = update_credential_provider_path(params.logsearch_env_config,
+ 'logsearch-env',
+ params.logsearch_env_jceks_file,
+ params.logsearch_user,
+ params.user_group
+ )
+ params.logsearch_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logsearch_env_jceks_file
PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"),
properties=params.logsearch_properties
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 1c79c5c..00e8e1f 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -55,33 +55,14 @@ class TestLogFeeder(RMFTestCase):
cd_access='a',
mode=0755
)
- self.assertResourceCalled('Directory', '/etc/ambari-logsearch-logfeeder/conf/keys',
- owner = 'logsearch',
- group = 'hadoop',
- cd_access = 'a',
- mode = 0755
- )
-
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/keys/ks_pass.txt',
- owner='logsearch',
- group='hadoop',
- mode=0600,
- content='bigdata'
- )
-
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/keys/ts_pass.txt',
- owner='logsearch',
- group='hadoop',
- mode=0600,
- content='bigdata'
- )
self.assertResourceCalled('File', '/var/log/ambari-logsearch-logfeeder/logfeeder.out',
mode=0644,
content=''
)
self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
- properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
+ properties={'hadoop.security.credential.provider.path': 'jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.jceks',
+ 'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
'logfeeder.config.files': 'output.config.json,input.config-ambari.json,global.config.json,input.config-logsearch.json,input.config-zookeeper.json',
'logfeeder.metrics.collector.hosts': '',
'logfeeder.metrics.collector.path': '/ws/v1/timeline/metrics',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1bd2987/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index f63cd42..380151c 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -70,20 +70,6 @@ class TestLogSearch(RMFTestCase):
mode = 0755
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/keys/ks_pass.txt',
- owner='logsearch',
- group='hadoop',
- mode=0600,
- content='bigdata'
- )
-
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/keys/ts_pass.txt',
- owner='logsearch',
- group='hadoop',
- mode=0600,
- content='bigdata'
- )
-
self.assertResourceCalled('File', '/var/log/ambari-logsearch-portal/logsearch.out',
owner = 'logsearch',
group = 'hadoop',
@@ -91,7 +77,8 @@ class TestLogSearch(RMFTestCase):
content = ''
)
self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
- properties = {'logsearch.audit.logs.split.interval.mins': '1',
+ properties = {'hadoop.security.credential.provider.path': 'jceks://file/etc/ambari-logsearch-portal/conf/logsearch-env.jceks',
+ 'logsearch.audit.logs.split.interval.mins': '1',
'logsearch.auth.external_auth.enabled': 'false',
'logsearch.auth.external_auth.host_url': 'http://c6401.ambari.apache.org:8080',
'logsearch.auth.external_auth.login_url': '/api/v1/users/$USERNAME/privileges?fields=*',
@@ -152,6 +139,8 @@ class TestLogSearch(RMFTestCase):
self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/etc/ambari-logsearch-portal/conf/solr_configsets'),
sudo = True
)
+
+
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logsearch.py",