You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/08/05 16:04:47 UTC
[2/2] ambari git commit: AMBARI-12649. Handle !p for password in
commands when run using sudo=True (aonishuk)
AMBARI-12649. Handle !p for password in commands when run using sudo=True (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e6d1ff8a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e6d1ff8a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e6d1ff8a
Branch: refs/heads/branch-2.1
Commit: e6d1ff8a46b67774df850237f9ac492041a0f924
Parents: 58c3c07
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Aug 5 17:04:29 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Aug 5 17:04:29 2015 +0300
----------------------------------------------------------------------
.../TestMonitorWebserverResource.py | 8 +--
.../python/resource_management/core/base.py | 33 +++------
.../python/resource_management/core/logger.py | 76 ++++++++++----------
.../python/resource_management/core/utils.py | 17 +++++
.../libraries/functions/format.py | 5 +-
.../0.4.0/package/scripts/setup_ranger.py | 4 +-
.../0.4.0/package/scripts/setup_ranger_xml.py | 11 +--
.../RANGER_KMS/0.5.0.2.3/package/scripts/kms.py | 9 +--
8 files changed, 84 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-agent/src/test/python/resource_management/TestMonitorWebserverResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestMonitorWebserverResource.py b/ambari-agent/src/test/python/resource_management/TestMonitorWebserverResource.py
index 5cd55a8..a982390 100644
--- a/ambari-agent/src/test/python/resource_management/TestMonitorWebserverResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestMonitorWebserverResource.py
@@ -42,7 +42,7 @@ class TestMonitorWebserverResource(TestCase):
'/etc/httpd/conf/httpd.conf && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E sed -i ' \
'\'s/KeepAlive Off/KeepAlive On/\' /etc/httpd/conf/httpd.conf || echo \'KeepAlive On\' ' \
'| ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E tee --append /etc/httpd/conf/httpd.conf > /dev/null\']' \
- ', Execute[\'(\'/etc/init.d/httpd\', \'start\')\']]'
+ ', Execute[(\'/etc/init.d/httpd\', \'start\')]]'
self.assertEqual(str(defined_resources), expected_resources)
@patch.object(OSCheck, "is_suse_family")
@@ -62,7 +62,7 @@ class TestMonitorWebserverResource(TestCase):
'/etc/apache2/httpd.conf && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E sed -i ' \
'\'s/KeepAlive Off/KeepAlive On/\' /etc/apache2/httpd.conf || echo \'KeepAlive On\' ' \
'| ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E tee --append /etc/apache2/httpd.conf > /dev/null\'],' \
- ' Execute[\'(\'/etc/init.d/apache2\', \'start\')\']]'
+ ' Execute[(\'/etc/init.d/apache2\', \'start\')]]'
self.assertEqual(str(defined_resources), expected_resources)
@patch.object(OSCheck, "is_suse_family")
@@ -76,7 +76,7 @@ class TestMonitorWebserverResource(TestCase):
with Environment(test_mode=True) as env:
MonitorWebserverProvider(MonitorWebserver("stop")).action_stop()
defined_resources = env.resource_list
- expected_resources = '[MonitorWebserver[\'stop\'], Execute[\'(\'/etc/init.d/httpd\', \'stop\')\']]'
+ expected_resources = '[MonitorWebserver[\'stop\'], Execute[(\'/etc/init.d/httpd\', \'stop\')]]'
self.assertEqual(str(defined_resources), expected_resources)
@patch.object(OSCheck, "is_suse_family")
@@ -91,5 +91,5 @@ class TestMonitorWebserverResource(TestCase):
with Environment(test_mode=True) as env:
MonitorWebserverProvider(MonitorWebserver("stop")).action_stop()
defined_resources = env.resource_list
- expected_resources = '[MonitorWebserver[\'stop\'], Execute[\'(\'/etc/init.d/apache2\', \'stop\')\']]'
+ expected_resources = '[MonitorWebserver[\'stop\'], Execute[(\'/etc/init.d/apache2\', \'stop\')]]'
self.assertEqual(str(defined_resources), expected_resources)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-common/src/main/python/resource_management/core/base.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/base.py b/ambari-common/src/main/python/resource_management/core/base.py
index b291769..1711c22 100644
--- a/ambari-common/src/main/python/resource_management/core/base.py
+++ b/ambari-common/src/main/python/resource_management/core/base.py
@@ -26,6 +26,7 @@ __all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
from resource_management.core.exceptions import Fail, InvalidArgument
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
+from resource_management.core.utils import PasswordString
class ResourceArgument(object):
def __init__(self, default=None, required=False):
@@ -41,10 +42,6 @@ class ResourceArgument(object):
raise InvalidArgument("Required argument %s missing" % self.name)
return value
- def log_str(self, key, value):
- return Logger.get_arg_repr(key, value)
-
-
class ForcedListArgument(ResourceArgument):
def validate(self, value):
value = super(ForcedListArgument, self).validate(value)
@@ -65,7 +62,7 @@ class BooleanArgument(ResourceArgument):
class PasswordArgument(ResourceArgument):
def log_str(self, key, value):
# Hide the passwords from text representations
- return "********"
+ return repr(PasswordString(value))
class Accessor(object):
@@ -159,30 +156,16 @@ class Resource(object):
def validate(self):
pass
- def get_function_repr(self):
- name = repr(self)
-
- arguments_str = ""
- for x, y in self.arguments.iteritems():
- try:
- arg = self._arguments[x]
- except KeyError:
- raise Fail("%s received unsupported argument %s" % (self, x))
-
- val = arg.log_str(x, y)
-
- arguments_str += "'{0}': {1}, ".format(x, val)
-
- if arguments_str:
- arguments_str = arguments_str[:-2]
-
- return unicode("{0} {{{1}}}").format(name, arguments_str)
-
def __repr__(self):
return unicode(self)
def __unicode__(self):
- return u"%s['%s']" % (self.__class__.__name__, self.name)
+ if isinstance(self.name, basestring) and not isinstance(self.name, PasswordString):
+ name = "'" + self.name + "'" # print string cutely not with repr
+ else:
+ name = repr(self.name)
+
+ return u"%s[%s]" % (self.__class__.__name__, name)
def __getstate__(self):
return dict(
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-common/src/main/python/resource_management/core/logger.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/logger.py b/ambari-common/src/main/python/resource_management/core/logger.py
index f3ea4e9..7642c6d 100644
--- a/ambari-common/src/main/python/resource_management/core/logger.py
+++ b/ambari-common/src/main/python/resource_management/core/logger.py
@@ -24,6 +24,7 @@ __all__ = ["Logger"]
import sys
import logging
from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+from resource_management.core.utils import PasswordString
MESSAGE_MAX_LEN = 512
DICTIONARY_MAX_LEN = 5
@@ -71,19 +72,19 @@ class Logger:
@staticmethod
def error_resource(resource):
- Logger.error(Logger.filter_text(resource.get_function_repr()))
+ Logger.error(Logger.filter_text(Logger._get_resource_repr(resource)))
@staticmethod
def warning_resource(resource):
- Logger.warning(Logger.filter_text(resource.get_function_repr()))
+ Logger.warning(Logger.filter_text(Logger._get_resource_repr(resource)))
@staticmethod
def info_resource(resource):
- Logger.info(Logger.filter_text(resource.get_function_repr()))
+ Logger.info(Logger.filter_text(Logger._get_resource_repr(resource)))
@staticmethod
def debug_resource(resource):
- Logger.debug(Logger.filter_text(resource.get_function_repr()))
+ Logger.debug(Logger.filter_text(Logger._get_resource_repr(resource)))
@staticmethod
def filter_text(text):
@@ -99,47 +100,48 @@ class Logger:
text = text.replace(placeholder, '')
return text
-
+
@staticmethod
- def get_function_repr(name, arguments):
+ def _get_resource_repr(resource):
+ return Logger.get_function_repr(repr(resource), resource.arguments, resource)
+
+ @staticmethod
+ def get_function_repr(name, arguments, resource=None):
logger_level = logging._levelNames[Logger.logger.level]
arguments_str = ""
for x,y in arguments.iteritems():
- val = Logger.get_arg_repr(x, y)
+ # for arguments which want to override the output
+ if resource and 'log_str' in dir(resource._arguments[x]):
+ val = resource._arguments[x].log_str(x, y)
+ # don't show long arguments
+ elif isinstance(y, basestring) and len(y) > MESSAGE_MAX_LEN:
+ val = '...'
+ # strip unicode 'u' sign
+ elif isinstance(y, unicode):
+ val = repr(y).lstrip('u')
+ # don't show dicts of configurations
+ # usually too long
+ elif isinstance(y, dict) and len(y) > DICTIONARY_MAX_LEN:
+ val = "..."
+ # for configs which didn't come
+ elif isinstance(y, UnknownConfiguration):
+ val = "[EMPTY]"
+ # correctly output 'mode' (as they are octal values like 0755)
+ elif y and x == 'mode':
+ try:
+ val = oct(y)
+ except:
+ val = repr(y)
+ # for functions show only function name
+ elif hasattr(y, '__call__') and hasattr(y, '__name__'):
+ val = y.__name__
+ else:
+ val = repr(y)
arguments_str += "'{0}': {1}, ".format(x, val)
if arguments_str:
arguments_str = arguments_str[:-2]
-
- return unicode("{0} {{{1}}}").format(name, arguments_str)
-
- @staticmethod
- def get_arg_repr(x, y):
- if isinstance(y, basestring) and len(y) > MESSAGE_MAX_LEN:
- y = '...'
- # strip unicode 'u' sign
- if isinstance(y, unicode):
- # don't show long messages
- val = repr(y).lstrip('u')
- # don't show dicts of configurations
- # usually too long
- elif isinstance(y, dict) and len(y) > DICTIONARY_MAX_LEN:
- val = "..."
- # for configs which didn't come
- elif isinstance(y, UnknownConfiguration):
- val = "[EMPTY]"
- # correctly output 'mode' (as they are octal values like 0755)
- elif y and x == 'mode':
- try:
- val = oct(y)
- except:
- val = repr(y)
- # for functions show only function name
- elif hasattr(y, '__call__') and hasattr(y, '__name__'):
- val = y.__name__
- else:
- val = repr(y)
- return val
+ return unicode("{0} {{{1}}}").format(name, arguments_str)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-common/src/main/python/resource_management/core/utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py
index d9f678b..247f068 100644
--- a/ambari-common/src/main/python/resource_management/core/utils.py
+++ b/ambari-common/src/main/python/resource_management/core/utils.py
@@ -25,6 +25,8 @@ import sys
import cStringIO
from resource_management.core.exceptions import Fail
+PASSWORDS_HIDE_STRING = "[PROTECTED]"
+
class AttributeDictionary(object):
def __init__(self, *args, **kwargs):
d = kwargs
@@ -116,3 +118,18 @@ def suppress_stdout():
sys.stdout = cStringIO.StringIO()
yield
sys.stdout = save_stdout
+
+class PasswordString(unicode):
+ """
+ Logger replaces this strings with [PROTECTED]
+ """
+
+ def __init__(self, value):
+ self.value = value
+
+ def __str__(self):
+ return value
+
+ def __repr__(self):
+ return PASSWORDS_HIDE_STRING
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-common/src/main/python/resource_management/libraries/functions/format.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/format.py b/ambari-common/src/main/python/resource_management/libraries/functions/format.py
index 6fdecc0..a137875 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/format.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/format.py
@@ -28,6 +28,7 @@ from resource_management.core.utils import checked_unite
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
from resource_management.core.shell import quote_bash_args
+from resource_management.core import utils
class ConfigurationFormatter(Formatter):
@@ -74,9 +75,9 @@ class ConfigurationFormatter(Formatter):
if conversion == 'e':
return quote_bash_args(unicode(value))
elif conversion == 'h':
- return "[PROTECTED]" if is_protected else value
+ return utils.PASSWORDS_HIDE_STRING if is_protected else value
elif conversion == 'p':
- return "[PROTECTED]" if is_protected else self._convert_field(value, 'e', is_protected)
+ return utils.PASSWORDS_HIDE_STRING if is_protected else self._convert_field(value, 'e', is_protected)
return super(ConfigurationFormatter, self).convert_field(value, conversion)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py
index 71996d2..e1498b6 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py
@@ -92,9 +92,9 @@ def check_db_connnection():
Logger.info('Checking DB connection')
env_dict = {}
if params.db_flavor.lower() == 'mysql':
- cmd = format('{sql_command_invoker} -u {db_root_user} --password={db_root_password} -h {db_host} -s -e "select version();"')
+ cmd = format('{sql_command_invoker} -u {db_root_user} --password={db_root_password!p} -h {db_host} -s -e "select version();"')
elif params.db_flavor.lower() == 'oracle':
- cmd = format('{sql_command_invoker} {db_root_user}/{db_root_password}@{db_host} AS SYSDBA')
+ cmd = format('{sql_command_invoker} {db_root_user}/{db_root_password!p}@{db_host} AS SYSDBA')
env_dict = {'ORACLE_HOME':params.oracle_home, 'LD_LIBRARY_PATH':params.oracle_home}
elif params.db_flavor.lower() == 'postgres':
cmd = 'true'
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py
index 6f6a02b..f20f5c9 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py
@@ -26,6 +26,7 @@ from resource_management.libraries.resources.modify_properties_file import Modif
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.core.utils import PasswordString
# This file contains functions used for setup/configure of Ranger Admin and Ranger Usersync.
# The design is to mimic what is done by the setup.sh script bundled by Ranger component currently.
@@ -183,7 +184,7 @@ def do_keystore_setup(rolling_upgrade=False):
if not is_empty(params.ranger_credential_provider_path):
jceks_path = params.ranger_credential_provider_path
- cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias, '-v', params.ranger_ambari_db_password, '-c', '1')
+ cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_db_password), '-c', '1')
Execute(cred_setup,
environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home},
@@ -197,7 +198,7 @@ def do_keystore_setup(rolling_upgrade=False):
if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password):
jceks_path = params.ranger_credential_provider_path
- cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_audit_jdbc_credential_alias, '-v', params.ranger_ambari_db_password, '-c', '1')
+ cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_audit_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_db_password), '-c', '1')
Execute(cred_setup,
environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home},
logoutput=True,
@@ -239,13 +240,13 @@ def setup_usersync():
cred_lib = os.path.join(params.usersync_home,"lib","*")
cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib)
- cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', params.ranger_usersync_keystore_password, '-c', '1')
+ cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString(params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
- cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', params.ranger_usersync_ldap_ldapbindpassword, '-c', '1')
+ cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString(params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
- cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', params.ranger_usersync_truststore_password, '-c', '1')
+ cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.ugsync_jceks_path,
http://git-wip-us.apache.org/repos/asf/ambari/blob/e6d1ff8a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
index 2551ccd..5cdb1ec 100755
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py
@@ -33,6 +33,7 @@ from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.ranger_functions import Rangeradmin
+from resource_management.core.utils import PasswordString
def setup_kms_db():
import params
@@ -99,7 +100,7 @@ def do_keystore_setup(cred_provider_path, credential_alias, credential_password)
import params
if cred_provider_path is not None:
- cred_setup = params.cred_setup_prefix + ('-f', cred_provider_path, '-k', credential_alias, '-v', credential_password, '-c', '1')
+ cred_setup = params.cred_setup_prefix + ('-f', cred_provider_path, '-k', credential_alias, '-v', PasswordString(credential_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
@@ -270,13 +271,13 @@ def enable_kms_plugin():
mode=0744)
if params.xa_audit_db_is_enabled:
- cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', params.xa_audit_db_password, '-c', '1')
+ cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
- cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', params.ssl_keystore_password, '-c', '1')
+ cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
- cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', params.ssl_truststore_password, '-c', '1')
+ cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.credential_file,