You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/05/10 13:37:27 UTC
[04/22] ambari git commit: AMBARI-20953. Insecure SSL: Server
Identity Verification Disabled (echekanskiy)
AMBARI-20953. Insecure SSL: Server Identity Verification Disabled (echekanskiy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/84586cc0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/84586cc0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/84586cc0
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 84586cc07860b9bf3434c07b427e3c8a067a6351
Parents: e20305b
Author: Eugene Chekanskiy <ec...@hortonworks.com>
Authored: Mon May 8 22:33:17 2017 +0300
Committer: Eugene Chekanskiy <ec...@hortonworks.com>
Committed: Mon May 8 22:33:17 2017 +0300
----------------------------------------------------------------------
.../main/python/ambari_agent/AmbariConfig.py | 67 +++++++++++---------
.../ambari_agent/CustomServiceOrchestrator.py | 3 +-
.../src/main/python/ambari_agent/NetUtil.py | 5 +-
.../python/ambari_agent/alerts/web_alert.py | 5 +-
.../main/python/ambari_commons/inet_utils.py | 43 +++++++++----
.../libraries/functions/curl_krb_request.py | 17 ++++-
.../libraries/script/script.py | 18 +++++-
.../HDFS/2.1.0.2.0/package/scripts/utils.py | 5 +-
8 files changed, 114 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index 4118ece..95e4712 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -27,6 +27,7 @@ import os
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
logger = logging.getLogger(__name__)
content = """
@@ -74,9 +75,8 @@ log_command_executes = 0
""".format(ps=os.sep)
-
servicesToPidNames = {
- 'GLUSTERFS' : 'glusterd.pid$',
+ 'GLUSTERFS': 'glusterd.pid$',
'NAMENODE': 'hadoop-{USER}-namenode.pid$',
'SECONDARY_NAMENODE': 'hadoop-{USER}-secondarynamenode.pid$',
'DATANODE': 'hadoop-{USER}-datanode.pid$',
@@ -97,13 +97,13 @@ servicesToPidNames = {
'KERBEROS_SERVER': 'kadmind.pid',
'HIVE_SERVER': 'hive-server.pid',
'HIVE_METASTORE': 'hive.pid',
- 'HIVE_SERVER_INTERACTIVE' : 'hive-interactive.pid',
+ 'HIVE_SERVER_INTERACTIVE': 'hive-interactive.pid',
'MYSQL_SERVER': 'mysqld.pid',
'HUE_SERVER': '/var/run/hue/supervisor.pid',
'WEBHCAT_SERVER': 'webhcat.pid',
}
-#Each service, which's pid depends on user should provide user mapping
+# Each service, which's pid depends on user should provide user mapping
servicesToLinuxUser = {
'NAMENODE': 'hdfs_user',
'SECONDARY_NAMENODE': 'hdfs_user',
@@ -120,30 +120,30 @@ servicesToLinuxUser = {
}
pidPathVars = [
- {'var' : 'glusterfs_pid_dir_prefix',
- 'defaultValue' : '/var/run'},
- {'var' : 'hadoop_pid_dir_prefix',
- 'defaultValue' : '/var/run/hadoop'},
- {'var' : 'hadoop_pid_dir_prefix',
- 'defaultValue' : '/var/run/hadoop'},
- {'var' : 'hbase_pid_dir',
- 'defaultValue' : '/var/run/hbase'},
- {'var' : 'zk_pid_dir',
- 'defaultValue' : '/var/run/zookeeper'},
- {'var' : 'oozie_pid_dir',
- 'defaultValue' : '/var/run/oozie'},
- {'var' : 'hcat_pid_dir',
- 'defaultValue' : '/var/run/webhcat'},
- {'var' : 'hive_pid_dir',
- 'defaultValue' : '/var/run/hive'},
- {'var' : 'mysqld_pid_dir',
- 'defaultValue' : '/var/run/mysqld'},
- {'var' : 'hcat_pid_dir',
- 'defaultValue' : '/var/run/webhcat'},
- {'var' : 'yarn_pid_dir_prefix',
- 'defaultValue' : '/var/run/hadoop-yarn'},
- {'var' : 'mapred_pid_dir_prefix',
- 'defaultValue' : '/var/run/hadoop-mapreduce'},
+ {'var': 'glusterfs_pid_dir_prefix',
+ 'defaultValue': '/var/run'},
+ {'var': 'hadoop_pid_dir_prefix',
+ 'defaultValue': '/var/run/hadoop'},
+ {'var': 'hadoop_pid_dir_prefix',
+ 'defaultValue': '/var/run/hadoop'},
+ {'var': 'hbase_pid_dir',
+ 'defaultValue': '/var/run/hbase'},
+ {'var': 'zk_pid_dir',
+ 'defaultValue': '/var/run/zookeeper'},
+ {'var': 'oozie_pid_dir',
+ 'defaultValue': '/var/run/oozie'},
+ {'var': 'hcat_pid_dir',
+ 'defaultValue': '/var/run/webhcat'},
+ {'var': 'hive_pid_dir',
+ 'defaultValue': '/var/run/hive'},
+ {'var': 'mysqld_pid_dir',
+ 'defaultValue': '/var/run/mysqld'},
+ {'var': 'hcat_pid_dir',
+ 'defaultValue': '/var/run/webhcat'},
+ {'var': 'yarn_pid_dir_prefix',
+ 'defaultValue': '/var/run/hadoop-yarn'},
+ {'var': 'mapred_pid_dir_prefix',
+ 'defaultValue': '/var/run/hadoop-mapreduce'},
]
@@ -323,7 +323,7 @@ class AmbariConfig:
if reg_resp and AmbariConfig.AMBARI_PROPERTIES_CATEGORY in reg_resp:
if not self.has_section(AmbariConfig.AMBARI_PROPERTIES_CATEGORY):
self.add_section(AmbariConfig.AMBARI_PROPERTIES_CATEGORY)
- for k,v in reg_resp[AmbariConfig.AMBARI_PROPERTIES_CATEGORY].items():
+ for k, v in reg_resp[AmbariConfig.AMBARI_PROPERTIES_CATEGORY].items():
self.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, k, v)
logger.info("Updating config property (%s) with value (%s)", k, v)
pass
@@ -345,6 +345,15 @@ class AmbariConfig:
import ssl
return getattr(ssl, self.get_force_https_protocol_name())
+ def get_ca_cert_file_path(self):
+ """
+ Get path to file with trusted certificates.
+
+ :return: trusted certificates file path
+ """
+ return self.get('security', 'ca_cert_path', default="")
+
+
def isSameHostList(hostlist1, hostlist2):
is_same = True
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 4c13eb3..7dd00de 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -82,6 +82,7 @@ class CustomServiceOrchestrator():
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
self.force_https_protocol = config.get_force_https_protocol_name()
+ self.ca_cert_file_path = config.get_ca_cert_file_path()
self.exec_tmp_dir = Constants.AGENT_TMP_DIR
self.file_cache = FileCache(config)
self.status_commands_stdout = os.path.join(self.tmp_dir,
@@ -395,7 +396,7 @@ class CustomServiceOrchestrator():
for py_file, current_base_dir in filtered_py_file_list:
log_info_on_failure = not command_name in self.DONT_DEBUG_FAILURES_FOR_COMMANDS
script_params = [command_name, json_path, current_base_dir, tmpstrucoutfile, logger_level, self.exec_tmp_dir,
- self.force_https_protocol]
+ self.force_https_protocol, self.ca_cert_file_path]
if log_out_files:
script_params.append("-o")
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-agent/src/main/python/ambari_agent/NetUtil.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/NetUtil.py b/ambari-agent/src/main/python/ambari_agent/NetUtil.py
index fc19605..fe32efe 100644
--- a/ambari-agent/src/main/python/ambari_agent/NetUtil.py
+++ b/ambari-agent/src/main/python/ambari_agent/NetUtil.py
@@ -29,7 +29,10 @@ LOG_REQUEST_MESSAGE = "GET %s -> %s, body: %s"
logger = logging.getLogger(__name__)
-ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol_name())
+ensure_ssl_using_protocol(
+ AmbariConfig.get_resolved_config().get_force_https_protocol_name(),
+ AmbariConfig.get_resolved_config().get_ca_cert_file_path()
+)
class NetUtil:
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
index 79b1c3b..8ce4405 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
@@ -55,7 +55,10 @@ DEFAULT_CONNECTION_TIMEOUT = 5
WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg')
-ensure_ssl_using_protocol(AmbariConfig.get_resolved_config().get_force_https_protocol_name())
+ensure_ssl_using_protocol(
+ AmbariConfig.get_resolved_config().get_force_https_protocol_name(),
+ AmbariConfig.get_resolved_config().get_ca_cert_file_path()
+)
class WebAlert(BaseAlert):
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-common/src/main/python/ambari_commons/inet_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index d44107d..66f6544 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -183,23 +183,42 @@ def resolve_address(address):
return '127.0.0.1'
return address
-def ensure_ssl_using_protocol(protocol):
+def ensure_ssl_using_protocol(protocol="PROTOCOL_TLSv1", ca_certs=None):
"""
Monkey patching ssl module to force it use tls_v1. Do this in common module to avoid problems with
PythonReflectiveExecutor.
+
:param protocol: one of ("PROTOCOL_SSLv2", "PROTOCOL_SSLv3", "PROTOCOL_SSLv23", "PROTOCOL_TLSv1", "PROTOCOL_TLSv1_1", "PROTOCOL_TLSv1_2")
+ :param ca_certs: path to ca_certs file
:return:
"""
from functools import wraps
import ssl
- if hasattr(ssl.wrap_socket, "_ambari_patched"):
- return # do not create chain of wrappers, patch only once
- def sslwrap(func):
- @wraps(func)
- def bar(*args, **kw):
- import ssl
- kw['ssl_version'] = getattr(ssl, protocol)
- return func(*args, **kw)
- bar._ambari_patched = True
- return bar
- ssl.wrap_socket = sslwrap(ssl.wrap_socket)
+
+ if not hasattr(ssl.wrap_socket, "_ambari_patched"):
+ def sslwrap(func):
+ @wraps(func)
+ def bar(*args, **kw):
+ import ssl
+ kw['ssl_version'] = getattr(ssl, protocol)
+ if ca_certs and not 'ca_certs' in kw:
+ kw['ca_certs'] = ca_certs
+ kw['cert_reqs'] = ssl.CERT_REQUIRED
+ return func(*args, **kw)
+ bar._ambari_patched = True
+ return bar
+ ssl.wrap_socket = sslwrap(ssl.wrap_socket)
+
+ # python 2.7 stuff goes here
+ if hasattr(ssl, "_create_default_https_context"):
+ if not hasattr(ssl._create_default_https_context, "_ambari_patched"):
+ @wraps(ssl._create_default_https_context)
+ def _create_default_https_context_patched():
+ context = ssl.SSLContext(protocol = getattr(ssl, protocol))
+ if ca_certs:
+ context.load_verify_locations(ca_certs)
+ context.verify_mode = ssl.CERT_REQUIRED
+ context.check_hostname = False
+ return context
+ _create_default_https_context_patched._ambari_patched = True
+ ssl._create_default_https_context = _create_default_https_context_patched
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py b/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py
index 557db58..72bc5c6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py
@@ -62,6 +62,7 @@ KERBEROS_KINIT_TIMER_PARAMETER = "kerberos.kinit.timer"
def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix,
krb_exec_search_paths, return_only_http_code, caller_label, user,
connection_timeout = CONNECTION_TIMEOUT_DEFAULT,
+ ca_certs = None,
kinit_timer_ms=DEFAULT_KERBEROS_KINIT_TIMER_MS, method = '',body='',header=''):
"""
Makes a curl request using the kerberos credentials stored in a calculated cache file. The
@@ -84,13 +85,20 @@ def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix,
:param caller_label: an identifier to give context into the caller of this module (used for logging)
:param user: the user to invoke the curl command as
:param connection_timeout: if specified, a connection timeout for curl (default 10 seconds)
+ :param ca_certs: path to certificates
:param kinit_timer_ms: if specified, the time (in ms), before forcing a kinit even if the
klist cache is still valid.
:return:
"""
import uuid
-
+ # backward compatibility with old code and management packs, etc. All new code need pass ca_certs explicitly
+ if ca_certs is None:
+ try:
+ from ambari_agent.AmbariConfig import AmbariConfig
+ ca_certs = AmbariConfig.get_resolved_config().get_ca_cert_file_path()
+ except:
+ pass
# start off false
is_kinit_required = False
@@ -174,13 +182,16 @@ def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix,
connection_timeout = int(connection_timeout)
maximum_timeout = connection_timeout + 2
+ ssl_options = ['-k']
+ if ca_certs:
+ ssl_options = ['--cacert', ca_certs]
try:
if return_only_http_code:
- _, curl_stdout, curl_stderr = get_user_call_output(['curl', '--location-trusted', '-k', '--negotiate', '-u', ':', '-b', cookie_file, '-c', cookie_file, '-w',
+ _, curl_stdout, curl_stderr = get_user_call_output(['curl', '--location-trusted'] + ssl_options + ['--negotiate', '-u', ':', '-b', cookie_file, '-c', cookie_file, '-w',
'%{http_code}', url, '--connect-timeout', str(connection_timeout), '--max-time', str(maximum_timeout), '-o', '/dev/null'],
user=user, env=kerberos_env)
else:
- curl_command = ['curl', '--location-trusted', '-k', '--negotiate', '-u', ':', '-b', cookie_file, '-c', cookie_file,
+ curl_command = ['curl', '--location-trusted'] + ssl_options + ['--negotiate', '-u', ':', '-b', cookie_file, '-c', cookie_file,
url, '--connect-timeout', str(connection_timeout), '--max-time', str(maximum_timeout)]
# returns response body
if len(method) > 0 and len(body) == 0 and len(header) == 0:
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 0dd9c02..bcad6c3 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -36,6 +36,7 @@ from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING
from ambari_commons.constants import UPGRADE_TYPE_ROLLING
from ambari_commons.constants import UPGRADE_TYPE_HOST_ORDERED
from ambari_commons.network import reconfigure_urllib2_opener
+from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from resource_management.libraries.resources import XmlConfig
from resource_management.libraries.resources import PropertiesFile
@@ -151,6 +152,7 @@ class Script(object):
# Class variable
tmp_dir = ""
force_https_protocol = "PROTOCOL_TLSv1"
+ ca_cert_file_path = None
def load_structured_out(self):
Script.structuredOut = {}
@@ -277,9 +279,11 @@ class Script(object):
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
- # optional script argument for forcing https protocol
+ # optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
+ if len(sys.argv) >= 9:
+ Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
@@ -297,6 +301,9 @@ class Script(object):
Script.structuredOut = {}
self.put_structured_out({})
+ # make sure that script has forced https protocol and ca_certs file passed from agent
+ ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
+
try:
with open(self.command_data_file) as f:
pass
@@ -536,6 +543,15 @@ class Script(object):
return getattr(ssl, Script.get_force_https_protocol_name())
@staticmethod
+ def get_ca_cert_file_path():
+ """
+ Get path to file with trusted certificates.
+
+ :return: trusted certificates file path
+ """
+ return Script.ca_cert_file_path
+
+ @staticmethod
def get_component_from_role(role_directory_map, default_role):
"""
Gets the <stack-root>/current/<component> component given an Ambari role,
http://git-wip-us.apache.org/repos/asf/ambari/blob/84586cc0/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
index c9b63ff..d861ba9 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
@@ -38,7 +38,10 @@ from resource_management.libraries.functions.show_logs import show_logs
from ambari_commons.inet_utils import ensure_ssl_using_protocol
from zkfc_slave import ZkfcSlaveDefault
-ensure_ssl_using_protocol(Script.get_force_https_protocol_name())
+ensure_ssl_using_protocol(
+ Script.get_force_https_protocol_name(),
+ Script.get_ca_cert_file_path()
+)
def safe_zkfc_op(action, env):
"""