You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2015/06/04 12:07:11 UTC
ambari git commit: AMBARI-11667. Start NN in HA, Mapreduce,
Oozie and some other are failing with HTTP SPNEGO
configured.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 0dc911e3a -> 2209b7203
AMBARI-11667. Start NN in HA, Mapreduce, Oozie and some other are failing with HTTP SPNEGO configured.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2209b720
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2209b720
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2209b720
Branch: refs/heads/trunk
Commit: 2209b7203d92e25efd0b12fd1eb1bde61a039c72
Parents: 0dc911e
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Thu Jun 4 05:06:27 2015 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Thu Jun 4 05:06:27 2015 +0300
----------------------------------------------------------------------
.../libraries/functions/jmx.py | 19 ++++++++++++++++---
.../libraries/functions/namenode_ha_utils.py | 14 +++++++-------
.../libraries/providers/hdfs_resource.py | 6 ++++--
.../package/scripts/namenode_ha_state.py | 4 ++--
.../2.1.0.2.0/package/scripts/params_linux.py | 4 ++++
.../python/stacks/2.0.6/HDFS/test_journalnode.py | 6 ++++--
6 files changed, 37 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py b/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py
index a3cfb1b..be346e5 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/jmx.py
@@ -19,13 +19,26 @@ limitations under the License.
'''
import urllib2
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+from resource_management.core import shell
+from resource_management.core.logger import Logger
-def get_value_from_jmx(qry, property):
+def get_value_from_jmx(qry, property, security_enabled, run_user, is_https_enabled):
try:
- response = urllib2.urlopen(qry)
- data = response.read()
+ if security_enabled:
+ cmd = ['curl', '--negotiate', '-u', ':']
+ else:
+ cmd = ['curl']
+
+ if is_https_enabled:
+ cmd.append("-k")
+
+ cmd.append(qry)
+
+ _, data = shell.checked_call(cmd, user=run_user, quiet=False)
+
if data:
data_dict = json.loads(data)
return data_dict["beans"][0][property]
except:
+ Logger.exception("Getting jmx metrics from NN failed. URL: " + str(qry))
return None
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
index ab53ba7..0d9e28b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
@@ -31,7 +31,7 @@ NAMENODE_HTTP_FRAGMENT = 'dfs.namenode.http-address.{0}.{1}'
NAMENODE_HTTPS_FRAGMENT = 'dfs.namenode.https-address.{0}.{1}'
JMX_URI_FRAGMENT = "http://{0}/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus"
-def get_namenode_states(hdfs_site):
+def get_namenode_states(hdfs_site, security_enabled, run_user):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
@@ -59,7 +59,7 @@ def get_namenode_states(hdfs_site):
value = str(hdfs_site[key])
jmx_uri = JMX_URI_FRAGMENT.format(value)
- state = get_value_from_jmx(jmx_uri,'State')
+ state = get_value_from_jmx(jmx_uri, 'State', security_enabled, run_user, is_https_enabled)
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
@@ -86,17 +86,17 @@ def is_ha_enabled(hdfs_site):
return False
-def get_active_namenode(hdfs_site):
+def get_active_namenode(hdfs_site, security_enabled, run_user):
"""
return format is nn_unique_id and it's address ('nn1', 'hdfs://hostname1:port1')
"""
- active_namenodes = get_namenode_states(hdfs_site)[0]
+ active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0]
if active_namenodes:
return active_namenodes[0]
else:
return UnknownConfiguration('fs_root')
-def get_property_for_active_namenode(hdfs_site, property_name):
+def get_property_for_active_namenode(hdfs_site, property_name, security_enabled, run_user):
"""
For dfs.namenode.rpc-address:
- In non-ha mode it will return hdfs_site[dfs.namenode.rpc-address]
@@ -104,7 +104,7 @@ def get_property_for_active_namenode(hdfs_site, property_name):
"""
if is_ha_enabled(hdfs_site):
name_service = hdfs_site['dfs.nameservices']
- active_namenodes = get_namenode_states(hdfs_site)[0]
+ active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0]
if not len(active_namenodes):
raise Fail("There is no active namenodes.")
@@ -114,4 +114,4 @@ def get_property_for_active_namenode(hdfs_site, property_name):
return hdfs_site[format("{property_name}.{name_service}.{active_namenode_id}")]
else:
return hdfs_site[property_name]
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 4b1c27c..f9c512f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -120,8 +120,10 @@ class HdfsResourceJar:
class WebHDFSUtil:
def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None):
- https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address')
- http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address')
+ https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address',
+ security_enabled, run_user)
+ http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address',
+ security_enabled, run_user)
self.is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(hdfs_site['dfs.https.enable']) else False
address = https_nn_address if self.is_https_enabled else http_nn_address
http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py
index 50cfc0c..f3185bf 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py
@@ -81,7 +81,7 @@ class NamenodeHAState:
raise Exception("Could not retrieve hostname from address " + actual_value)
jmx_uri = jmx_uri_fragment.format(actual_value)
- state = get_value_from_jmx(jmx_uri, "State")
+ state = get_value_from_jmx(jmx_uri, "State", params.security_enabled, params.hdfs_user, params.is_https_enabled)
if not state:
raise Exception("Could not retrieve Namenode state from URL " + jmx_uri)
@@ -175,4 +175,4 @@ class NamenodeHAState:
"""
active_hosts = self.namenode_state_to_hostnames[NAMENODE_STATE.ACTIVE] if NAMENODE_STATE.ACTIVE in self.namenode_state_to_hostnames else []
standby_hosts = self.namenode_state_to_hostnames[NAMENODE_STATE.STANDBY] if NAMENODE_STATE.STANDBY in self.namenode_state_to_hostnames else []
- return len(active_hosts) == 1 and len(standby_hosts) == 1
\ No newline at end of file
+ return len(active_hosts) == 1 and len(standby_hosts) == 1
http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 077a4f5..c6b10b8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -37,6 +37,7 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.format_jvm_option import format_jvm_option
from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
+from resource_management.libraries.functions.is_empty import is_empty
config = Script.get_config()
@@ -384,6 +385,9 @@ policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
+is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enabled'] if \
+ not is_empty(config['configurations']['hdfs-site']['dfs.https.enabled']) else False
+
if has_ranger_admin:
enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')
xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/2209b720/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
index 39d2f10..646432d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
@@ -283,7 +283,7 @@ class TestJournalnode(RMFTestCase):
namenode_status_standby = open(namenode_status_standby_file, 'r').read()
url_stream_mock = MagicMock()
- url_stream_mock.read.side_effect = [namenode_status_active, namenode_status_standby] + (num_journalnodes * [namenode_jmx, journalnode_jmx])
+ url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx])
urlopen_mock.return_value = url_stream_mock
@@ -291,6 +291,7 @@ class TestJournalnode(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
classname = "JournalNode", command = "post_rolling_restart",
config_file = "journalnode-upgrade.json",
+ checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))],
hdp_stack_version = self.UPGRADE_STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES )
@@ -299,7 +300,7 @@ class TestJournalnode(RMFTestCase):
urlopen_mock.assert_called_with("http://c6407.ambari.apache.org:8480/jmx")
url_stream_mock.reset_mock()
- url_stream_mock.read.side_effect = [namenode_status_active, namenode_status_standby] + (num_journalnodes * [namenode_jmx, journalnode_jmx])
+ url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx])
urlopen_mock.return_value = url_stream_mock
@@ -307,6 +308,7 @@ class TestJournalnode(RMFTestCase):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
classname = "JournalNode", command = "post_rolling_restart",
config_file = "journalnode-upgrade-hdfs-secure.json",
+ checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))],
hdp_stack_version = self.UPGRADE_STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES )