You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/04/28 17:26:52 UTC
ambari git commit: AMBARI-16162. Reduce NN start time by removing
redundant haadmin calls. (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 126626857 -> 35ebd80d8
AMBARI-16162. Reduce NN start time by removing redundant haadmin calls. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/35ebd80d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/35ebd80d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/35ebd80d
Branch: refs/heads/trunk
Commit: 35ebd80d87e2a46c1dfcccc3a243bf59fb3c510c
Parents: 1266268
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Apr 28 18:26:49 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Apr 28 18:26:49 2016 +0300
----------------------------------------------------------------------
.../2.1.0.2.0/package/scripts/hdfs_namenode.py | 19 ++++++-----
.../package/scripts/setup_ranger_hdfs.py | 6 ++--
.../python/stacks/2.0.6/HDFS/test_namenode.py | 36 ++------------------
3 files changed, 16 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/35ebd80d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
index 357eb71..635f159 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
@@ -180,6 +180,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None,
# EU non-HA | no change | no check |
check_for_safemode_off = False
+ is_active_namenode = False
msg = ""
if params.dfs_ha_enabled:
if upgrade_type is not None:
@@ -187,14 +188,16 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None,
msg = "Must wait to leave safemode since High Availability is enabled during a Stack Upgrade"
else:
Logger.info("Wait for NameNode to become active.")
- if is_active_namenode(hdfs_binary): # active
+ if check_is_active_namenode(hdfs_binary): # active
check_for_safemode_off = True
+ is_active_namenode = True
msg = "Must wait to leave safemode since High Availability is enabled and this is the Active NameNode."
else:
msg = "Will remain in the current safemode state."
else:
msg = "Must wait to leave safemode since High Availability is not enabled."
check_for_safemode_off = True
+ is_active_namenode = True
Logger.info(msg)
@@ -209,8 +212,11 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None,
wait_for_safemode_off(hdfs_binary)
# Always run this on non-HA, or active NameNode during HA.
- create_hdfs_directories(is_active_namenode_cmd)
- create_ranger_audit_hdfs_directories(is_active_namenode_cmd)
+ if is_active_namenode:
+ create_hdfs_directories()
+ create_ranger_audit_hdfs_directories()
+ else:
+ Logger.info("Skipping creating hdfs directories as is not active NN.")
elif action == "stop":
import params
@@ -267,7 +273,7 @@ def create_name_dirs(directories):
)
-def create_hdfs_directories(check):
+def create_hdfs_directories():
import params
params.HdfsResource(params.hdfs_tmp_dir,
@@ -275,18 +281,15 @@ def create_hdfs_directories(check):
action="create_on_execute",
owner=params.hdfs_user,
mode=0777,
- only_if=check
)
params.HdfsResource(params.smoke_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.smoke_user,
mode=params.smoke_hdfs_user_mode,
- only_if=check
)
params.HdfsResource(None,
action="execute",
- only_if=check #skip creation when HA not active
)
def format_namenode(force=None):
@@ -508,7 +511,7 @@ def is_namenode_bootstrapped(params):
return marked
-def is_active_namenode(hdfs_binary):
+def check_is_active_namenode(hdfs_binary):
"""
Checks if current NameNode is active. Waits up to 30 seconds. If other NameNode is active returns False.
:return: True if current NameNode is active, False otherwise
http://git-wip-us.apache.org/repos/asf/ambari/blob/35ebd80d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
index 040c69c..72eb3ad 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py
@@ -95,7 +95,7 @@ def setup_ranger_hdfs(upgrade_type=None):
else:
Logger.info('Ranger admin not installed')
-def create_ranger_audit_hdfs_directories(check):
+def create_ranger_audit_hdfs_directories():
import params
if params.has_ranger_admin:
@@ -107,7 +107,6 @@ def create_ranger_audit_hdfs_directories(check):
group=params.hdfs_user,
mode=0755,
recursive_chmod=True,
- only_if=check
)
params.HdfsResource("/ranger/audit/hdfs",
type="directory",
@@ -116,8 +115,7 @@ def create_ranger_audit_hdfs_directories(check):
group=params.hdfs_user,
mode=0700,
recursive_chmod=True,
- only_if=check
)
- params.HdfsResource(None, action="execute", only_if=check)
+ params.HdfsResource(None, action="execute")
else:
Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/35ebd80d/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index f56a0d2..34b639e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -100,7 +100,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if=True,
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'wasb://abc@c6401.ambari.apache.org',
@@ -118,7 +117,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if=True,
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'wasb://abc@c6401.ambari.apache.org',
@@ -136,7 +134,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if=True,
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'wasb://abc@c6401.ambari.apache.org',
@@ -218,7 +215,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = True,
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://c6401.ambari.apache.org:8020',
@@ -236,7 +232,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = True,
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://c6401.ambari.apache.org:8020',
@@ -254,7 +249,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = True,
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://c6401.ambari.apache.org:8020',
@@ -361,8 +355,7 @@ class TestNamenode(RMFTestCase):
hadoop_conf_dir = '/etc/hadoop/conf',
type = 'directory',
action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
- mode = 0777,
- only_if = True
+ mode = 0777
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
@@ -376,13 +369,11 @@ class TestNamenode(RMFTestCase):
hadoop_conf_dir = '/etc/hadoop/conf',
type = 'directory',
action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
- mode = 0770,
- only_if = True
+ mode = 0770
)
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = True,
- only_if = True,
keytab = '/etc/security/keytabs/hdfs.headless.keytab',
hadoop_bin_dir = '/usr/bin',
kinit_path_local = '/usr/bin/kinit',
@@ -453,7 +444,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -471,7 +461,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -489,7 +478,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -555,7 +543,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -573,7 +560,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -591,7 +577,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -663,7 +648,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = True,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = '/etc/security/keytabs/hdfs.headless.keytab',
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -681,7 +665,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = True,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = '/etc/security/keytabs/hdfs.headless.keytab',
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -699,7 +682,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = True,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = '/etc/security/keytabs/hdfs.headless.keytab',
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -771,7 +753,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -789,7 +770,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -807,7 +787,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -881,7 +860,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -899,7 +877,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -917,7 +894,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -999,7 +975,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -1017,7 +992,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -1035,7 +1009,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -1115,7 +1088,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/tmp',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -1133,7 +1105,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -1151,7 +1122,6 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('HdfsResource', None,
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
- only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'",
keytab = UnknownConfigurationMock(),
hadoop_bin_dir = '/usr/bin',
default_fs = 'hdfs://ns1',
@@ -1592,7 +1562,7 @@ class TestNamenode(RMFTestCase):
self.assertTrue(calls[0].startsWith("conf-select create-conf-dir --package hadoop --stack-version 2.3.2.0-2844 --conf-version 0"))
- @patch("hdfs_namenode.is_active_namenode")
+ @patch("hdfs_namenode.check_is_active_namenode")
@patch("resource_management.libraries.functions.setup_ranger_plugin_xml.setup_ranger_plugin")
@patch("utils.get_namenode_states")
def test_upgrade_restart_eu_with_ranger(self, get_namenode_states_mock, setup_ranger_plugin_mock, is_active_nn_mock):