You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2017/04/06 12:22:15 UTC
ambari git commit: AMBARI-20670 Node manager start extremely slow
when YARN NM local dirs are very large (dgrinenko via dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk c11d00452 -> 28238ae84
AMBARI-20670 Node manager start extremely slow when YARN NM local dirs are very large (dgrinenko via dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/28238ae8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/28238ae8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/28238ae8
Branch: refs/heads/trunk
Commit: 28238ae84e24aa72613e59b6af341f1c2452a0aa
Parents: c11d004
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Apr 6 15:22:10 2017 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Thu Apr 6 15:22:10 2017 +0300
----------------------------------------------------------------------
.../python/resource_management/TestUtils.py | 39 +++++++++++++
.../python/resource_management/core/sudo.py | 58 +++++++++++++++-----
.../python/resource_management/core/utils.py | 56 ++++++++++++++++++-
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 16 ++++--
.../YARN/3.0.0.3.0/package/scripts/yarn.py | 14 +++--
5 files changed, 159 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-agent/src/test/python/resource_management/TestUtils.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestUtils.py b/ambari-agent/src/test/python/resource_management/TestUtils.py
new file mode 100644
index 0000000..0fbae3e
--- /dev/null
+++ b/ambari-agent/src/test/python/resource_management/TestUtils.py
@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import stat
+from unittest import TestCase
+from resource_management.core.utils import attr_to_bitmask
+
+
+class TestUtils(TestCase):
+
+ def test_attr_to_bitmask(self):
+ test_set = [
+ ["+r", stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, 0],
+ ["u+w", stat.S_IWUSR, 0],
+ ["uo+x", stat.S_IXUSR | stat.S_IXOTH, 0],
+ ["-x", stat.S_IRUSR, stat.S_IXUSR | stat.S_IXOTH | stat.S_IRUSR],
+ ["=x", stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP, stat.S_IRUSR | stat.S_IRGRP]
+ ]
+
+ for test in test_set:
+ test_pattern, expected, initial_val = test
+ bitmask = attr_to_bitmask(test_pattern, initial_bitmask= initial_val)
+ self.assertEquals(expected, bitmask, "Test set \"{0}\" failed, expected: {1} but got {2}".format(
+ test_pattern, expected, bitmask))
http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-common/src/main/python/resource_management/core/sudo.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py
index d6fd71f..c350161 100644
--- a/ambari-common/src/main/python/resource_management/core/sudo.py
+++ b/ambari-common/src/main/python/resource_management/core/sudo.py
@@ -30,6 +30,7 @@ from resource_management.core import shell
from resource_management.core.exceptions import Fail
import subprocess
+from resource_management.core.utils import attr_to_bitmask
if os.geteuid() == 0:
def chown(path, owner, group):
@@ -54,16 +55,45 @@ if os.geteuid() == 0:
def chmod(path, mode):
+ """
+ Wrapper around python function
+
+ :type path str
+ :type mode int
+ """
return os.chmod(path, mode)
- mode_to_stat = {"a+x": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, "a+rx": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, "u+x": stat.S_IXUSR, "g+x": stat.S_IXGRP, "o+x": stat.S_IXOTH}
+
def chmod_extended(path, mode):
- if mode in mode_to_stat:
- st = os.stat(path)
- os.chmod(path, st.st_mode | mode_to_stat[mode])
- else:
- shell.checked_call(["chmod", mode, path])
-
+ """
+ :type path str
+ :type mode str
+ """
+ st = os.stat(path)
+ os.chmod(path, attr_to_bitmask(mode, initial_bitmask=st.st_mode))
+
+ def chmod_recursive(path, recursive_mode_flags, recursion_follow_links=False):
+ """
+ Change recursively permissions on directories or files
+
+ :type path str
+ :type recursive_mode_flags
+ :type recursion_follow_links bool
+ """
+ dir_attrib = recursive_mode_flags["d"] if "d" in recursive_mode_flags else None
+ files_attrib = recursive_mode_flags["f"] if "d" in recursive_mode_flags else None
+
+ for root, dirs, files in os.walk(path, followlinks=recursion_follow_links):
+ if dir_attrib is not None:
+ for dir_name in dirs:
+ full_dir_path = os.path.join(root, dir_name)
+ chmod(full_dir_path, attr_to_bitmask(dir_attrib, initial_bitmask=os.stat(full_dir_path).st_mode))
+
+ if files_attrib is not None:
+ for file_name in files:
+ full_file_path = os.path.join(root, file_name)
+ chmod(full_file_path, attr_to_bitmask(files_attrib, initial_bitmask=os.stat(full_file_path).st_mode))
+
def copy(src, dst):
shutil.copy(src, dst)
@@ -278,10 +308,10 @@ else:
return files
-def chmod_recursive(path, recursive_mode_flags, recursion_follow_links):
- find_flags = []
- if recursion_follow_links:
- find_flags.append('-L')
-
- for key, flags in recursive_mode_flags.iteritems():
- shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod", flags ,"{}" ,";"])
+ def chmod_recursive(path, recursive_mode_flags, recursion_follow_links):
+ find_flags = []
+ if recursion_follow_links:
+ find_flags.append('-L')
+
+ for key, flags in recursive_mode_flags.iteritems():
+ shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod", flags ,"{}" ,";"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-common/src/main/python/resource_management/core/utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py
index 265b2f2..53f6b8b 100644
--- a/ambari-common/src/main/python/resource_management/core/utils.py
+++ b/ambari-common/src/main/python/resource_management/core/utils.py
@@ -27,10 +27,16 @@ import sys
import signal
import cStringIO
from functools import wraps
+
+import re
+
from resource_management.core.exceptions import Fail
from itertools import chain, repeat, islice
PASSWORDS_HIDE_STRING = "[PROTECTED]"
+PERM_STRING_REGEXP = re.compile("(?P<scope>[ugoa]*)(?P<direction>[-+=])(?P<attr>[rwx]*)")
+PERM_REGISTER = {"u": 0o100, "g": 0o010, "o": 0o001}
+PERM_BITS = {"r": 0o004, "w": 0o002, "x": 0o001}
class AttributeDictionary(object):
def __init__(self, *args, **kwargs):
@@ -157,4 +163,52 @@ def pad_infinite(iterable, padding=None):
return chain(iterable, repeat(padding))
def pad(iterable, size, padding=None):
- return islice(pad_infinite(iterable, padding), size)
\ No newline at end of file
+ return islice(pad_infinite(iterable, padding), size)
+
+
+def attr_to_bitmask(attr, initial_bitmask=0o0):
+ """
+ Function able to generate permission bits from passed named permission string (chmod like style)
+
+ Supports:
+ - scope modifications: u,g,o or a
+ - setting mode: +,-,-
+ - attributes: r,x,w
+
+ Samples:
+ uo+rw, a+x, u-w, o=r
+
+ :type attr str
+ :type initial_bitmask int
+ """
+ attr_dict = {"scope": "", "direction": "", "attr": ""}
+ re_match_result = PERM_STRING_REGEXP.match(attr)
+
+ if re_match_result:
+ attr_dict = re_match_result.groupdict(default=attr_dict)
+
+ if attr_dict["scope"] == "":
+ attr_dict["scope"] = "a"
+
+ if "a" in attr_dict["scope"]:
+ attr_dict["scope"] = "ugo"
+
+ attr_dict["scope"] = list(attr_dict["scope"])
+ attr_dict["attr"] = list(attr_dict["attr"])
+
+ if attr_dict["direction"] == "=":
+ clear_mask = 0o0
+ for scope in attr_dict["scope"]:
+ clear_mask = clear_mask | 0o007 * PERM_REGISTER[scope]
+
+ initial_bitmask = initial_bitmask ^ (initial_bitmask & clear_mask)
+ attr_dict["direction"] = "+"
+
+ for scope in attr_dict["scope"]:
+ for attr in attr_dict["attr"]:
+ if attr_dict["direction"] == "-" and (initial_bitmask & (PERM_BITS[attr] * PERM_REGISTER[scope])) > 0:
+ initial_bitmask = initial_bitmask ^ (PERM_BITS[attr] * PERM_REGISTER[scope])
+ elif attr_dict["direction"] == "+":
+ initial_bitmask = initial_bitmask | (PERM_BITS[attr] * PERM_REGISTER[scope])
+
+ return initial_bitmask
http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 204ab56..52338df 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -347,7 +347,6 @@ def setup_nodemanager():
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
-
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params)
@@ -459,17 +458,24 @@ def create_log_dir(dir_name):
ignore_failures=True,
)
+
def create_local_dir(dir_name):
import params
+
+ directory_args = {}
+
+ if params.toggle_nm_security:
+ directory_args["recursive_mode_flags"] = {'f': 'a+rw', 'd': 'a+rwx'}
+
Directory(dir_name,
- create_parents = True,
+ create_parents=True,
cd_access="a",
mode=0755,
owner=params.yarn_user,
group=params.user_group,
ignore_failures=True,
- recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
- )
+ **directory_args
+ )
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
def yarn(name = None):
@@ -500,4 +506,4 @@ def yarn(name = None):
ServiceConfig(service_name,
action="change_user",
username = params.yarn_user,
- password = Script.get_password(params.yarn_user))
\ No newline at end of file
+ password = Script.get_password(params.yarn_user))
http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
index ae1b425..d601f8f 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
@@ -347,7 +347,6 @@ def setup_nodemanager():
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
-
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params)
@@ -459,16 +458,23 @@ def create_log_dir(dir_name):
ignore_failures=True,
)
+
def create_local_dir(dir_name):
import params
+
+ directory_args = {}
+
+ if params.toggle_nm_security:
+ directory_args["recursive_mode_flags"] = {'f': 'a+rw', 'd': 'a+rwx'}
+
Directory(dir_name,
- create_parents = True,
+ create_parents=True,
cd_access="a",
mode=0755,
owner=params.yarn_user,
group=params.user_group,
ignore_failures=True,
- recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
+ **directory_args
)
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
@@ -500,4 +506,4 @@ def yarn(name = None):
ServiceConfig(service_name,
action="change_user",
username = params.yarn_user,
- password = Script.get_password(params.yarn_user))
\ No newline at end of file
+ password = Script.get_password(params.yarn_user))