You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/12/07 18:38:34 UTC
ambari git commit: AMBARI-19121. Add ability for users to provide
mount exclusion list before cluster deployment (Dmytro Grinenko via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 ebe954bb1 -> c346320e8
AMBARI-19121. Add ability for users to provide mount exclusion list before cluster deployment (Dmytro Grinenko via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c346320e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c346320e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c346320e
Branch: refs/heads/branch-2.5
Commit: c346320e8f59e86997feea2b1a15726c542f8267
Parents: ebe954b
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Dec 7 10:35:05 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Dec 7 10:35:05 2016 -0800
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 1 +
.../src/main/python/ambari_agent/Hardware.py | 52 ++++++++++++++-
.../TestCustomServiceOrchestrator.py | 4 +-
.../test/python/ambari_agent/TestHardware.py | 70 ++++++++++++++++++++
4 files changed, 122 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c346320e/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index 43740ad..c1d4c02 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -39,6 +39,7 @@ alert_kinit_timeout=14400000
system_resource_overrides=/etc/resource_overrides
; memory_threshold_soft_mb=400
; memory_threshold_hard_mb=1000
+; ignore_mount_points=/mnt/custom1,/mnt/custom2
[security]
keysdir=/var/lib/ambari-agent/keys
http://git-wip-us.apache.org/repos/asf/ambari/blob/c346320e/ambari-agent/src/main/python/ambari_agent/Hardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Hardware.py b/ambari-agent/src/main/python/ambari_agent/Hardware.py
index 3c94d28..0d431a3 100644
--- a/ambari-agent/src/main/python/ambari_agent/Hardware.py
+++ b/ambari-agent/src/main/python/ambari_agent/Hardware.py
@@ -41,6 +41,7 @@ class Hardware:
CHECK_REMOTE_MOUNTS_TIMEOUT_DEFAULT = '10'
IGNORE_ROOT_MOUNTS = ["proc", "dev", "sys"]
IGNORE_DEVICES = ["proc", "tmpfs", "cgroup", "mqueue", "shm"]
+ LINUX_PATH_SEP = "/"
def __init__(self, config):
self.hardware = {
@@ -88,6 +89,37 @@ class Hardware:
return True
@classmethod
+ def _is_mount_blacklisted(cls, blacklist, mount_point):
+ """
+ Verify if particular mount point is in the black list.
+
+ :return True if mount_point or a part of mount point is in the blacklist, otherwise return False
+
+ Example:
+ Mounts: /, /mnt/my_mount, /mnt/my_mount/sub_mount
+ Blacklist: /mnt/my_mount
+ Result: /
+
+ :type blacklist list
+ :type mount_point str
+ :rtype bool
+ """
+
+ if not blacklist or not mount_point:
+ return False
+
+ mount_point_elements = mount_point.split(cls.LINUX_PATH_SEP)
+
+ for el in blacklist:
+ el_list = el.split(cls.LINUX_PATH_SEP)
+ # making patch elements comparision
+ if el_list == mount_point_elements[:len(el_list)]:
+ return True
+
+ return False
+
+
+ @classmethod
@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
def osdisks(cls, config=None):
""" Run df to find out the disks on the host. Only works on linux
@@ -95,6 +127,11 @@ class Hardware:
and any mounts with spaces. """
timeout = cls._get_mount_check_timeout(config)
command = ["timeout", timeout, "df", "-kPT"]
+ blacklisted_mount_points = []
+
+ if config:
+ ignore_mount_value = config.get("agent", "ignore_mount_points", default="")
+ blacklisted_mount_points = [item.strip() for item in ignore_mount_value.split(",")]
if not cls._check_remote_mounts(config):
command.append("-l")
@@ -103,6 +140,7 @@ class Hardware:
dfdata = df.communicate()[0]
mounts = [cls._parse_df_line(line) for line in dfdata.splitlines() if line]
result_mounts = []
+ ignored_mounts = []
for mount in mounts:
if not mount:
@@ -113,13 +151,21 @@ class Hardware:
- mounted device is not in the ignored list
- is accessible to user under which current process running
- it is not file-mount (docker environment)
+ - mount path or a part of mount path is not in the blacklist
"""
- if mount["device"] not in cls.IGNORE_DEVICES and \
+ if mount["device"] not in cls.IGNORE_DEVICES and\
mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\
- cls._chk_writable_mount(mount['mountpoint']) and \
- not path_isfile(mount["mountpoint"]):
+ cls._chk_writable_mount(mount['mountpoint']) and\
+ not path_isfile(mount["mountpoint"]) and\
+ not cls._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]):
result_mounts.append(mount)
+ else:
+ ignored_mounts.append(mount)
+
+ if len(ignored_mounts) > 0:
+ ignore_list = [el["mountpoint"] for el in ignored_mounts]
+ logger.info("Some mount points was ignored: {0}".format(', '.join(ignore_list)))
return result_mounts
http://git-wip-us.apache.org/repos/asf/ambari/blob/c346320e/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 5323d9a..563d250 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -68,7 +68,7 @@ class TestCustomServiceOrchestrator(TestCase):
def test_add_reg_listener_to_controller(self, FileCache_mock):
FileCache_mock.return_value = None
dummy_controller = MagicMock()
- config = AmbariConfig().getConfig()
+ config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
CustomServiceOrchestrator(config, dummy_controller)
@@ -204,7 +204,7 @@ class TestCustomServiceOrchestrator(TestCase):
def test_resolve_script_path(self, FileCache_mock, exists_mock):
FileCache_mock.return_value = None
dummy_controller = MagicMock()
- config = AmbariConfig().getConfig()
+ config = AmbariConfig()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
# Testing existing path
exists_mock.return_value = True
http://git-wip-us.apache.org/repos/asf/ambari/blob/c346320e/ambari-agent/src/test/python/ambari_agent/TestHardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
index 038b2f8..ff3b40b 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
@@ -25,6 +25,7 @@ from mock.mock import patch, MagicMock, Mock
import unittest
import platform
import socket
+import os
from only_for_platform import not_for_platform, PLATFORM_WINDOWS
from ambari_agent import hostname
from ambari_agent.Hardware import Hardware
@@ -373,6 +374,75 @@ SwapFree: 1598676 kB
self.assertEquals(2, json_mock.call_count)
self.assertEquals('value', result['key'])
+ @patch.object(Hardware, "_chk_writable_mount")
+ @patch("ambari_agent.Hardware.path_isfile")
+ def test_osdisks_blacklist(self, isfile_mock, chk_writable_mount_mock):
+ df_output = \
+ """Filesystem Type 1024-blocks Used Available Capacity Mounted on
+ /dev/mapper/docker-253:0-4980899-d45c264d37ab18c8ed14f890f4d59ac2b81e1c52919eb36a79419787209515f3 xfs 31447040 1282384 30164656 5% /
+ tmpfs tmpfs 32938336 4 32938332 1% /dev
+ tmpfs tmpfs 32938336 0 32938336 0% /sys/fs/cgroup
+ /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/resolv.conf
+ /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hostname
+ /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hosts
+ shm tmpfs 65536 0 65536 0% /dev/shm
+ /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /run/secrets
+ /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /mnt/blacklisted_mount
+ /dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /mnt/blacklisted_mount/sub-dir
+ """
+
+ def isfile_side_effect(path):
+ assume_files = ["/etc/resolv.conf", "/etc/hostname", "/etc/hosts"]
+ return path in assume_files
+
+ def chk_writable_mount_side_effect(path):
+ assume_read_only = ["/run/secrets"]
+ return path not in assume_read_only
+
+ isfile_mock.side_effect = isfile_side_effect
+ chk_writable_mount_mock.side_effect = chk_writable_mount_side_effect
+
+ config_dict = {
+ "agent": {
+ "ignore_mount_points": "/mnt/blacklisted_mount"
+ }
+ }
+
+ with patch("subprocess.Popen") as open_mock:
+ proc_mock = Mock()
+ attr = {
+ 'communicate.return_value': [
+ df_output
+ ]
+ }
+ proc_mock.configure_mock(**attr)
+ open_mock.return_value = proc_mock
+
+ def conf_get(section, key, default=""):
+ if section in config_dict and key in config_dict[section]:
+ return config_dict[section][key]
+
+ return default
+
+ def has_option(section, key):
+ return section in config_dict and key in config_dict[section]
+
+ conf = Mock()
+ attr = {
+ 'get.side_effect': conf_get,
+ 'has_option.side_effect': has_option
+ }
+ conf.configure_mock(**attr)
+
+ result = Hardware.osdisks(conf)
+
+ self.assertEquals(1, len(result))
+
+ expected_mounts_left = ["/"]
+ mounts_left = [item["mountpoint"] for item in result]
+
+ self.assertEquals(expected_mounts_left, mounts_left)
+
if __name__ == "__main__":
unittest.main()